tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

mail handling 1st working version

Browse Source
This commit is contained in:
dockes
2005-03-31 10:04:07 +00:00
parent d392d317bb
commit 04b279dcd5
12 changed files with 373 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/internfile/mh_html.cpp
+13 -8
View File
@@ -56,7 +56,18 @@ MimeHandlerHtml::worker1(RclConfig *conf, const string &,
const string& htext,
const string &mtype, Rcl::Doc &docout)
{
// Character set handling:
//LOGDEB(("textHtmlToDoc: htext: %s\n", htext.c_str()));
// Character set handling: the initial guessed charset depends on
// external factors: possible hint (ie mime charset in a mail
// message), charset guessing, or default configured charset.
string charset;
if (!charsethint.empty()) {
charset = charsethint;
if (conf->getGuessCharset()) {
charset = csguess(htext, conf->getDefCharset());
} else
charset = conf->getDefCharset();
}
// - We first try to convert from the default configured charset
// (which may depend of the current directory) to utf-8. If this
@@ -64,12 +75,6 @@ MimeHandlerHtml::worker1(RclConfig *conf, const string &,
// - During parsing, if we find a charset parameter, and it differs from
// what we started with, we abort and restart with the parameter value
// instead of the configuration one.
string charset;
if (conf->getGuessCharset()) {
charset = csguess(htext, conf->getDefCharset());
} else
charset = conf->getDefCharset();
LOGDEB(("textHtmlToDoc: charset before parsing: %s\n", charset.c_str()));
MyHtmlParser pres;
@@ -108,7 +113,7 @@ MimeHandlerHtml::worker1(RclConfig *conf, const string &,
Rcl::Doc out;
out.origcharset = charset;
out.text = pres.dump;
// LOGDEB(("textHtmlToDoc: dump : %s\n", pres.dump.c_str()));
// LOGDEB(("textHtmlToDoc: dump : %s\n", pres.dump.c_str()));
out.title = pres.title;
out.keywords = pres.keywords;
out.abstract = pres.sample;
src/internfile/mh_html.h
+3 -1
View File
@@ -1,7 +1,8 @@
#ifndef _HTML_H_INCLUDED_
#define _HTML_H_INCLUDED_
/* @(#$Id: mh_html.h,v 1.3 2005-03-25 09:40:27 dockes Exp $ (C) 2004 J.F.Dockes */
/* @(#$Id: mh_html.h,v 1.4 2005-03-31 10:04:07 dockes Exp $ (C) 2004 J.F.Dockes */
#include "mimehandler.h"
#include <string>
// Code to turn an html document into an internal one. There are 2
// interfaces, depending if we're working on a file, or on a
@@ -11,6 +12,7 @@
// carry titles, abstracts, whatever)
class MimeHandlerHtml : public MimeHandler {
public:
std::string charsethint;
virtual MimeHandler::Status worker(RclConfig *conf, const string &fn,
const string &mtype, Rcl::Doc &docout, string&);
virtual MimeHandler::Status worker1(RclConfig *conf, const string &fn,
src/internfile/mh_mail.cpp
+179 -42
View File
@@ -1,11 +1,14 @@
#ifndef lint
static char rcsid[] = "@(#$Id: mh_mail.cpp,v 1.1 2005-03-25 09:40:27 dockes Exp $ (C) 2005 J.F.Dockes";
static char rcsid[] = "@(#$Id: mh_mail.cpp,v 1.2 2005-03-31 10:04:07 dockes Exp $ (C) 2005 J.F.Dockes";
#endif
#include <stdio.h>
#include <fcntl.h>
#include <errno.h>
#include <map>
#include <sstream>
using std::stringstream;
using std::map;
#include "mimehandler.h"
@@ -19,65 +22,183 @@ using std::map;
#include "debuglog.h"
#include "smallut.h"
#include "mimeparse.h"
#include "html.h"
// binc imap mime definitions
#include "mime.h"
static void
walkmime(RclConfig *cnf, string &out, Binc::MimePart& doc, int depth);
using namespace std;
MimeHandlerMail::~MimeHandlerMail()
{
if (vfp) {
fclose((FILE *)vfp);
vfp = 0;
}
}
// We are called for two different file types: mbox-type folders
// holding multiple messages, and maildir-type files with one rfc822
// message
// holding multiple messages, and maildir-type files with one message
MimeHandler::Status
MimeHandlerMail::worker(RclConfig *cnf, const string &fn,
const string &mtype, Rcl::Doc &docout, string&)
const string &mtype, Rcl::Doc &docout, string& ipath)
{
LOGDEB(("MimeHandlerMail::worker: %s [%s]\n", mtype.c_str(), fn.c_str()));
conf = cnf;
if (!stringlowercmp("message/rfc822", mtype)) {
return processone(fn, docout);
ipath = "";
int fd;
if ((fd = open(fn.c_str(), 0)) < 0) {
LOGERR(("MimeHandlerMail::worker: open(%s) errno %d\n",
fn.c_str(), errno));
return MimeHandler::MHError;
}
Binc::MimeDocument doc;
doc.parseFull(fd);
MimeHandler::Status ret = processone(fn, doc, docout);
close(fd);
return ret;
} else if (!stringlowercmp("text/x-mail", mtype)) {
return MimeHandler::MHError;
} else
return processmbox(fn, docout, ipath);
} else // hu ho
return MimeHandler::MHError;
}
MimeHandler::Status
MimeHandlerMail::processmbox(const string &fn, Rcl::Doc &docout, string& ipath)
{
int mtarg = 0;
if (ipath != "") {
sscanf(ipath.c_str(), "%d", &mtarg);
}
LOGDEB(("MimeHandlerMail::processmbox: fn %s, mtarg %d\n", fn.c_str(),
mtarg));
#include "mime.h"
FILE *fp;
if (vfp) {
fp = (FILE *)vfp;
} else {
fp = fopen(fn.c_str(), "r");
if (fp == 0) {
LOGERR(("MimeHandlerMail::processmbox: error opening %s\n",
fn.c_str()));
return MimeHandler::MHError;
}
vfp = fp;
}
if (mtarg > 0) {
fseek(fp, 0, SEEK_SET);
msgnum = 0;
}
const char *hnames[] = {"Subject", "Content-type"};
int nh = sizeof(hnames) / sizeof(char *);
off_t start, end;
bool iseof = false;
do {
// Look for next 'From ' Line, start of message. Set start to
// line after this
char line[301];
for (;;) {
if (!fgets(line, 300, fp)) {
// Eof hit while looking for 'From ' -> file done. We'd need
// another return code here
return MimeHandler::MHError;
}
if (!strncmp("From ", line, 5)) {
start = ftello(fp);
break;
}
}
// Look for next 'From ' line or eof, end of message (we let a
// spurious empty line in)
for (;;) {
end = ftello(fp);
if (!fgets(line, 300, fp) || !strncmp("From ", line, 5)) {
if (ferror(fp) || feof(fp))
iseof = true;
break;
}
}
msgnum++;
LOGDEB(("MimeHandlerMail::processmbox: got msg %d\n", msgnum));
fseek(fp, end, SEEK_SET);
} while (mtarg > 0 && msgnum < mtarg);
size_t size = end - start;
fseek(fp, start, SEEK_SET);
char *cp = (char *)malloc(size);
if (cp == 0) {
LOGERR(("MimeHandlerMail::processmbox: malloc(%d) failed\n", size));
return MimeHandler::MHError;
}
if (fread(cp, 1, size, fp) != size) {
LOGERR(("MimeHandlerMail::processmbox: fread failed (errno %d)\n",
errno));
free(cp);
return MimeHandler::MHError;
}
string msgbuf(cp, size);
free(cp);
stringstream s(msgbuf);
Binc::MimeDocument doc;
doc.parseFull(s);
MimeHandler::Status ret = processone(fn, doc, docout);
if (ret == MimeHandler::MHError)
return ret;
char buf[20];
sprintf(buf, "%d", msgnum);
ipath = buf;
return iseof ? MimeHandler::MHDone :
(mtarg > 0) ? MimeHandler::MHDone : MimeHandler::MHAgain;
}
void walkmime(string &out, Binc::MimePart& doc, int fd, int depth);
// Transform a single message into a document. The subject becomes the
// title, and any simple body part with a content-type of text or html
// and content-disposition inline gets concatenated as text.
MimeHandler::Status
MimeHandlerMail::processone(const string &fn, Rcl::Doc &docout)
MimeHandlerMail::processone(const string &fn, Binc::MimeDocument& doc,
Rcl::Doc &docout)
{
int fd;
if ((fd = open(fn.c_str(), 0)) < 0) {
LOGERR(("MimeHandlerMail::processone: open(%s) errno %d\n",
fn.c_str(), errno));
return MimeHandler::MHError;
}
Binc::MimeDocument doc;
doc.parseFull(fd);
if (!doc.isHeaderParsed() && !doc.isAllParsed()) {
LOGERR(("MimeHandlerMail::processone: parse error for %s\n",
LOGERR(("MimeHandlerMail::processone: mime parse error for %s\n",
fn.c_str()));
close(fd);
return MimeHandler::MHError;
}
// Handle some headers. We should process rfc2047 encoding here
Binc::HeaderItem hi;
if (doc.h.getFirstHeader("Subject", hi)) {
docout.title = hi.getValue();
}
if (doc.h.getFirstHeader("From", hi)) {
docout.text += string("From: ") + hi.getValue() + string("\n");
}
if (doc.h.getFirstHeader("To", hi)) {
docout.text += string("To: ") + hi.getValue() + string("\n");
}
if (doc.h.getFirstHeader("Date", hi)) {
docout.text += string("Date: ") + hi.getValue() + string("\n");
}
LOGDEB(("MimeHandlerMail::processone: ismultipart %d mime subtype '%s'\n",
doc.isMultipart(), doc.getSubType().c_str()));
walkmime(docout.text, doc, fd, 0);
close(fd);
LOGDEB(("MimeHandlerMail::processone: text: '%s'\n", docout.text.c_str()));
return MimeHandler::MHError;
walkmime(conf, docout.text, doc, 0);
LOGDEB(("MimeHandlerMail::processone: text: '%s'\n", docout.text.c_str()));
return MimeHandler::MHDone;
}
void walkmime(string &out, Binc::MimePart& doc, int fd, int depth)
// Recursively walk the message mime parts and concatenate all the
// inline html or text that we find anywhere.
static void walkmime(RclConfig *cnf, string &out, Binc::MimePart& doc,
int depth)
{
if (depth > 5) {
LOGINFO(("walkmime: max depth exceeded\n"));
@@ -88,12 +209,12 @@ void walkmime(string &out, Binc::MimePart& doc, int fd, int depth)
LOGDEB(("walkmime: ismultipart %d subtype '%s'\n",
doc.isMultipart(), doc.getSubType().c_str()));
// We only handle alternative and mixed for now. For
// alternative, we look for a text/plain part, else html and process it
// For mixed, we process each part.
// alternative, we look for a text/plain part, else html and
// process it For mixed, we process each part.
std::vector<Binc::MimePart>::iterator it;
if (!stringicmp("mixed", doc.getSubType())) {
for (it = doc.members.begin(); it != doc.members.end();it++) {
walkmime(out, *it, fd, depth+1);
walkmime(cnf, out, *it, depth+1);
}
} else if (!stringicmp("alternative", doc.getSubType())) {
std::vector<Binc::MimePart>::iterator ittxt, ithtml;
@@ -103,7 +224,17 @@ void walkmime(string &out, Binc::MimePart& doc, int fd, int depth)
Binc::HeaderItem hi;
if (!doc.h.getFirstHeader("Content-Type", hi))
continue;
LOGDEB(("walkmime:content-type: %s\n", hi.getValue().c_str()));
MimeHeaderValue content_type;
parseMimeHeaderValue(hi.getValue(), content_type);
if (!stringlowercmp("text/plain", content_type.value))
ittxt = it;
else if (!stringlowercmp("text/html", content_type.value))
ithtml = it;
}
if (ittxt != doc.members.end()) {
walkmime(cnf, out, *ittxt, depth+1);
} else if (ithtml != doc.members.end()) {
walkmime(cnf, out, *ithtml, depth+1);
}
}
} else {
@@ -149,30 +280,36 @@ void walkmime(string &out, Binc::MimePart& doc, int fd, int depth)
LOGDEB(("walkmime: final: body start offset %d, length %d\n",
doc.getBodyStartOffset(), doc.getBodyLength()));
string body;
doc.getBody(fd, body, 0, doc.bodylength);
doc.getBody(body, 0, doc.bodylength);
// Decode content transfer encoding
if (stringlowercmp("quoted-printable", content_disposition.value)) {
if (!stringlowercmp("quoted-printable", cte)) {
string decoded;
qp_decode(body, decoded);
body = decoded;
} else if (stringlowercmp("base64", content_disposition.value)) {
} else if (!stringlowercmp("base64", cte)) {
string decoded;
base64_decode(body, decoded);
body = decoded;
}
// Transcode to utf-8
string transcoded;
if (!transcode(body, transcoded, charset, "UTF-8")) {
LOGERR(("walkmime: transcode failed from cs '%s' to UTF-8\n",
charset.c_str()));
transcoded = body;
if (!stringlowercmp("text/html", content_type.value)) {
MimeHandlerHtml mh;
Rcl::Doc hdoc;
mh.charsethint = charset;
mh.worker1(cnf, "", body, content_type.value, hdoc);
transcoded = hdoc.text;
} else {
// Transcode to utf-8
if (!transcode(body, transcoded, charset, "UTF-8")) {
LOGERR(("walkmime: transcode failed from cs '%s' to UTF-8\n",
charset.c_str()));
transcoded = body;
}
}
out += string("\r\n") + transcoded;
}
}
src/internfile/mh_mail.h
+12 -3
View File
@@ -1,14 +1,23 @@
#ifndef _MAIL_H_INCLUDED_
#define _MAIL_H_INCLUDED_
/* @(#$Id: mh_mail.h,v 1.1 2005-03-25 09:40:27 dockes Exp $ (C) 2004 J.F.Dockes */
/* @(#$Id: mh_mail.h,v 1.2 2005-03-31 10:04:07 dockes Exp $ (C) 2004 J.F.Dockes */
#include "mimehandler.h"
namespace Binc {
class MimeDocument;
}
// Code to turn a mail folder file into internal documents
class MimeHandlerMail : public MimeHandler {
void *vfp;
int msgnum;
RclConfig *conf;
MimeHandler::Status processone(const string &fn, Rcl::Doc &docout);
MimeHandler::Status processone(const string &fn, Binc::MimeDocument& doc,
Rcl::Doc &docout);
MimeHandler::Status processmbox(const string &fn, Rcl::Doc &docout,
string &ipath);
public:
MimeHandlerMail() : conf(0) {}
MimeHandlerMail() : vfp(0), msgnum(0), conf(0) {}
virtual ~MimeHandlerMail();
virtual MimeHandler::Status
worker(RclConfig *conf, const string &fn,
const string &mtype, Rcl::Doc &docout, string& ipath);