mail ckpt

src/internfile/internfile.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: internfile.cpp,v 1.3 2005-03-17 15:35:49 dockes Exp $ (C) 2004 J.F.Dockes";
+static char rcsid[] = "@(#$Id: internfile.cpp,v 1.4 2005-03-25 09:40:27 dockes Exp $ (C) 2004 J.F.Dockes";
 #endif
 #include <unistd.h>
 #include <sys/types.h>
@@ -70,30 +70,27 @@ static bool uncompressfile(RclConfig *conf, const string& ifn,
     return true;
 }
 
-static void tmpcleanup(const string& tdir, const string& tfile)
+void FileInterner::tmpcleanup()
 {
     if (tdir.empty() || tfile.empty())
 	return;
     if (unlink(tfile.c_str()) < 0) {
-	LOGERR(("tmpcleanup: unlink(%s) errno %d\n", tfile.c_str(), 
-		errno));
+	LOGERR(("FileInterner::tmpcleanup: unlink(%s) errno %d\n", 
+		tfile.c_str(), errno));
 	return;
     }
 }
 
-bool internfile(const std::string &ifn, RclConfig *config, Rcl::Doc& doc,
-		const string& tdir)
+// Handler==0 on return says we're in error
+FileInterner::FileInterner(const std::string &f, RclConfig *cnf, 
+			   const string& td)
+    : fn(f), config(cnf), tdir(td), handler(0) 
 {
-    string fn = ifn;
-    string tfile;
-    MimeHandler *handler = 0;
-    bool ret = false;
-
-    string mime = mimetype(fn, config->getMimeMap());
+    mime = mimetype(fn, config->getMimeMap());
     if (mime.empty()) {
 	// No mime type: not listed in our map.
-	LOGDEB(("internfile: (no mime) [%s]\n", fn.c_str()));
-	return false;
+	LOGDEB(("FileInterner::FileInterner: (no mime) [%s]\n", fn.c_str()));
+	return;
     }
 
     // First check for a compressed file. If so, create a temporary
@@ -101,8 +98,9 @@ bool internfile(const std::string &ifn, RclConfig *config, Rcl::Doc& doc,
     // rest with the temp file.
     list<string>ucmd;
     if (getUncompressor(mime, config->getMimeConf(), ucmd)) {
-	if (!uncompressfile(config, fn, ucmd, tdir, tfile)) 
-	    return false;
+	if (!uncompressfile(config, fn, ucmd, tdir, tfile)) {
+	    return;
+	}
 	LOGDEB(("internfile: after ucomp: tdir %s, tfile %s\n", 
 		tdir.c_str(), tfile.c_str()));
 	fn = tfile;
@@ -110,33 +108,43 @@ bool internfile(const std::string &ifn, RclConfig *config, Rcl::Doc& doc,
 	if (mime.empty()) {
 	    // No mime type ?? pass on.
 	    LOGDEB(("internfile: (no mime) [%s]\n", fn.c_str()));
-	    goto out;
+	    return;
 	}
-
     }
-    
 
     // Look for appropriate handler
     handler = getMimeHandler(mime, config->getMimeConf());
     if (!handler) {
 	// No handler for this type, for now :(
-	LOGDEB(("internfile: %s : no handler\n", mime.c_str()));
-	goto out;
+	LOGDEB(("FileInterner::FileInterner: %s: no handler\n", mime.c_str()));
+	return;
     }
 
-    LOGDEB(("internfile: %s [%s]\n", mime.c_str(), fn.c_str()));
+    LOGDEB(("FileInterner::FileInterner: %s [%s]\n",mime.c_str(), fn.c_str()));
+}
+
+FileInterner::Status FileInterner::internfile(Rcl::Doc& doc, string& ipath)
+{
+    if (!handler)
+	return FIError;
 
     // Turn file into a document. The document has fields for title, body 
     // etc.,  all text converted to utf8
-    if (!handler->worker(config, fn,  mime, doc)) {
-	goto out;
+    MimeHandler::Status mhs = handler->worker(config, fn,  mime, doc, ipath);
+    FileInterner::Status ret = FIError;
+    switch (mhs) {
+    case MimeHandler::MHError: break;
+    case MimeHandler::MHDone: ret = FIDone;break;
+    case MimeHandler::MHAgain: ret = FIAgain;break;
     }
-    doc.mimetype = mime;
 
-    // Clean up. We delete the temp file and its father directory
-    ret = true;
- out:
-    delete handler;
-    tmpcleanup(tdir, tfile);
+    doc.mimetype = mime;
     return ret;
 }
+
+FileInterner::~FileInterner()
+{
+    delete handler; 
+    handler = 0;
+    tmpcleanup();
+}

src/internfile/internfile.h

@@ -1,14 +1,31 @@
 #ifndef _INTERNFILE_H_INCLUDED_
 #define _INTERNFILE_H_INCLUDED_
-/* @(#$Id: internfile.h,v 1.2 2005-02-09 12:07:29 dockes Exp $  (C) 2004 J.F.Dockes */
+/* @(#$Id: internfile.h,v 1.3 2005-03-25 09:40:27 dockes Exp $  (C) 2004 J.F.Dockes */
 
 #include <string>
 
 #include "rclconfig.h"
 #include "rcldb.h"
 
+class MimeHandler;
+
 /// Turn external file into internal representation, according to mime type etc
-extern bool internfile(const std::string &fn, RclConfig *config, 
-		       Rcl::Doc& doc, const string& tdir);
+class FileInterner {
+    string fn;
+    RclConfig *config;
+    const string &tdir;
+    MimeHandler *handler;
+    string tfile;
+    string mime;
+
+    void tmpcleanup();
+
+ public:
+    FileInterner(const std::string &f, RclConfig *cnf, const string& td);
+    ~FileInterner();
+
+    enum Status {FIError, FIDone, FIAgain};
+    Status internfile(Rcl::Doc& doc, string &ipath);
+};
 
 #endif /* _INTERNFILE_H_INCLUDED_ */

src/internfile/mh_html.cpp

@@ -38,21 +38,23 @@
 using namespace std;
 
 
-bool MimeHandlerHtml::worker(RclConfig *conf, const string &fn, 
-			     const string &mtype, Rcl::Doc &docout)
+MimeHandler::Status 
+MimeHandlerHtml::worker(RclConfig *conf, const string &fn, 
+			const string &mtype, Rcl::Doc &docout, string&)
 {
     LOGDEB(("textHtmlToDoc: %s\n", fn.c_str()));
     string otext;
     if (!file_to_string(fn, otext)) {
 	LOGINFO(("textHtmlToDoc: cant read: %s\n", fn.c_str()));
-	return false;
+	return MimeHandler::MHError;
     }
     return worker1(conf, fn, otext, mtype, docout);
 }
 
-bool MimeHandlerHtml::worker1(RclConfig *conf, const string &, 
-			     const string& htext,
-			     const string &mtype, Rcl::Doc &docout)
+MimeHandler::Status 
+MimeHandlerHtml::worker1(RclConfig *conf, const string &, 
+			 const string& htext,
+			 const string &mtype, Rcl::Doc &docout)
 {
     // Character set handling:
 
@@ -111,5 +113,5 @@ bool MimeHandlerHtml::worker1(RclConfig *conf, const string &,
     out.keywords = pres.keywords;
     out.abstract = pres.sample;
     docout = out;
-    return true;
+    return MimeHandler::MHDone;
 }

src/internfile/mh_html.h

@@ -1,6 +1,6 @@
 #ifndef _HTML_H_INCLUDED_
 #define _HTML_H_INCLUDED_
-/* @(#$Id: mh_html.h,v 1.2 2005-03-17 15:35:49 dockes Exp $  (C) 2004 J.F.Dockes */
+/* @(#$Id: mh_html.h,v 1.3 2005-03-25 09:40:27 dockes Exp $  (C) 2004 J.F.Dockes */
 #include "mimehandler.h"
 
 // Code to turn an html document into an internal one. There are 2
@@ -11,9 +11,9 @@
 // carry titles, abstracts, whatever)
 class MimeHandlerHtml : public MimeHandler {
  public:
-    virtual bool worker(RclConfig *conf, const string &fn, 
-			const string &mtype, Rcl::Doc &docout);
-    virtual bool worker1(RclConfig *conf, const string &fn, 
+    virtual MimeHandler::Status worker(RclConfig *conf, const string &fn, 
+			const string &mtype, Rcl::Doc &docout, string&);
+    virtual MimeHandler::Status worker1(RclConfig *conf, const string &fn, 
 			 const string& htext,
 			 const string &mtype, Rcl::Doc &docout);
 };

src/internfile/mh_mail.cpp

@@ -0,0 +1,178 @@
+#ifndef lint
+static char rcsid[] = "@(#$Id: mh_mail.cpp,v 1.1 2005-03-25 09:40:27 dockes Exp $ (C) 2005 J.F.Dockes";
+#endif
+
+#include <fcntl.h>
+#include <errno.h>
+
+#include <map>
+using std::map;
+
+#include "mimehandler.h"
+#include "debuglog.h"
+#include "csguess.h"
+#include "readfile.h"
+#include "transcode.h"
+#include "mimeparse.h"
+#include "indextext.h"
+#include "mail.h"
+#include "debuglog.h"
+#include "smallut.h"
+#include "mimeparse.h"
+
+using namespace std;
+
+// We are called for two different file types: mbox-type folders
+// holding multiple messages, and maildir-type files with one rfc822
+// message
+MimeHandler::Status 
+MimeHandlerMail::worker(RclConfig *cnf, const string &fn, 
+			const string &mtype, Rcl::Doc &docout, string&)
+{
+    LOGDEB(("MimeHandlerMail::worker: %s [%s]\n", mtype.c_str(), fn.c_str()));
+    conf = cnf;
+
+    if (!stringlowercmp("message/rfc822", mtype)) {
+	return processone(fn, docout);
+    } else  if (!stringlowercmp("text/x-mail", mtype)) {
+	return MimeHandler::MHError;
+    } else
+	return MimeHandler::MHError;
+}
+
+
+#include "mime.h"
+
+const char *hnames[] = {"Subject", "Content-type"};
+int nh = sizeof(hnames) / sizeof(char *);
+
+void walkmime(string &out, Binc::MimePart& doc, int fd, int depth);
+
+// Transform a single message into a document. The subject becomes the
+// title, and any simple body part with a content-type of text or html
+// and content-disposition inline gets concatenated as text.
+MimeHandler::Status 
+MimeHandlerMail::processone(const string &fn, Rcl::Doc &docout)
+{
+    int fd;
+    if ((fd = open(fn.c_str(), 0)) < 0) {
+	LOGERR(("MimeHandlerMail::processone: open(%s) errno %d\n",
+		fn.c_str(), errno));
+	return MimeHandler::MHError;
+    }
+    Binc::MimeDocument doc;
+    doc.parseFull(fd);
+
+    if (!doc.isHeaderParsed() && !doc.isAllParsed()) {
+	LOGERR(("MimeHandlerMail::processone: parse error for %s\n", 
+		fn.c_str()));
+	close(fd);
+	return MimeHandler::MHError;
+    }
+    LOGDEB(("MimeHandlerMail::processone: ismultipart %d mime subtype '%s'\n", 
+	    doc.isMultipart(), doc.getSubType().c_str()));
+    walkmime(docout.text, doc, fd, 0);
+    close(fd);
+    LOGDEB(("MimeHandlerMail::processone: text: '%s'\n",  docout.text.c_str()));
+    return MimeHandler::MHError;
+}
+
+void walkmime(string &out, Binc::MimePart& doc, int fd, int depth)
+{
+    if (depth > 5) {
+	LOGINFO(("walkmime: max depth exceeded\n"));
+	return;
+    }
+
+    if (doc.isMultipart()) {
+	LOGDEB(("walkmime: ismultipart %d subtype '%s'\n", 
+		doc.isMultipart(), doc.getSubType().c_str()));
+	// We only handle alternative and mixed for now. For
+	// alternative, we look for a text/plain part, else html and process it
+	// For mixed, we process each part.
+	std::vector<Binc::MimePart>::iterator it;
+	if (!stringicmp("mixed", doc.getSubType())) {
+	    for (it = doc.members.begin(); it != doc.members.end();it++) {
+		walkmime(out, *it, fd, depth+1);
+	    }
+	} else if (!stringicmp("alternative", doc.getSubType())) {
+	    std::vector<Binc::MimePart>::iterator ittxt, ithtml;
+	    ittxt = ithtml = doc.members.end();
+	    for (it = doc.members.begin(); it != doc.members.end();it++) {
+		// Get and parse content-type header
+		Binc::HeaderItem hi;
+		if (!doc.h.getFirstHeader("Content-Type", hi)) 
+		    continue;
+		LOGDEB(("walkmime:content-type: %s\n", hi.getValue().c_str()));
+	    }
+	}
+    } else {
+	// If content-type is text or html and content-disposition is inline, 
+	// decode and add to text.
+
+	// Get and parse content-type header.
+	Binc::HeaderItem hi;
+	string ctt = "text/plain";
+	if (doc.h.getFirstHeader("Content-Type", hi)) {
+	    ctt = hi.getValue();
+	}
+	LOGDEB(("walkmime:content-type: %s\n", ctt.c_str()));
+	MimeHeaderValue content_type;
+	parseMimeHeaderValue(ctt, content_type);
+	if (stringlowercmp("text/plain", content_type.value) && 
+	    stringlowercmp("text/html", content_type.value)) {
+	    return;
+	}
+	string charset = "us-ascii";
+	map<string,string>::const_iterator it;
+	it = content_type.params.find(string("charset"));
+	if (it != content_type.params.end())
+	    charset = it->second;
+
+	// Content disposition
+	string ctd = "inline";
+	if (doc.h.getFirstHeader("Content-Disposition", hi)) {
+	    ctd = hi.getValue();
+	}
+	MimeHeaderValue content_disposition;
+	parseMimeHeaderValue(ctd, content_disposition);
+	if (stringlowercmp("inline", content_disposition.value)) {
+	    return;
+	}
+
+	// Content transfer encoding
+	string cte = "7bit";
+	if (doc.h.getFirstHeader("Content-Transfer-Encoding", hi)) {
+	    cte = hi.getValue();
+	} 
+
+	LOGDEB(("walkmime: final: body start offset %d, length %d\n", 
+		doc.getBodyStartOffset(), doc.getBodyLength()));
+	string body;
+	doc.getBody(fd, body, 0, doc.bodylength);
+
+	// Decode content transfer encoding
+	if (stringlowercmp("quoted-printable", content_disposition.value)) {
+	    string decoded;
+	    qp_decode(body, decoded);
+	    body = decoded;
+	} else if (stringlowercmp("base64", content_disposition.value)) {
+	    string decoded;
+	    base64_decode(body, decoded);
+	    body = decoded;
+	}
+
+
+        // Transcode to utf-8 
+	string transcoded;
+	if (!transcode(body, transcoded, charset, "UTF-8")) {
+	    LOGERR(("walkmime: transcode failed from cs '%s' to UTF-8\n",
+		    charset.c_str()));
+	    transcoded = body;
+	}
+
+	out += string("\r\n") + transcoded;
+    }
+}
+
+

src/internfile/mh_mail.h

@@ -0,0 +1,16 @@
+#ifndef _MAIL_H_INCLUDED_
+#define _MAIL_H_INCLUDED_
+/* @(#$Id: mh_mail.h,v 1.1 2005-03-25 09:40:27 dockes Exp $  (C) 2004 J.F.Dockes */
+#include "mimehandler.h"
+
+// Code to turn a mail folder file into internal documents
+class MimeHandlerMail : public MimeHandler {
+    RclConfig *conf;
+    MimeHandler::Status processone(const string &fn, Rcl::Doc &docout);
+ public:
+    MimeHandlerMail() : conf(0) {}
+    virtual MimeHandler::Status 
+	worker(RclConfig *conf, const string &fn, 
+	       const string &mtype, Rcl::Doc &docout, string& ipath);
+};
+#endif /* _MAIL_H_INCLUDED_ */

src/internfile/mimehandler.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: mimehandler.cpp,v 1.8 2005-02-04 14:21:17 dockes Exp $ (C) 2004 J.F.Dockes";
+static char rcsid[] = "@(#$Id: mimehandler.cpp,v 1.9 2005-03-25 09:40:27 dockes Exp $ (C) 2004 J.F.Dockes";
 #endif
 
 #include <iostream>
@@ -13,23 +13,24 @@ using namespace std;
 #include "debuglog.h"
 #include "smallut.h"
 #include "html.h"
+#include "mail.h"
 #include "execmd.h"
 #include "pathut.h"
 
 class MimeHandlerText : public MimeHandler {
  public:
-    bool worker(RclConfig *conf, const string &fn, 
-		const string &mtype, Rcl::Doc &docout);
+    MimeHandler::Status worker(RclConfig *conf, const string &fn, 
+		const string &mtype, Rcl::Doc &docout, string&);
     
 };
 
 // Process a plain text file
-bool MimeHandlerText::worker(RclConfig *conf, const string &fn, 
-			     const string &mtype, Rcl::Doc &docout)
+MimeHandler::Status MimeHandlerText::worker(RclConfig *conf, const string &fn, 
+			     const string &mtype, Rcl::Doc &docout, string&)
 {
     string otext;
     if (!file_to_string(fn, otext))
-	return false;
+	return MimeHandler::MHError;
 	
     // Try to guess charset, then convert to utf-8, and fill document
     // fields The charset guesser really doesnt work well in general
@@ -46,36 +47,38 @@ bool MimeHandlerText::worker(RclConfig *conf, const string &fn,
 	cerr << "textPlainToDoc: transcode failed: charset '" << charset
 	     << "' to UTF-8: "<< utf8 << endl;
 	otext.erase();
-	return 0;
+	return MimeHandler::MHError;
     }
 
     Rcl::Doc out;
     out.origcharset = charset;
     out.text = utf8;
     docout = out;
-    return true;
+    return MimeHandler::MHDone;
 }
 
 class MimeHandlerExec : public MimeHandler {
  public:
     list<string> params;
     virtual ~MimeHandlerExec() {}
-    virtual bool worker(RclConfig *conf, const string &fn, 
-			const string &mtype, Rcl::Doc &docout);
+    virtual MimeHandler::Status worker(RclConfig *conf, const string &fn, 
+				       const string &mtype, Rcl::Doc &docout, 
+				       string&);
 
 };
 
     
 // Execute an external program to translate a file from its native format
 // to html. Then call the html parser to do the actual indexing
-bool MimeHandlerExec::worker(RclConfig *conf, const string &fn, 
-			     const string &mtype, Rcl::Doc &docout)
+MimeHandler::Status 
+MimeHandlerExec::worker(RclConfig *conf, const string &fn, 
+			const string &mtype, Rcl::Doc &docout, string&)
 {
     if (params.empty()) {
 	// Hu ho
 	LOGERR(("MimeHandlerExec::worker: empty params for mime %s\n",
 		mtype.c_str()));
-	return false;
+	return MimeHandler::MHError;
     }
     // Command name
     string cmd = find_filter(conf, params.front());
@@ -92,7 +95,7 @@ bool MimeHandlerExec::worker(RclConfig *conf, const string &fn,
     if (status) {
 	LOGERR(("MimeHandlerExec: command status 0x%x: %s\n", 
 		status, cmd.c_str()));
-	return false;
+	return MimeHandler::MHError;
     }
 
     // Process/index  the html
@@ -106,6 +109,10 @@ static MimeHandler *mhfact(const string &mime)
 	return new MimeHandlerText;
     else if (!stringlowercmp("text/html", mime))
 	return new MimeHandlerHtml;
+    else if (!stringlowercmp("text/x-mail", mime))
+	return new MimeHandlerMail;
+    else if (!stringlowercmp("message/rfc822", mime))
+	return new MimeHandlerMail;
     return 0;
 }
 
@@ -117,7 +124,7 @@ MimeHandler *getMimeHandler(const std::string &mtype, ConfTree *mhandlers)
     // Return handler definition for mime type
     string hs;
     if (!mhandlers->get(mtype, hs, "index")) {
-	LOGDEB(("getMimeHandler: no handler for %s\n", mtype.c_str()));
+	LOGDEB(("getMimeHandler: no handler for '%s'\n", mtype.c_str()));
 	return 0;
     }
 

src/internfile/mimehandler.h

@@ -1,6 +1,6 @@
 #ifndef _MIMEHANDLER_H_INCLUDED_
 #define _MIMEHANDLER_H_INCLUDED_
-/* @(#$Id: mimehandler.h,v 1.5 2005-02-04 09:39:44 dockes Exp $  (C) 2004 J.F.Dockes */
+/* @(#$Id: mimehandler.h,v 1.6 2005-03-25 09:40:27 dockes Exp $  (C) 2004 J.F.Dockes */
 
 #include <string>
 #include <list>
@@ -10,13 +10,34 @@
 
 
 /**
- * Document interner class. We sometimes have data to pass to an interner
+ * Document interner class. 
  */
 class MimeHandler {
  public:
     virtual ~MimeHandler() {}
-    virtual bool worker(RclConfig *, const std::string &filename, 
-			const std::string &mimetype, Rcl::Doc& outdoc) = 0;
+
+    /**
+     * Transform external data into internal utf8 document
+     *
+     * @param conf the global configuration
+     * @param filename File from which the data comes from
+     * @param mimetype its mime type (from the mimemap configuration file)
+     * @param outdoc   The output document
+     * @param ipath the access path for the document inside the file. 
+     *              For mono-document file types, this will always be empty. 
+     *              It is used, for example for mbox files which may contain
+     *              multiple emails. If this is not empty in input, then the
+     *              caller is requesting a single document (ie: for display).
+     *              If this is empty (during indexation), it will be filled-up
+     *              by the function, and all the file's documents will be 
+     *              returned by successive calls.
+     * @return the return value indicates if there are more documents to be 
+     *         fetched from the same file.
+     */
+    enum Status {MHError, MHDone, MHAgain};
+    virtual Status worker(RclConfig * conf, const std::string &filename, 
+			  const std::string &mimetype, Rcl::Doc& outdoc,
+			  string& ipath) = 0;
 };
 
 /**
@@ -30,6 +51,11 @@ extern MimeHandler *getMimeHandler(const std::string &mtyp, ConfTree *mhdlers);
  */
 extern std::string getMimeViewer(const std::string &mtyp, ConfTree *mhandlers);
 
+/** 
+ * Return command to uncompress the given type. The returned command has
+ * substitutable places for input file name and temp dir name, and will
+ * return output name
+ */
 bool getUncompressor(const std::string &mtype, ConfTree *mhandlers,
 		     std::list<std::string>& cmd);
 

src/internfile/myhtmlparse.cpp

@@ -150,7 +150,8 @@ MyHtmlParser::opening_tag(const string &tag, const map<string,string> &p)
 			lowercase_term(hequiv);
 			if (hequiv == "content-type") {
 			    string value = i->second;
-			    MimeHeaderValue p = parseMimeHeaderValue(value);
+			    MimeHeaderValue p;
+			    parseMimeHeaderValue(value, p);
 			    map<string, string>::const_iterator k;
 			    if ((k = p.params.find("charset")) != 
 				p.params.end()) {