compute md5 checksums for all docs and optionally collapse duplicates in results

src/internfile/mh_exec.cpp

@@ -25,6 +25,7 @@ static char rcsid[] = "@(#$Id: mh_exec.cpp,v 1.14 2008-10-09 09:19:37 dockes Exp
 #include "cancelcheck.h"
 #include "smallut.h"
 #include "transcode.h"
+#include "md5.h"
 
 #include <sys/types.h>
 #include <sys/wait.h>
@@ -130,5 +131,14 @@ bool MimeHandlerExec::next_document()
     // could still be overridden by the content-type meta tag.
     m_metaData["charset"] = charset;
     m_metaData["mimetype"] = mt;
+
+    string md5, xmd5, reason;
+    if (MD5File(m_fn, md5, &reason)) {
+	m_metaData["md5"] = MD5HexPrint(md5, xmd5);
+    } else {
+	LOGERR(("MimeHandlerExec: cant compute md5 for [%s]: %s\n", 
+		m_fn.c_str(), reason.c_str()));
+    }
+
     return true;
 }

src/internfile/mh_html.cpp

@@ -29,6 +29,7 @@ static char rcsid[] = "@(#$Id: mh_html.cpp,v 1.26 2008-10-03 06:17:46 dockes Exp
 #include "indextext.h"
 #include "mh_html.h"
 #include "smallut.h"
+#include "md5.h"
 
 #include <iostream>
 
@@ -53,6 +54,12 @@ bool MimeHandlerHtml::set_document_string(const string& htext)
 {
     m_html = htext;
     m_havedoc = true;
+
+    // We want to compute the md5 now because we may modify m_html later
+    string md5, xmd5;
+    MD5String(htext, md5);
+    m_metaData["md5"] = MD5HexPrint(md5, xmd5);
+
     return true;
 }
 

src/internfile/mh_mail.cpp

@@ -38,6 +38,7 @@ static char rcsid[] = "@(#$Id: mh_mail.cpp,v 1.35 2008-10-04 14:26:59 dockes Exp
 #include "mh_html.h"
 #include "rclconfig.h"
 #include "mimetype.h"
+#include "md5.h"
 
 // binc imap mime definitions
 #include "mime.h"
@@ -81,6 +82,18 @@ bool MimeHandlerMail::set_document_file(const string &fn)
 	close(m_fd);
 	m_fd = -1;
     }
+
+    // Yes, we read the file twice. It would be possible in theory to add
+    // the md5 computation to the mime analysis, but ...
+    string md5, xmd5, reason;
+    if (MD5File(fn, md5, &reason)) {
+	m_metaData["md5"] = MD5HexPrint(md5, xmd5);
+    } else {
+	LOGERR(("MimeHandlerMail: cant compute md5 for [%s]: %s\n", fn.c_str(),
+		reason.c_str()));
+    }
+
+
     m_fd = open(fn.c_str(), 0);
     if (m_fd < 0) {
 	LOGERR(("MimeHandlerMail::set_document_file: open(%s) errno %d\n",
@@ -104,6 +117,11 @@ bool MimeHandlerMail::set_document_string(const string &msgtxt)
     LOGDEB1(("MimeHandlerMail::set_document_string\n"));
     LOGDEB2(("Message text: [%s]\n", msgtxt.c_str()));
     delete m_stream;
+
+    string md5, xmd5;
+    MD5String(msgtxt, md5);
+    m_metaData["md5"] = MD5HexPrint(md5, xmd5);
+
     m_stream = new stringstream(msgtxt);
     delete m_bincdoc;
     m_bincdoc = new Binc::MimeDocument;

src/internfile/mh_text.cpp

@@ -29,19 +29,26 @@ using namespace std;
 #include "debuglog.h"
 #include "readfile.h"
 #include "transcode.h"
+#include "md5.h"
 
 // Process a plain text file
 bool MimeHandlerText::set_document_file(const string &fn)
 {
     string otext;
-    if (!file_to_string(fn, otext))
+    string reason;
+    if (!file_to_string(fn, otext, &reason)) {
+	LOGERR(("MimeHandlerText: can't read file: %s\n", reason.c_str()));
 	return false;
+    }
     return set_document_string(otext);
 }
     
 bool MimeHandlerText::set_document_string(const string& otext)
 {
     m_text = otext;
+    string md5, xmd5;
+    MD5String(m_text, md5);
+    m_metaData["md5"] = MD5HexPrint(md5, xmd5);
     m_havedoc = true;
     return true;
 }