compute md5 checksums for all docs and optionally collapse duplicates in results

src/rcldb/rcldb.cpp

@@ -48,7 +48,7 @@ using namespace std;
 #include "searchdata.h"
 #include "rclquery.h"
 #include "rclquery_p.h"
-
+#include "md5.h"
 
 #ifndef MAX
 #define MAX(A,B) (A>B?A:B)
@@ -57,15 +57,6 @@ using namespace std;
 #define MIN(A,B) (A<B?A:B)
 #endif
 
-// Omega compatible values. We leave a hole for future omega values. Not sure 
-// it makes any sense to keep any level of omega compat given that the index
-// is incompatible anyway.
-enum value_slot {
-    VALUE_LASTMOD = 0,	// 4 byte big endian value - seconds since 1970.
-    VALUE_MD5 = 1,	// 16 byte MD5 checksum of original document.
-    VALUE_SIG = 10      // Doc sig as chosen by app (ex: mtime+size
-};
-
 // Recoll index format version is stored in user metadata. When this change,
 // we can't open the db and will have to reindex.
 static const string RCL_IDX_VERSION_KEY("RCL_IDX_VERSION_KEY");
@@ -1078,6 +1069,15 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi,
 	}
     }
 
+    // If the file's md5 was computed, add value. This is optionally
+    // used for query result duplicate elimination.
+    string& md5 = doc.meta[Doc::keymd5];
+    if (!md5.empty()) {
+	string digest;
+	MD5HexScan(md5, digest);
+	newdocument.add_value(VALUE_MD5, digest);
+    }
+
     LOGDEB0(("Rcl::Db::add: new doc record:\n%s\n", record.c_str()));
     newdocument.set_data(record);
 

src/rcldb/rcldb.h

@@ -56,6 +56,15 @@ class RclConfig;
 namespace Rcl {
 #endif
 
+// Omega compatible values. We leave a hole for future omega values. Not sure 
+// it makes any sense to keep any level of omega compat given that the index
+// is incompatible anyway.
+enum value_slot {
+    VALUE_LASTMOD = 0,	// 4 byte big endian value - seconds since 1970.
+    VALUE_MD5 = 1,	// 16 byte MD5 checksum of original document.
+    VALUE_SIG = 10      // Doc sig as chosen by app (ex: mtime+size
+};
+
 class SearchData;
 class TermIter;
 class Query;

src/rcldb/rcldoc.cpp

@@ -22,4 +22,5 @@ namespace Rcl {
     const string Doc::keyau("author");
     const string Doc::keytt("title");
     const string Doc::keykw("keywords");
+    const string Doc::keymd5("md5");
 }

src/rcldb/rcldoc.h

@@ -157,6 +157,7 @@ class Doc {
     static const string keyau;  // author
     static const string keytt;  // title
     static const string keykw;  // keywords
+    static const string keymd5; // file md5 checksum
 };
 
 

src/rcldb/rclquery.cpp

@@ -89,7 +89,8 @@ private:
 };
 
 Query::Query(Db *db)
-    : m_nq(new Native(this)), m_db(db), m_sorter(0), m_sortAscending(true)
+    : m_nq(new Native(this)), m_db(db), m_sorter(0), m_sortAscending(true),
+      m_collapseDuplicates(false)
 {
 }
 
@@ -155,7 +156,11 @@ bool Query::setQuery(RefCntr<SearchData> sdata)
     string d;
     try {
 	m_nq->enquire = new Xapian::Enquire(m_db->m_ndb->db);
-	m_nq->enquire->set_query(m_nq->query);
+	if (m_collapseDuplicates) {
+	    m_nq->enquire->set_collapse_key(Rcl::VALUE_MD5);
+	} else {
+	    m_nq->enquire->set_collapse_key(Xapian::BAD_VALUENO);
+	}
 	if (!m_sortField.empty()) {
 	    if (m_sorter) {
 		delete (QSorter*)m_sorter;
@@ -167,6 +172,7 @@ bool Query::setQuery(RefCntr<SearchData> sdata)
 	    m_nq->enquire->set_sort_by_key((QSorter*)m_sorter, 
 					   !m_sortAscending);
 	}
+	m_nq->enquire->set_query(m_nq->query);
 	m_nq->mset = Xapian::MSet();
 	// Get the query description and trim the "Xapian::Query"
 	d = m_nq->query.get_description();

src/rcldb/rclquery.h

@@ -54,6 +54,7 @@ class Query {
 
     /** Choose sort order. Must be called before setQuery */
     void setSortBy(const string& fld, bool ascending = true);
+    void setCollapseDuplicates(bool on) {m_collapseDuplicates = on;}
     const string& getSortBy() const {return m_sortField;}
     bool getSortAscending() const {return m_sortAscending;}
 
@@ -90,8 +91,9 @@ private:
     string m_reason; // Error explanation
     Db    *m_db;
     void  *m_sorter;
-    string       m_sortField;
-    bool         m_sortAscending;
+    string m_sortField;
+    bool   m_sortAscending;
+    bool   m_collapseDuplicates;     
     /* Copyconst and assignement private and forbidden */
     Query(const Query &) {}
     Query & operator=(const Query &) {return *this;};