general field name handling cleanup + sort facility in rclquery

src/rcldb/rcldb.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.144 2008-09-09 12:58:23 dockes Exp $ (C) 2004 J.F.Dockes";
+static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.145 2008-09-16 08:18:30 dockes Exp $ (C) 2004 J.F.Dockes";
 #endif
 /*
  *   This program is free software; you can redistribute it and/or modify
@@ -129,12 +129,10 @@ bool Db::Native::subDocs(const string &udi, vector<Xapian::docid>& docids)
     return false;
 }
 
+// Only ONE field name inside the index data record differs from the
+// Rcl::Doc ones: caption<->title, for a remnant of compatibility with
+// omega
 static const string keycap("caption");
-static const string keymtp("mtype");
-static const string keyfmt("fmtime");
-static const string keydmt("dmtime");
-static const string keyoc("origcharset");
-static const string keyurl("url");
 
 // Turn data record from db into document fields
 bool Db::Native::dbDataToRclDoc(Xapian::docid docid, std::string &data, 
@@ -144,11 +142,11 @@ bool Db::Native::dbDataToRclDoc(Xapian::docid docid, std::string &data,
     ConfSimple parms(&data);
     if (!parms.ok())
 	return false;
-    parms.get(keyurl, doc.url);
-    parms.get(keymtp, doc.mimetype);
-    parms.get(keyfmt, doc.fmtime);
-    parms.get(keydmt, doc.dmtime);
-    parms.get(keyoc, doc.origcharset);
+    parms.get(Doc::keyurl, doc.url);
+    parms.get(Doc::keytp, doc.mimetype);
+    parms.get(Doc::keyfmt, doc.fmtime);
+    parms.get(Doc::keydmt, doc.dmtime);
+    parms.get(Doc::keyoc, doc.origcharset);
     parms.get(keycap, doc.meta[Doc::keytt]);
     parms.get(Doc::keykw, doc.meta[Doc::keykw]);
     parms.get(Doc::keyabs, doc.meta[Doc::keyabs]);
@@ -162,10 +160,10 @@ bool Db::Native::dbDataToRclDoc(Xapian::docid docid, std::string &data,
     char buf[20];
     sprintf(buf,"%.2f", float(percent) / 100.0);
     doc.meta[Doc::keyrr] = buf;
-    parms.get(string("ipath"), doc.ipath);
-    parms.get(string("fbytes"), doc.fbytes);
-    parms.get(string("dbytes"), doc.dbytes);
-    parms.get(string("sig"), doc.sig);
+    parms.get(Doc::keyipt, doc.ipath);
+    parms.get(Doc::keyfs, doc.fbytes);
+    parms.get(Doc::keyds, doc.dbytes);
+    parms.get(Doc::keysig, doc.sig);
     doc.xdocid = docid;
 
     // Other, not predefined meta fields:
@@ -691,24 +689,25 @@ bool Db::isopen()
 // indexed with no prefix (ie: abstract)
 bool Db::fieldToPrefix(const string& fldname, string &pfx)
 {
-    // This is the default table
+    // This is the default table. We prefer the data from rclconfig if 
+    // available
     static map<string, string> fldToPrefs;
     if (fldToPrefs.empty()) {
 	fldToPrefs[Doc::keyabs] = string();
 	fldToPrefs["ext"] = "XE";
-	fldToPrefs["filename"] = "XSFN";
+	fldToPrefs[Doc::keyfn] = "XSFN";
 
-	fldToPrefs["title"] = "S";
 	fldToPrefs[keycap] = "S";
+	fldToPrefs[Doc::keytt] = "S";
 	fldToPrefs["subject"] = "S";
 
 	fldToPrefs[Doc::keyau] = "A";
 	fldToPrefs["creator"] = "A";
 	fldToPrefs["from"] = "A";
 
+	fldToPrefs[Doc::keykw] = "K";
 	fldToPrefs["keyword"] = "K";
 	fldToPrefs["tag"] = "K";
-	fldToPrefs[Doc::keykw] = "K";
 	fldToPrefs["tags"] = "K";
     }
 
@@ -719,6 +718,7 @@ bool Db::fieldToPrefix(const string& fldname, string &pfx)
     if (config && config->getFieldPrefix(fld, pfx))
 	return true;
 
+    // No data in rclconfig? Check default values
     map<string, string>::const_iterator it = fldToPrefs.find(fld);
     if (it != fldToPrefs.end()) {
 	pfx = it->second;
@@ -816,9 +816,17 @@ void Db::setAbstractParams(int idxtrunc, int syntlen, int syntctxlen)
 	m_synthAbsWordCtxLen = syntctxlen;
 }
 
+static inline void leftzeropad(string& s, unsigned len)
+{
+    if (s.length() && s.length() < len)
+	s = s.insert(0, len-s.length(), '0');
+}
+
 static const int MB = 1024 * 1024;
 static const string nc("\n\r\x0c");
 
+#define RECORD_APPEND(R, NM, VAL) {R += NM + "=" + VAL + "\n";}
+
 // Add document in internal form to the database: index the terms in
 // the title abstract and body and add special terms for file name,
 // date, mime type ... , create the document data record (more
@@ -958,39 +966,43 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi,
     // reasonable lengths and suppress newlines (so that the data
     // record can keep a simple syntax)
 
-    string record = "url=" + doc.url;
-    record += "\nmtype=" + doc.mimetype;
-    record += "\nfmtime=" + doc.fmtime;
+    string record;
+    RECORD_APPEND(record, Doc::keyurl, doc.url);
+    RECORD_APPEND(record, Doc::keytp, doc.mimetype);
+    // We left-zero-pad the times so that they are lexico-sortable
+    leftzeropad(doc.fmtime, 11);
+    RECORD_APPEND(record, Doc::keyfmt, doc.fmtime);
     if (!doc.dmtime.empty()) {
-	record += "\ndmtime=" + doc.dmtime;
+	leftzeropad(doc.dmtime, 11);
+	RECORD_APPEND(record, Doc::keydmt, doc.dmtime);
     }
-    record += "\norigcharset=" + doc.origcharset;
+    RECORD_APPEND(record, Doc::keyoc, doc.origcharset);
 
     if (!doc.fbytes.empty())
-	record += string("\nfbytes=") + doc.fbytes;
+	RECORD_APPEND(record, Doc::keyfs, doc.fbytes);
     // Note that we add the signature both as a value and in the data record
     if (!doc.sig.empty())
-	record += string("\nsig=") + doc.sig;
+	RECORD_APPEND(record, Doc::keysig, doc.sig);
     newdocument.add_value(VALUE_SIG, doc.sig);
 
     char sizebuf[30]; 
     sprintf(sizebuf, "%u", (unsigned int)doc.text.length());
-    record += string("\ndbytes=") + sizebuf;
+    RECORD_APPEND(record, Doc::keyds, sizebuf);
 
     if (!doc.ipath.empty())
-	record += "\nipath=" + doc.ipath;
+	RECORD_APPEND(record, Doc::keyipt, doc.ipath);
 
     if (doc.meta[Doc::keytt].empty())
 	doc.meta[Doc::keytt] = doc.utf8fn;
     doc.meta[Doc::keytt] = 
 	neutchars(truncate_to_word(doc.meta[Doc::keytt], 150), nc);
     if (!doc.meta[Doc::keytt].empty())
-	record += "\n" + keycap + "=" + doc.meta[Doc::keytt];
+	RECORD_APPEND(record, keycap, doc.meta[Doc::keytt]);
 
     doc.meta[Doc::keykw] = 
 	neutchars(truncate_to_word(doc.meta[Doc::keykw], 300), nc);
     if (!doc.meta[Doc::keykw].empty())
-	record += "\n" + Doc::keykw + "=" + doc.meta[Doc::keykw];
+	RECORD_APPEND(record, Doc::keykw, doc.meta[Doc::keykw]);
 
     // If abstract is empty, we make up one with the beginning of the
     // document. This is then not indexed, but part of the doc data so
@@ -1010,22 +1022,23 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi,
 		      nc);
     }
     if (!doc.meta[Doc::keyabs].empty())
-	record += "\n" + Doc::keyabs + "=" + doc.meta[Doc::keyabs];
+	RECORD_APPEND(record, Doc::keyabs, doc.meta[Doc::keyabs]);
 
     RclConfig *config = RclConfig::getMainConfig();
     if (config) {
 	const set<string>& stored = config->getStoredFields();
 	for (set<string>::const_iterator it = stored.begin();
 	     it != stored.end(); it++) {
+	    string nm = stringtolower(config->fieldCanon(*it));
 	    if (!doc.meta[*it].empty()) {
 		string value = 
 		    neutchars(truncate_to_word(doc.meta[*it], 150), nc);
-		record += "\n" + *it + "=" + value;
+		RECORD_APPEND(record, nm, value);
 	    }
 	}
     }
-    record += "\n";
-    LOGDEB0(("Rcl::Db::add: new doc record:\n %s\n", record.c_str()));
+
+    LOGDEB0(("Rcl::Db::add: new doc record:\n%s\n", record.c_str()));
     newdocument.set_data(record);
 
     const char *fnc = udi.c_str();
@@ -1105,21 +1118,6 @@ bool Db::needUpdate(const string &udi, const string& sig)
 
 	    // Retrieve old file/doc signature from value
 	    string osig = doc.get_value(VALUE_SIG);
-#if 0
-	    // Get old  sig from data record
-	    string data = doc.get_data();
-	    string::size_type i1, i2;
-	    i1 = data.find("sig=");
-	    if (i1 == string::npos) 
-		return true;
-	    i1 += 4;
-	    if (i1 >= data.length())
-		return true;
-	    i2 = data.find_first_of("\n\r", i1);
-	    if (i2 == string::npos)
-		return true;
-	    string osig = data.substr(i1, i2-i1);
-#endif
 	    LOGDEB2(("Db::needUpdate: oldsig [%s] new [%s]\n",
 		     osig.c_str(), sig.c_str()));
 	    // Compare new/old sig
@@ -1287,14 +1285,12 @@ bool Db::purgeFile(const string &udi)
     return false;
 }
 
+// File name wild card expansion. This is a specialisation ot termMatch
 bool Db::filenameWildExp(const string& fnexp, list<string>& names)
 {
-    // File name search, with possible wildcards. 
-    // We expand wildcards by scanning the filename terms (prefixed 
-    // with XSFN) from the database. 
-    // We build an OR query with the expanded values if any.
     string pattern;
     dumb_string(fnexp, pattern);
+    names.clear();
 
     // If pattern is not quoted, and has no wildcards, we add * at
     // each end: match any substring
@@ -1303,33 +1299,14 @@ bool Db::filenameWildExp(const string& fnexp, list<string>& names)
     } else if (pattern.find_first_of("*?[") == string::npos) {
 	pattern = "*" + pattern + "*";
     } // else let it be
+    LOGDEB(("Rcl::Db::filenameWildExp: pattern: [%s]\n", pattern.c_str()));
 
-    LOGDEB((" pattern: [%s]\n", pattern.c_str()));
-
-    // Match pattern against all file names in the db
-    string ermsg;
-    try {
-	Xapian::TermIterator it = m_ndb->db.allterms_begin(); 
-	it.skip_to("XSFN");
-	for (;it != m_ndb->db.allterms_end(); it++) {
-	    if ((*it).find("XSFN") != 0)
-		break;
-	    string fn = (*it).substr(4);
-	    LOGDEB2(("Matching [%s] and [%s]\n", pattern.c_str(), fn.c_str()));
-	    if (fnmatch(pattern.c_str(), fn.c_str(), 0) != FNM_NOMATCH) {
-		names.push_back((*it).c_str());
-	    }
-	    // Limit the match count
-	    if (names.size() > 1000) {
-		LOGERR(("Db::filenameWildExp: too many matched file names\n"));
-		break;
-	    }
-	}
-    } XCATCHERROR(ermsg);
-    if (!ermsg.empty()) {
-	LOGERR(("filenameWildExp: xapian error: %s\n", ermsg.c_str()));
+    list<TermMatchEntry> entries;
+    if (!termMatch(ET_WILD, string(), pattern, entries, 1000, Doc::keyfn))
 	return false;
-    }
+    for (list<TermMatchEntry>::const_iterator it = entries.begin();
+	 it != entries.end(); it++) 
+	names.push_back("XSFN"+it->term);
 
     if (names.empty()) {
 	// Build an impossible query: we know its impossible because we
@@ -1385,11 +1362,11 @@ const string regSpecChars = "(.[{";
 bool Db::termMatch(MatchType typ, const string &lang,
 		   const string &root, 
 		   list<TermMatchEntry>& res,
-		   int max)
+		   int max, 
+		   const string& field)
 {
     if (!m_ndb || !m_ndb->m_isopen)
 	return false;
-
     Xapian::Database db = m_ndb->m_iswritable ? m_ndb->wdb: m_ndb->db;
 
     res.clear();
@@ -1399,6 +1376,11 @@ bool Db::termMatch(MatchType typ, const string &lang,
     dumb_string(root, droot);
     string nochars = typ == ET_WILD ? wildSpecChars : regSpecChars;
 
+    string prefix;
+    if (!field.empty()) {
+	(void)fieldToPrefix(field, prefix); 
+    }
+
     if (typ == ET_STEM) {
 	if (!stemExpand(lang, root, res, max))
 	    return false;
@@ -1429,33 +1411,43 @@ bool Db::termMatch(MatchType typ, const string &lang,
 	string::size_type es = droot.find_first_of(nochars);
 	string is;
 	switch (es) {
-	case string::npos: is = droot;break;
-	case 0: break;
-	default: is = droot.substr(0, es);break;
+	case string::npos: is = prefix + droot; break;
+	case 0: is = prefix; break;
+	default: is = prefix + droot.substr(0, es); break;
 	}
 	LOGDEB(("termMatch: initsec: [%s]\n", is.c_str()));
 
-	Xapian::TermIterator it = db.allterms_begin(); 
-	if (!is.empty())
-	    it.skip_to(is.c_str());
-	for (int n = 0;it != db.allterms_end(); it++) {
-	    // If we're beyond the terms matching the initial string, end
-	    if (!is.empty() && (*it).find(is) != 0)
-		break;
-	    // Don't match special internal terms beginning with uppercase ascii
-	    if ((*it).at(0) >= 'A' && (*it).at(0) <= 'Z')
-		continue;
-	    if (typ == ET_WILD) {
-		if (fnmatch(droot.c_str(), (*it).c_str(), 0) == FNM_NOMATCH)
-		    continue;
-	    } else {
-		if (regexec(&reg, (*it).c_str(), 0, 0, 0))
-		    continue;
+	string ermsg;
+	try {
+	    Xapian::TermIterator it = db.allterms_begin(); 
+	    if (!is.empty())
+		it.skip_to(is.c_str());
+	    for (int n = 0; it != db.allterms_end(); it++) {
+		// If we're beyond the terms matching the initial string, end
+		if (!is.empty() && (*it).find(is) != 0)
+		    break;
+		string term;
+		if (!prefix.empty())
+		    term = (*it).substr(prefix.length());
+		else
+		    term = *it;
+		if (typ == ET_WILD) {
+		    if (fnmatch(droot.c_str(), term.c_str(), 0) == FNM_NOMATCH)
+			continue;
+		} else {
+		    if (regexec(&reg, term.c_str(), 0, 0, 0))
+			continue;
+		}
+		// Do we want stem expansion here? We don't do it for now
+		res.push_back(TermMatchEntry(term, it.get_termfreq()));
+		++n;
 	    }
-	    // Do we want stem expansion here? We don't do it for now
-	    res.push_back(TermMatchEntry(*it, it.get_termfreq()));
-	    ++n;
+	} XCATCHERROR(ermsg);
+	if (!ermsg.empty()) {
+	    LOGERR(("termMatch: %s\n", ermsg.c_str()));
+	    return false;
 	}
+
 	if (typ == ET_REGEXP) {
 	    regfree(&reg);
 	}

src/rcldb/rcldb.h

@@ -16,7 +16,7 @@
  */
 #ifndef _DB_H_INCLUDED_
 #define _DB_H_INCLUDED_
-/* @(#$Id: rcldb.h,v 1.61 2008-08-26 07:38:29 dockes Exp $  (C) 2004 J.F.Dockes */
+/* @(#$Id: rcldb.h,v 1.62 2008-09-16 08:18:30 dockes Exp $  (C) 2004 J.F.Dockes */
 
 #include <string>
 #include <list>
@@ -156,9 +156,11 @@ class Db {
      * Stem expansion is performed if lang is not empty */
     enum MatchType {ET_WILD, ET_REGEXP, ET_STEM};
     bool termMatch(MatchType typ, const string &lang, const string &s, 
-		   list<TermMatchEntry>& result, int max = -1);
+		   list<TermMatchEntry>& result, int max = -1, 
+		   const string& field = "");
 
-    /** Specific filename wildcard expansion */
+    /** Special filename wildcard to XSFN terms expansion.
+	internal/searchdata use only */
     bool filenameWildExp(const string& exp, list<string>& names);
 
     /** Set parameters for synthetic abstract generation */

src/rcldb/rcldb_p.h

@@ -4,7 +4,7 @@
 #include "xapian.h"
 
 namespace Rcl {
-/* @(#$Id: rcldb_p.h,v 1.4 2008-09-05 10:34:17 dockes Exp $  (C) 2007 J.F.Dockes */
+/* @(#$Id: rcldb_p.h,v 1.5 2008-09-16 08:18:30 dockes Exp $  (C) 2007 J.F.Dockes */
 
 // Generic Xapian exception catching code. We do this quite often,
 // and I have no idea how to do this except for a macro
@@ -70,5 +70,14 @@ class Db::Native {
     bool subDocs(const string &udi, vector<Xapian::docid>& docids);
 
 };
+
+// Field names inside the index data record may differ from the rcldoc ones
+// (esp.: caption / title)
+inline const string& docfToDatf(const string& df)
+{
+    static const string keycap("caption");
+    return df.compare(Doc::keytt) ? df : keycap;
+}
+
 }
 #endif /* _rcldb_p_h_included_ */

src/rcldb/rcldoc.cpp

@@ -1,14 +1,25 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: rcldoc.cpp,v 1.1 2008-09-08 16:49:10 dockes Exp $ (C) 2007 J.F.Dockes";
+static char rcsid[] = "@(#$Id: rcldoc.cpp,v 1.2 2008-09-16 08:18:30 dockes Exp $ (C) 2007 J.F.Dockes";
 #endif
 
 
 #include "rcldoc.h"
 namespace Rcl {
-const string Doc::keyabs("abstract");
-const string Doc::keyau("author");
-const string Doc::keyfn("filename");
-const string Doc::keykw("keywords");
-const string Doc::keyrr("relevancyrating");
-const string Doc::keytt("title");
+    const string Doc::keyurl("url");
+    const string Doc::keyfn("filename");
+    const string Doc::keyipt("ipath");
+    const string Doc::keytp("mtype");
+    const string Doc::keyfmt("fmtime");
+    const string Doc::keydmt("dmtime");
+    const string Doc::keymt("mtime");
+    const string Doc::keyoc("origcharset");
+    const string Doc::keyfs("fbytes");
+    const string Doc::keyds("dbytes");
+    const string Doc::keysz("size");
+    const string Doc::keysig("sig");
+    const string Doc::keyrr("relevancyrating");
+    const string Doc::keyabs("abstract");
+    const string Doc::keyau("author");
+    const string Doc::keytt("title");
+    const string Doc::keykw("keywords");
 }

src/rcldb/rcldoc.h

@@ -16,7 +16,7 @@
  */
 #ifndef _RCLDOC_H_INCLUDED_
 #define _RCLDOC_H_INCLUDED_
-/* @(#$Id: rcldoc.h,v 1.9 2008-09-08 16:49:10 dockes Exp $  (C) 2006 J.F.Dockes */
+/* @(#$Id: rcldoc.h,v 1.10 2008-09-16 08:18:30 dockes Exp $  (C) 2006 J.F.Dockes */
 
 #include <string>
 #include <map>
@@ -44,9 +44,9 @@ class Doc {
     // can be accessed after a query without fetching the actual document).
     // We indicate the routine that sets them up during indexing
     
-    // This is just "file://" + binary filename. No transcoding: this
-    // is used to access files
-    // Index: computed from fn by Db::add caller. Query: from doc data.
+    // This is just "file://" + binary or url-encoded filename. No
+    // transcoding: this is used to access files Index: computed from
+    // fn by Db::add caller. Query: from doc data.
     string url;
 
     // Transcoded version of the simple file name for SFN-prefixed
@@ -134,12 +134,29 @@ class Doc {
 	pc = 0;
 	xdocid = 0;
     }
-    static const string keyfn;
-    static const string keyrr;
-    static const string keyabs;
-    static const string keyau;
-    static const string keytt;
-    static const string keykw;
+
+    // The official names for recoll native fields when used in a text
+    // context (ie: the python interface duplicates some of the fixed
+    // fields in the meta array, these are the names used). Defined in
+    // rcldoc.cpp. For fields stored in the meta[] array (ie, title,
+    // author), filters _must_ use these values
+    static const string keyurl; // url
+    static const string keyfn;  // file name
+    static const string keyipt; // ipath
+    static const string keytp;  // mime type
+    static const string keyfmt; // file mtime
+    static const string keydmt; // document mtime
+    static const string keymt;  // mtime dmtime if set else fmtime
+    static const string keyoc;  // original charset
+    static const string keyfs;  // file size
+    static const string keyds;  // document size
+    static const string keysz;  // dbytes if set else fbytes
+    static const string keysig; // sig
+    static const string keyrr;  // relevancy rating
+    static const string keyabs; // abstract
+    static const string keyau;  // author
+    static const string keytt;  // title
+    static const string keykw;  // keywords
 };
 
 

src/rcldb/rclquery.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: rclquery.cpp,v 1.5 2008-09-05 11:45:16 dockes Exp $ (C) 2008 J.F.Dockes";
+static char rcsid[] = "@(#$Id: rclquery.cpp,v 1.6 2008-09-16 08:18:30 dockes Exp $ (C) 2008 J.F.Dockes";
 #endif
 
 #include <stdlib.h>
@@ -8,6 +8,8 @@ static char rcsid[] = "@(#$Id: rclquery.cpp,v 1.5 2008-09-05 11:45:16 dockes Exp
 #include <list>
 #include <vector>
 
+#include "xapian/sorter.h"
+
 #include "rcldb.h"
 #include "rcldb_p.h"
 #include "rclquery.h"
@@ -20,6 +22,8 @@ static char rcsid[] = "@(#$Id: rclquery.cpp,v 1.5 2008-09-05 11:45:16 dockes Exp
 #ifndef NO_NAMESPACES
 namespace Rcl {
 #endif
+
+
 class FilterMatcher : public Xapian::MatchDecider {
 public:
     FilterMatcher(const string &topdir)
@@ -41,7 +45,7 @@ public:
 
 	// The only filtering for now is on file path (subtree)
 	string url;
-	parms.get(string("url"), url);
+	parms.get(Doc::keyurl, url);
 	LOGDEB2(("FilterMatcher topdir [%s] url [%s]\n",
 		 m_topdir.c_str(), url.c_str()));
 	if (url.find(m_topdir, 7) == 7) {
@@ -55,14 +59,46 @@ private:
     string m_topdir;
 };
 
+// Sort helper class
+class QSorter : public Xapian::Sorter {
+public:
+    QSorter(const string& f) : m_fld(docfToDatf(f) + "=") {}
+
+    virtual std::string operator()(const Xapian::Document& xdoc) const {
+	string data = xdoc.get_data();
+
+	// It would be simpler to do the record->Rcl::Doc thing, but
+	// hand-doing this will be faster. It makes more assumptions
+	// about the format than a ConfTree though:
+	string::size_type i1, i2;
+	i1 = data.find(m_fld);
+	if (i1 == string::npos) 
+	    return string();
+	i1 += m_fld.length();
+	if (i1 >= data.length())
+	    return string();
+	i2 = data.find_first_of("\n\r", i1);
+	if (i2 == string::npos)
+	    return string();
+	return data.substr(i1, i2-i1);
+    }
+
+private:
+    string m_fld;
+};
+
 Query::Query(Db *db)
-    : m_nq(new Native(this)), m_db(db)
+    : m_nq(new Native(this)), m_db(db), m_sorter(0)
 {
 }
 
 Query::~Query()
 {
     deleteZ(m_nq);
+    if (m_sorter) {
+	delete (QSorter*)m_sorter;
+	m_sorter = 0;
+    }
 }
 
 string Query::getReason() const
@@ -75,6 +111,7 @@ Db *Query::whatDb()
     return m_db;
 }
 
+
 //#define ISNULL(X) (X).isNull()
 #define ISNULL(X) !(X)
 
@@ -114,6 +151,17 @@ bool Query::setQuery(RefCntr<SearchData> sdata, int opts,
     try {
 	m_nq->enquire = new Xapian::Enquire(m_db->m_ndb->db);
 	m_nq->enquire->set_query(m_nq->query);
+	if (!sdata->getSortBy().empty()) {
+	    if (m_sorter) {
+		delete (QSorter*)m_sorter;
+		m_sorter = 0;
+	    }
+	    m_sorter = new QSorter(sdata->getSortBy());
+	    // It really seems there is a xapian bug about sort order, we 
+	    // invert here.
+	    m_nq->enquire->set_sort_by_key((QSorter*)m_sorter, 
+					   !sdata->getSortAscending());
+	}
 	m_nq->mset = Xapian::MSet();
 	// Get the query description and trim the "Xapian::Query"
 	d = m_nq->query.get_description();

src/rcldb/rclquery.h

@@ -1,6 +1,6 @@
 #ifndef _rclquery_h_included_
 #define _rclquery_h_included_
-/* @(#$Id: rclquery.h,v 1.2 2008-07-01 08:31:08 dockes Exp $  (C) 2008 J.F.Dockes */
+/* @(#$Id: rclquery.h,v 1.3 2008-09-16 08:18:30 dockes Exp $  (C) 2008 J.F.Dockes */
 /*
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -87,6 +87,7 @@ private:
     string m_filterTopDir; // Current query filter on subtree top directory 
     string m_reason; // Error explanation
     Db    *m_db;
+    void  *m_sorter;
     unsigned int m_qOpts;
     /* Copyconst and assignement private and forbidden */
     Query(const Query &) {}

src/rcldb/searchdata.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: searchdata.cpp,v 1.22 2008-08-28 15:42:43 dockes Exp $ (C) 2006 J.F.Dockes";
+static char rcsid[] = "@(#$Id: searchdata.cpp,v 1.23 2008-09-16 08:18:30 dockes Exp $ (C) 2006 J.F.Dockes";
 #endif
 /*
  *   This program is free software; you can redistribute it and/or modify
@@ -153,6 +153,14 @@ void SearchData::erase() {
     m_reason.erase();
 }
 
+void SearchData::setSortBy(const string& fld, bool ascending) {
+    RclConfig *cfg = RclConfig::getMainConfig();
+    m_sortField = cfg->fieldCanon(stringtolower(fld));
+    m_sortAscending = ascending;
+    LOGDEB0(("SearchData::setSortBy: [%s] %s\n", m_sortField.c_str(),
+	     m_sortAscending ? "ascending" : "descending"));
+}
+
 // Am I a file name only search ? This is to turn off term highlighting
 bool SearchData::fileNameOnly() 
 {
@@ -572,9 +580,9 @@ bool SearchDataClauseFilename::toNativeQuery(Rcl::Db &db, void *p,
     list<string> names;
     for (list<string>::iterator it = patterns.begin();
 	 it != patterns.end(); it++) {
-	// This relies on filenameWildExp not resetting and always
-	// adding to the input
-	db.filenameWildExp(*it, names);
+	list<string> more;
+	db.filenameWildExp(*it, more);
+	names.splice(names.end(), more);
     }
     // Build a query out of the matching file name terms.
     *qp = Xapian::Query(Xapian::Query::OP_OR, names.begin(), names.end());

src/rcldb/searchdata.h

@@ -16,7 +16,7 @@
  */
 #ifndef _SEARCHDATA_H_INCLUDED_
 #define _SEARCHDATA_H_INCLUDED_
-/* @(#$Id: searchdata.h,v 1.17 2008-09-08 15:47:44 dockes Exp $  (C) 2004 J.F.Dockes */
+/* @(#$Id: searchdata.h,v 1.18 2008-09-16 08:18:30 dockes Exp $  (C) 2004 J.F.Dockes */
 
 /** 
  * Structures to hold data coming almost directly from the gui
@@ -110,11 +110,17 @@ public:
     /** Add file type for filtering results */
     void addFiletype(const string& ft) {m_filetypes.push_back(ft);}
 
+    /** Choose sort order. Should this be in RclQuery instead ? */
+    void setSortBy(const string& fld, bool ascending = true);
+    const string& getSortBy() const {return m_sortField;}
+    bool getSortAscending() const {return m_sortAscending;}
 private:
-    SClType                    m_tp; // Only SCLT_AND or SCLT_OR here
-    vector<SearchDataClause *> m_query;
-    vector<string>             m_filetypes; // Restrict to filetypes if set.
-    string                     m_topdir; // Restrict to subtree.
+    SClType                   m_tp; // Only SCLT_AND or SCLT_OR here
+    vector<SearchDataClause*> m_query;
+    vector<string>            m_filetypes; // Restrict to filetypes if set.
+    string                    m_topdir; // Restrict to subtree.
+    string                    m_sortField;
+    bool                      m_sortAscending;
     // Printable expanded version of the complete query, retrieved/set
     // from rcldb after the Xapian::setQuery() call
     string m_description;