recollindex: do not retry files which previously failed to be indexed, except if they were changed since, or option -k is set

2015-04-23 10:37:37 +02:00
parent 906312519c
commit e6c0ca403d
9 changed files with 209 additions and 86 deletions
@@ -789,7 +789,6 @@ bool Db::open(OpenMode mode, OpenError *error)
                // (now: Xapian 1.2) and the separate objects seem to
                // trigger other Xapian issues, so the query db is now
                // a clone of the update one.
-//		m_ndb->xrdb = Xapian::Database(dir);
 		m_ndb->xrdb = m_ndb->xwdb;
 		LOGDEB(("Db::open: lastdocid: %d\n", 
 			m_ndb->xwdb.get_lastdocid()));
@@ -1725,23 +1724,70 @@ bool Db::doFlush()
    return true;
 }

+void Db::setExistingFlags(const string& udi, unsigned int docid)
+{
+    if (m_mode == DbRO)
+        return;
+    if (docid == (unsigned int)-1) {
+        LOGERR(("Db::setExistingFlags: called with bogus docid !!\n"));
+        return;
+    }
+#ifdef IDX_THREADS
+    PTMutexLocker lock(m_ndb->m_mutex);
+#endif
+    i_setExistingFlags(udi, docid);
+}
+
+void Db::i_setExistingFlags(const string& udi, unsigned int docid)
+{
+    // Set the up to date flag for the document and its subdocs
+    if (docid >= updated.size()) {
+        LOGERR(("needUpdate: existing docid beyond "
+                "updated.size(). Udi [%s], docid %u, "
+                "updated.size() %u\n", udi.c_str(), 
+                unsigned(docid), (unsigned)updated.size()));
+        return;
+    } else {
+        updated[docid] = true;
+    }
+
+    // Set the existence flag for all the subdocs (if any)
+    vector<Xapian::docid> docids;
+    if (!m_ndb->subDocs(udi, 0, docids)) {
+        LOGERR(("Rcl::Db::needUpdate: can't get subdocs\n"));
+        return;
+    }
+    for (vector<Xapian::docid>::iterator it = docids.begin();
+         it != docids.end(); it++) {
+        if (*it < updated.size()) {
+            LOGDEB2(("Db::needUpdate: docid %d set\n", *it));
+            updated[*it] = true;
+        }
+    }
+}
+
 // Test if doc given by udi has changed since last indexed (test sigs)
-bool Db::needUpdate(const string &udi, const string& sig, bool *existed)
+bool Db::needUpdate(const string &udi, const string& sig, 
+                    unsigned int *docidp, string *osigp)
 {
    if (m_ndb == 0)
        return false;

+    if (osigp)
+        osigp->clear();
+    if (docidp)
+        *docidp = 0;
+
    // If we are doing an in place or full reset, no need to test.
    if (o_inPlaceReset || m_mode == DbTrunc) {
-	// For in place reset, pretend the doc existed, to enable subdoc purge
-	if (existed)
-	    *existed = o_inPlaceReset;
+	// For in place reset, pretend the doc existed, to enable
+	// subdoc purge. The value is only used as a boolean in this case.
+	if (docidp && o_inPlaceReset) {
+	    *docidp = -1;
+        }
 	return true;
    }

-    if (existed)
-        *existed = false;
-
    string uniterm = make_uniterm(udi);
    string ermsg;

@@ -1773,8 +1819,9 @@ bool Db::needUpdate(const string &udi, const string& sig, bool *existed)
        return true;
    }

-    if (existed)
-        *existed = true;
+    if (docidp) {
+        *docidp = *docid;
+    }

    // Retrieve old file/doc signature from value
    string osig;
@@ -1785,6 +1832,11 @@ bool Db::needUpdate(const string &udi, const string& sig, bool *existed)
    }
    LOGDEB2(("Db::needUpdate: oldsig [%s] new [%s]\n",
             osig.c_str(), sig.c_str()));
+
+    if (osigp) {
+        *osigp = osig;
+    }
+
    // Compare new/old sig
    if (sig != osig) {
        LOGDEB(("Db::needUpdate:yes: olsig [%s] new [%s] [%s]\n",
@@ -1793,34 +1845,10 @@ bool Db::needUpdate(const string &udi, const string& sig, bool *existed)
        return true;
    }

-    // Up to date. 
+    // Up to date. Set the existance flags in the map for the doc and
+    // its subdocs.
    LOGDEB(("Db::needUpdate:no: [%s]\n", uniterm.c_str()));
-
-    if (m_mode 	!= DbRO) {
-        // Set the up to date flag for the document and its subdocs
-        if (*docid >= updated.size()) {
-            LOGERR(("needUpdate: existing docid beyond "
-                    "updated.size(). Udi [%s], docid %u, "
-                    "updated.size() %u\n", udi.c_str(), 
-                    unsigned(*docid), (unsigned)updated.size()));
-        } else {
-            updated[*docid] = true;
-        }
-
-        // Set the existence flag for all the subdocs (if any)
-        vector<Xapian::docid> docids;
-        if (!m_ndb->subDocs(udi, 0, docids)) {
-            LOGERR(("Rcl::Db::needUpdate: can't get subdocs\n"));
-            return true;
-        }
-        for (vector<Xapian::docid>::iterator it = docids.begin();
-             it != docids.end(); it++) {
-            if (*it < updated.size()) {
-                LOGDEB2(("Db::needUpdate: docid %d set\n", *it));
-                updated[*it] = true;
-            }
-        }
-    }
+    i_setExistingFlags(udi, *docid);
    return false;
 }

@@ -228,15 +228,36 @@ class Db {

    /* Update-related methods ******************************************/

-    /** Test if the db entry for the given udi is up to date (by
-     * comparing the input and stored sigs). This is used both when 
-     * indexing and querying (before opening a document using stale info),
+    /** Test if the db entry for the given udi is up to date.
+     *
+     * This is done by comparing the input and stored sigs. This is
+     * used both when indexing and querying (before opening a document 
+     * using stale info).
+     *
     * **This assumes that the udi pertains to the main index (idxi==0).**
-     * Side-effect when the db is writeable: set the existence flag
-     * for the file document and all subdocs if any (for later use by
-     * 'purge()')
+     *
+     * Side-effect when the db is writeable and the document up to
+     * date: set the existence flag for the file document and all
+     * subdocs if any (for later use by 'purge()')
+     *
+     * @param udi Unique Document Identifier (as chosen by indexer).
+     * @param sig New signature (as computed by indexer).
+     * @param xdocid[output] Non-zero if doc existed. Should be considered 
+     *    as opaque, to be used for a possible later call to setExistingFlags()
+     *    Note that if inplaceReset is set, the return value is non-zero but not
+     *    an actual docid, it's only used as a flag in this case.
+     * @param osig[output] old signature.
     */
-    bool needUpdate(const string &udi, const string& sig, bool *existed=0);
+    bool needUpdate(const string &udi, const string& sig, 
+                    unsigned int *xdocid = 0, std::string *osig = 0);
+
+    /** Set the existance flags for the document and its eventual subdocuments
+     * 
+     * This can be called by the indexer after needUpdate() has returned true,
+     * if the indexer does not wish to actually re-index (e.g.: the doc is 
+     * known to cause errors).
+     */
+    void setExistingFlags(const string& udi, unsigned int docid);

    /** Indicate if we are doing a systematic reindex. This complements
 	needUpdate() return */
@@ -488,6 +509,8 @@ private:
    friend void *DbUpdWorker(void*);
 #endif // IDX_THREADS

+    // Internal form of setExistingFlags: no locking
+    void i_setExistingFlags(const string& udi, unsigned int docid);
    // Internal form of close, can be called during destruction
    bool i_close(bool final);
    // Reinitialize when adding/removing additional dbs