Avoid purging existing subdocuments on file indexing error (e.g.: maybe a file lock issue that will go away)

src/index/fsindexer.cpp

@@ -586,18 +586,14 @@ FsIndexer::processone(const std::string &fn, const struct stat *stp,
 
     // If we're changing directories, possibly adjust parameters (set
     // the current directory in configuration object)
-    if (flg == FsTreeWalker::FtwDirEnter || 
+    if (flg == FsTreeWalker::FtwDirEnter || flg == FsTreeWalker::FtwDirReturn) {
-	flg == FsTreeWalker::FtwDirReturn) {
         m_config->setKeyDir(fn);
-
         // Set up filter/skipped patterns for this subtree. 
         m_walker.setOnlyNames(m_config->getOnlyNames());
         m_walker.setSkippedNames(m_config->getSkippedNames());
-
         // Adjust local fields from config for this subtree
         if (m_havelocalfields)
             localfieldsfromconf();
-
         if (flg == FsTreeWalker::FtwDirReturn)
             return FsTreeWalker::FtwOk;
     }
@@ -616,6 +612,25 @@ FsIndexer::processone(const std::string &fn, const struct stat *stp,
     return processonefile(m_config, fn, stp, m_localfields);
 }
 
+// Start db update, either by queueing or by direct call
+bool FsIndexer::launchAddOrUpdate(const string& udi, const string& parent_udi,
+                                  Rcl::Doc& doc)
+{
+#ifdef IDX_THREADS
+    if (m_haveSplitQ) {
+        DbUpdTask *tp = new DbUpdTask(udi, parent_udi, doc);
+        if (!m_dwqueue.put(tp)) {
+            LOGERR("processonefile: wqueue.put failed\n");
+            return false;
+        } else {
+            return true;
+        }
+    }
+#endif
+
+    return m_db->addOrUpdate(udi, parent_udi, doc);
+}
+
 FsTreeWalker::Status 
 FsIndexer::processonefile(RclConfig *config, 
                           const std::string &fn, const struct stat *stp,
@@ -633,10 +648,9 @@ FsIndexer::processonefile(RclConfig *config,
     // excludedmimetypes, etc.
     config->setKeyDir(path_getfather(fn));
     
-    // Document signature. This is based on m/ctime and size and used
+    // File signature and up to date check. The sig is based on
-    // for the uptodate check (the value computed here is checked
+    // m/ctime and size and the possibly new value is checked against
-    // against the stored one). Changing the computation forces a full
+    // the stored one.
-    // reindex of course.
     string sig;
     makesig(stp, sig);
     string udi;
@@ -738,8 +752,9 @@ FsIndexer::processonefile(RclConfig *config,
             
             // If there is an error and the base doc was already seen,
             // we're done
-            if (fis == FileInterner::FIError && hadNullIpath)
+            if (fis == FileInterner::FIError && hadNullIpath) {
                 return FsTreeWalker::FtwOk;
+            }
             
             // Internal access path for multi-document files. If empty, this is
             // for the main file.
@@ -794,23 +809,10 @@ FsIndexer::processonefile(RclConfig *config,
 
             // Add document to database. If there is an ipath, add it
             // as a child of the file document.
-#ifdef IDX_THREADS
+            if (!launchAddOrUpdate(udi, doc.ipath.empty() ? 
-	    if (m_haveSplitQ) {
-		DbUpdTask *tp = new DbUpdTask(udi, doc.ipath.empty() ? 
-					      cstr_null : parent_udi, doc);
-		if (!m_dwqueue.put(tp)) {
-		    LOGERR("processonefile: wqueue.put failed\n");
-		    return FsTreeWalker::FtwError;
-		} 
-	    } else {
-#endif
-		if (!m_db->addOrUpdate(udi, doc.ipath.empty() ? 
                                    cstr_null : parent_udi, doc)) {
                 return FsTreeWalker::FtwError;
             } 
-#ifdef IDX_THREADS
-	    }
-#endif
 
             // Tell what we are doing and check for interrupt request
             if (m_updater) {
@@ -835,6 +837,15 @@ FsIndexer::processonefile(RclConfig *config,
             }
         }
 
+        if (fis == FileInterner::FIError) {
+            // In case of error, avoid purging any existing
+            // subdoc. For example on windows, this will avoid erasing
+            // all the emails from a .ost because it is currently
+            // locked by Outlook.
+            LOGDEB("processonefile: internfile error, marking "
+                   "subdocs as existing\n");
+            m_db->udiTreeMarkExisting(parent_udi);
+        } else {
             // If this doc existed and it's a container, recording for
             // possible subdoc purge (this will be used only if we don't do a
             // db-wide purge, e.g. if we're called from indexfiles()).
@@ -844,6 +855,7 @@ FsIndexer::processonefile(RclConfig *config,
                 m_purgeCandidates.record(parent_udi);
             }
         }
+    }
 
     // If we had no instance with a null ipath, we create an empty
     // document to stand for the file itself, to be used mainly for up
@@ -873,17 +885,9 @@ FsIndexer::processonefile(RclConfig *config,
 
         fileDoc.sig = sig;
 
-#ifdef IDX_THREADS
+        if (!launchAddOrUpdate(parent_udi, cstr_null, fileDoc)) {
-	if (m_haveSplitQ) {
-	    DbUpdTask *tp = new DbUpdTask(parent_udi, cstr_null, fileDoc);
-	    if (!m_dwqueue.put(tp))
             return FsTreeWalker::FtwError;
-	    else
-		return FsTreeWalker::FtwOk;
         }
-#endif
-	if (!m_db->addOrUpdate(parent_udi, cstr_null, fileDoc)) 
-	    return FsTreeWalker::FtwError;
     }
 
     return FsTreeWalker::FtwOk;

src/index/fsindexer.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2009 J.F.Dockes
+/* Copyright (C) 2009-2019 J.F.Dockes
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
  *   the Free Software Foundation; either version 2 of the License, or
@@ -33,19 +33,23 @@ struct stat;
 class DbUpdTask;
 class InternfileTask;
 
+namespace Rcl {
+class Doc;
+}
+
 /** Index selected parts of the file system
  
-Tree indexing: we inherits FsTreeWalkerCB so that, the processone()
+    Tree indexing: we inherits FsTreeWalkerCB so that, the processone()
-method is called by the file-system tree walk code for each file and
+    method is called by the file-system tree walk code for each file and
-directory. We keep all state needed while indexing, and finally call
+    directory. We keep all state needed while indexing, and finally call
-the methods to purge the db of stale entries and create the stemming
+    the methods to purge the db of stale entries and create the stemming
-databases.
+    databases.
 
-Single file(s) indexing: there are also calls to index or purge lists of files.
+    Single file(s) indexing: there are also calls to index or purge lists of files.
-No database purging or stem db updating in this case.
+    No database purging or stem db updating in this case.
 */
 class FsIndexer : public FsTreeWalkerCB {
- public:
+public:
     /** Constructor does nothing but store parameters 
      *
      * @param cnf Configuration data
@@ -76,18 +80,17 @@ class FsIndexer : public FsTreeWalkerCB {
     /** Make signature for file up to date checks */
     static void makesig(const struct stat *stp, string& out);
 
- private:
+    
+private:
 
     class PurgeCandidateRecorder {
     public:
         PurgeCandidateRecorder() 
             : dorecord(false) {}
-	void setRecord(bool onoff) 
+        void setRecord(bool onoff) {
-	{
             dorecord = onoff;
         }
-	void record(const string& udi)
+        void record(const string& udi) {
-	{
             // This test does not need to be protected: the value is set at
             // init and never changed.
             if (!dorecord)
@@ -97,8 +100,7 @@ class FsIndexer : public FsTreeWalkerCB {
 #endif
             udis.push_back(udi);
         }
-	const vector<string>& getCandidates() 
+        const vector<string>& getCandidates() {
-	{
             return udis;
         }
     private:
@@ -109,6 +111,9 @@ class FsIndexer : public FsTreeWalkerCB {
         std::vector<std::string> udis;
     };
 
+    bool launchAddOrUpdate(const std::string& udi,
+                           const std::string& parent_udi, Rcl::Doc& doc);
+
     FsTreeWalker m_walker;
     RclConfig   *m_config;
     Rcl::Db     *m_db;

src/rcldb/rcldb.cpp

@@ -2590,7 +2590,7 @@ bool Db::getSubDocs(const Doc &idoc, vector<Doc>& subdocs)
 // used for absent FS mountable volumes.
 bool Db::udiTreeMarkExisting(const string& udi)
 {
-    LOGDEB("Db::udiTreeWalk: " << udi << endl);
+    LOGDEB("Db::udiTreeMarkExisting: " << udi << endl);
     string wrapd = wrap_prefix(udi_prefix);
     string expr = udi + "*";