Enable recollindex command line option to use posix_fadvise() to spare the page cache

src/common/autoconfig.h.in

@@ -57,6 +57,9 @@
 /* Define to 1 if you have the `mkdtemp' function. */
 #undef HAVE_MKDTEMP
 
+/* Define to 1 if you have the `posix_fadvise' function. */
+#undef HAVE_POSIX_FADVISE
+
 /* Define to 1 if you have the `posix_spawn' function. */
 #undef HAVE_POSIX_SPAWN
 
@@ -114,9 +117,6 @@
 /* Define to the sub-directory where libtool stores uninstalled libraries. */
 #undef LT_OBJDIR
 
-/* Evict indexed files from page cache */
-#undef NOCACHE_INDEXED
-
 /* Define to the address where bug reports for this package should be sent. */
 #undef PACKAGE_BUGREPORT
 

src/configure.ac

@@ -67,7 +67,7 @@ AC_CHECK_HEADERS([sys/mount.h sys/statfs.h sys/statvfs.h sys/vfs.h malloc.h mall
 #endif
 ])
 
-AC_CHECK_FUNCS([posix_spawn setrlimit kqueue vsnprintf malloc_trim])
+AC_CHECK_FUNCS([posix_spawn setrlimit kqueue vsnprintf malloc_trim posix_fadvise])
 
 # Use specific 'file' command ? (Useful on solaris to specify
 # /usr/local/bin/file instead of the system's which doesn't understand '-i'
@@ -483,12 +483,6 @@ if test X$enableQT = Xyes ; then
   ##################### End QT stuff
 fi
 
-AC_ARG_ENABLE([nocache],
-    AS_HELP_STRING([--enable-nocache], [Evict indexed files from page cache]))
-AS_IF([test "x$enable_nocache" = "xyes"], [
-    AC_DEFINE(NOCACHE_INDEXED, 1, [Evict indexed files from page cache])
-])
-
 ### X11: this is needed for the session monitoring code (in recollindex -m)
 AC_ARG_ENABLE(x11mon, 
     AC_HELP_STRING([--disable-x11mon],

src/index/fsindexer.cpp

@@ -47,7 +47,7 @@
 #include "rclinit.h"
 #include "extrameta.h"
 #include "utf8fn.h"
-#if defined(NOCACHE_INDEXED) && !defined(_WIN32)
+#if defined(HAVE_POSIX_FADVISE)
 #include <unistd.h>
 #include <fcntl.h>
 #endif
@@ -180,6 +180,7 @@ bool FsIndexer::index(int flags)
 {
     bool quickshallow = (flags & ConfIndexer::IxFQuickShallow) != 0;
     m_noretryfailed = (flags & ConfIndexer::IxFNoRetryFailed) != 0;
+    m_cleancache = (flags & ConfIndexer::IxFCleanCache) != 0;
     Chrono chron;
     if (!init())
         return false;
@@ -345,6 +346,7 @@ bool FsIndexer::indexFiles(list<string>& files, int flags)
 {
     LOGDEB("FsIndexer::indexFiles\n");
     m_noretryfailed = (flags & ConfIndexer::IxFNoRetryFailed) != 0;
+    m_cleancache = (flags & ConfIndexer::IxFCleanCache) != 0;
     bool ret = false;
 
     if (!init())
@@ -866,11 +868,11 @@ FsIndexer::processonefile(RclConfig *config,
                 m_purgeCandidates.record(parent_udi);
             }
         }
-#if defined(NOCACHE_INDEXED) && !defined(_WIN32) && defined(POSIX_FADV_DONTNEED)
+#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
         // See framagit issue 26. If this appears to be a good idea
         // after all (not sure), we'll need a command line switch to
         // control it. For now it's compile-time only.
-        if (nullptr == getenv("NO_NOCACHE_INDEXED")) {
+        if (m_cleancache) {
             int fd = open(fn.c_str(), O_RDONLY);
             if (fd >= 0) {
                 if (posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED)) {

src/index/fsindexer.h

@@ -143,7 +143,9 @@ private:
 
     // No retry of previously failed files
     bool         m_noretryfailed;
-
+    // use FADV_DONTNEED if available
+    bool         m_cleancache{false};
+    
 #ifdef IDX_THREADS
     friend void *FsIndexerDbUpdWorker(void*);
     friend void *FsIndexerInternfileWorker(void*);

src/index/indexer.h

@@ -82,6 +82,8 @@ public:
                  // Do perform purge pass even if we can't be sure we saw
                  // all files
                  IxFDoPurge = 16,
+                 // Evict each indexed file from the page cache.
+                 IxFCleanCache = 32,
     };
 
     /** Run indexers */

src/index/recollindex.cpp

@@ -62,30 +62,30 @@ using namespace std;
 
 // Command line options
 static int     op_flags;
-#define OPT_MOINS 0x1
 #define OPT_C 0x1     
-#define OPT_D 0x2     
+#define OPT_c 0x2
-#define OPT_E 0x4     
+#define OPT_d 0x4     
-#define OPT_K 0x8     
+#define OPT_D 0x8     
-#define OPT_P 0x10    
+#define OPT_E 0x10    
-#define OPT_R 0x20    
+#define OPT_e 0x20    
-#define OPT_S 0x40    
+#define OPT_f 0x40    
-#define OPT_Z 0x80    
+#define OPT_h 0x80    
-#define OPT_c 0x200   
+#define OPT_i 0x200   
-#define OPT_e 0x400   
+#define OPT_K 0x400   
-#define OPT_f 0x800   
+#define OPT_k 0x800   
-#define OPT_h 0x1000  
+#define OPT_l 0x1000  
-#define OPT_i 0x2000  
+#define OPT_m 0x2000  
-#define OPT_k 0x4000  
+#define OPT_n 0x4000  
-#define OPT_l 0x8000  
+#define OPT_P 0x8000  
-#define OPT_m 0x10000 
+#define OPT_p 0x10000 
-#define OPT_n 0x20000
+#define OPT_R 0x20000 
-#define OPT_p 0x40000
+#define OPT_r 0x40000 
-#define OPT_r 0x80000 
+#define OPT_S 0x80000 
 #define OPT_s 0x100000
 #define OPT_w 0x200000
 #define OPT_x 0x400000
-#define OPT_z 0x800000
+#define OPT_Z 0x800000
+#define OPT_z 0x1000000
 
 ReExec *o_reexec;
 
@@ -475,7 +475,11 @@ static const char usage [] =
 #endif
 "Common options:\n"
 "    -c <configdir> : specify config directory, overriding $RECOLL_CONFDIR\n"
-                                                           ;
+#if defined(HAVE_POSIX_FADVISE)
+"    -d : call fadvise() with the POSIX_FADV_DONTNEED flag on indexed files\n"
+"          (avoids trashing the page cache)\n";
+#endif
+;
 
 static void Usage()
 {
@@ -584,8 +588,8 @@ static std::string orig_cwd;
 // Unicode. It was first thought possible to use a temporary file to
 // hold the args, and make sure that the path for this would be ASCII,
 // based on using shortpath(). Unfortunately, this does not work in
-// all cases, so the second change was to use wmain(), but the
+// all cases, so the second change was to use wmain(). The
-// now largely redundant args-in-file passing trick was kept anyway.
+// args-in-file was removed quite a long time after.
 #if USE_WMAIN
 int wmain(int argc, wchar_t *argv[])
 #else
@@ -600,6 +604,8 @@ int main(int argc, char *argv[])
     o_reexec->init(argc, argv);
 #endif
 
+    // The bizarre conversion to vector stayed from the time when we
+    // used a file for passing options.
     vector<string> args = argstovector(argc, argv);
 
     vector<string> selpatterns;
@@ -620,6 +626,9 @@ int main(int argc, char *argv[])
 #ifdef RCL_MONITOR
             case 'C': op_flags |= OPT_C; break;
             case 'D': op_flags |= OPT_D; break;
+#endif
+#if defined(HAVE_POSIX_FADVISE)
+            case 'd': op_flags |= OPT_d; break;
 #endif
             case 'E': op_flags |= OPT_E; break;
             case 'e': op_flags |= OPT_e; break;
@@ -762,6 +771,12 @@ int main(int argc, char *argv[])
         LOGDEB("recollindex: files in error will be retried\n");
     }
 
+#if defined(HAVE_POSIX_FADVISE)
+    if (op_flags & OPT_d) {
+        indexerFlags |= ConfIndexer::IxFCleanCache;
+    }
+#endif
+    
     Pidfile pidfile(config->getPidfile());
     updater = new MyUpdater(config);
     lockorexit(&pidfile, config);