create stemming db on queue timeout if needed

src/index/indexer.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: indexer.cpp,v 1.40 2006-10-24 09:09:36 dockes Exp $ (C) 2004 J.F.Dockes";
+static char rcsid[] = "@(#$Id: indexer.cpp,v 1.41 2006-10-24 14:28:38 dockes Exp $ (C) 2004 J.F.Dockes";
 #endif
 /*
  *   This program is free software; you can redistribute it and/or modify
@@ -115,8 +115,27 @@ bool DbIndexer::indexDb(bool resetbefore, list<string> *topdirs)
     // filesystem anymore.
     m_db.purge();
 
-    // Create stemming databases. We also remove those which are not
+    createStemmingDatabases();
-    // configured.
+    createAspellDict();
+
+    // The close would be done in our destructor, but we want status here
+    if (m_updater) {
+	m_updater->status.phase = DbIxStatus::DBIXS_CLOSING;
+	m_updater->status.fn.erase();
+	m_updater->update();
+    }
+    if (!m_db.close()) {
+	LOGERR(("DbIndexer::index: error closing database in %s\n", 
+		m_dbdir.c_str()));
+	return false;
+    }
+    return true;
+}
+
+// Create stemming databases. We also remove those which are not
+// configured.
+bool DbIndexer::createStemmingDatabases()
+{
     string slangs;
     if (m_config->getConfParam("indexstemminglanguages", slangs)) {
 	list<string> langs;
@@ -139,20 +158,6 @@ bool DbIndexer::indexDb(bool resetbefore, list<string> *topdirs)
 	    m_db.createStemDb(*it);
 	}
     }
-
-    createAspellDict();
-
-    // The close would be done in our destructor, but we want status here
-    if (m_updater) {
-	m_updater->status.phase = DbIxStatus::DBIXS_CLOSING;
-	m_updater->status.fn.erase();
-	m_updater->update();
-    }
-    if (!m_db.close()) {
-	LOGERR(("DbIndexer::index: error closing database in %s\n", 
-		m_dbdir.c_str()));
-	return false;
-    }
     return true;
 }
 

src/index/indexer.h

@@ -16,7 +16,7 @@
  */
 #ifndef _INDEXER_H_INCLUDED_
 #define _INDEXER_H_INCLUDED_
-/* @(#$Id: indexer.h,v 1.20 2006-10-22 14:47:13 dockes Exp $  (C) 2004 J.F.Dockes */
+/* @(#$Id: indexer.h,v 1.21 2006-10-24 14:28:38 dockes Exp $  (C) 2004 J.F.Dockes */
 
 #include <string>
 #include <list>
@@ -119,6 +119,9 @@ class DbIndexer : public FsTreeWalkerCB {
     /** Purge a list of files. */
     bool purgeFiles(const std::list<string> &files);
 
+    /** Stemming reset to config: create needed, delete unconfigured */
+    bool createStemmingDatabases();
+
     /** Create stem database for given language */
     bool createStemDb(const string &lang);
 
@@ -148,5 +151,6 @@ class DbIndexer : public FsTreeWalkerCB {
  * a list of files (either from the monitor or the command line) */
 extern bool indexfiles(RclConfig *config, const list<string> &filenames);
 extern bool purgefiles(RclConfig *config, const list<string> &filenames);
+extern bool createAuxDbs(RclConfig *config);
 
 #endif /* _INDEXER_H_INCLUDED_ */

src/index/rclmon.h

@@ -2,7 +2,7 @@
 #define _RCLMON_H_INCLUDED_
 #include "autoconfig.h"
 #ifdef RCL_MONITOR
-/* @(#$Id: rclmon.h,v 1.4 2006-10-24 12:48:08 dockes Exp $  (C) 2006 J.F.Dockes */
+/* @(#$Id: rclmon.h,v 1.5 2006-10-24 14:28:38 dockes Exp $  (C) 2006 J.F.Dockes */
 /**
  * Definitions for the real-time monitoring recoll. 
  * We're interested in file modifications, deletions and renaming.
@@ -49,7 +49,7 @@ class RclMonEventQueue {
     ~RclMonEventQueue();
     /** Unlock queue and wait until there are new events. 
      *  Returns with the queue locked */
-    bool wait();
+    bool wait(int secs = -1, bool *timedout = 0);
     /** Unlock queue */
     bool unlock();
     /** Lock queue. */

src/index/rclmonprc.cpp

@@ -2,7 +2,7 @@
 
 #ifdef RCL_MONITOR
 #ifndef lint
-static char rcsid[] = "@(#$Id: rclmonprc.cpp,v 1.4 2006-10-24 12:48:09 dockes Exp $ (C) 2006 J.F.Dockes";
+static char rcsid[] = "@(#$Id: rclmonprc.cpp,v 1.5 2006-10-24 14:28:38 dockes Exp $ (C) 2006 J.F.Dockes";
 #endif
 /*
  *   This program is free software; you can redistribute it and/or modify
@@ -89,13 +89,34 @@ RclMonEvent RclMonEventQueue::pop()
 /** Wait until there is something to process on the queue.
  *  Must be called with the queue locked 
  */
-bool RclMonEventQueue::wait() 
+bool RclMonEventQueue::wait(int seconds, bool *top)
 {
     if (!empty())
 	return true;
-    if (pthread_cond_wait(&m_data->m_cond, &m_data->m_mutex)) {
+
-	LOGERR(("RclMonEventQueue::wait: pthread_cond_wait failed\n"));
+    int err;
-	return false;
+    if (seconds > 0) {
+	struct timespec to;
+	to.tv_sec = time(0L) + seconds;
+	to.tv_nsec = 0;
+	if (top)
+	    *top = false;
+	if ((err = 
+	     pthread_cond_timedwait(&m_data->m_cond, &m_data->m_mutex, &to))) {
+	    if (err == ETIMEDOUT) {
+		*top = true;
+		return true;
+	    }
+	    LOGERR(("RclMonEventQueue::wait:pthread_cond_timedwait failed"
+		    "with err %d\n", err));
+	    return false;
+	}
+    } else {
+	if ((err = pthread_cond_wait(&m_data->m_cond, &m_data->m_mutex))) {
+	    LOGERR(("RclMonEventQueue::wait: pthread_cond_wait failed"
+		    "with err %d\n", err));
+	    return false;
+	}
     }
     return true;
 }
@@ -240,7 +261,13 @@ bool startMonitor(RclConfig *conf, bool nofork)
 	return false;
     }
     LOGDEB(("start_monitoring: entering main loop\n"));
-    while (rclEQ.wait()) {
+    bool timedout;
+    bool didsomething = false;
+
+    // We set a timeout of 10mn. If we do timeout, and there have been some
+    // indexing activity since the last such operation, we'll update the 
+    // auxiliary data (stemming and spelling)
+    while (rclEQ.wait(10 * 60, &timedout)) {
 	LOGDEB2(("startMonitor: wait returned\n"));
 	if (!rclEQ.ok())
 	    break;
@@ -270,10 +297,29 @@ bool startMonitor(RclConfig *conf, bool nofork)
 	// Unlock queue before processing lists
 	rclEQ.unlock();
 	// Process
-	if (!indexfiles(conf, modified))
+	if (!modified.empty()) {
-	    break;
+	    if (!indexfiles(conf, modified))
-	if (!purgefiles(conf, deleted))
+		break;
-	    break;
+	    didsomething = true;
+	}
+	if (!deleted.empty()) {
+	    if (!purgefiles(conf, deleted))
+		break;
+	    didsomething = true;
+	}
+
+	if (timedout) {
+	    LOGDEB2(("Monitor: queue wait timed out\n"));
+	    // Timed out. there must not be much activity around here. 
+	    // If anything was modified, process the end-of-indexing
+	    // tasks: stemming and spelling database creations.
+	    if (didsomething) {
+		didsomething = false;
+		if (!createAuxDbs(conf))
+		    break;
+	    }
+	}
+
 	// Lock queue before waiting again
 	rclEQ.lock();
     }

src/index/recollindex.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.25 2006-10-22 14:47:14 dockes Exp $ (C) 2004 J.F.Dockes";
+static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.26 2006-10-24 14:28:38 dockes Exp $ (C) 2004 J.F.Dockes";
 #endif
 /*
  *   This program is free software; you can redistribute it and/or modify
@@ -171,6 +171,21 @@ bool purgefiles(RclConfig *config, const list<string> &filenames)
 	return dbindexer->purgeFiles(myfiles);
 }
 
+// Create stemming and spelling databases
+bool createAuxDbs(RclConfig *config)
+{
+    if (!makeDbIndexer(config) || !dbindexer)
+	return false;
+
+    if (!dbindexer->createStemmingDatabases())
+	return false;
+
+    if (!dbindexer->createAspellDict())
+	return false;
+
+    return true;
+}
+
 // Create additional stem database 
 static bool createstemdb(RclConfig *config, const string &lang)
 {