1st version of real time monitor

src/common/rclconfig.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: rclconfig.cpp,v 1.30 2006-09-08 09:02:47 dockes Exp $ (C) 2004 J.F.Dockes";
+static char rcsid[] = "@(#$Id: rclconfig.cpp,v 1.31 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes";
 #endif
 /*
  *   This program is free software; you can redistribute it and/or modify
@@ -150,6 +150,26 @@ bool RclConfig::getConfParam(const std::string &name, bool *bvp)
     return true;
 }
 
+list<string> RclConfig::getTopdirs()
+{
+    list<string> tdl;
+    // Retrieve the list of directories to be indexed.
+    string topdirs;
+    if (!getConfParam("topdirs", topdirs)) {
+	LOGERR(("RclConfig::getTopdirs: no top directories in config\n"));
+	return tdl;
+    }
+    if (!stringToStrings(topdirs, tdl)) {
+	LOGERR(("RclConfig::getTopdirs: parse error for directory list\n"));
+	return tdl;
+    }
+    for (list<string>::iterator it = tdl.begin(); it != tdl.end(); it++) {
+	*it = path_tildexpand(*it);
+	*it = path_canon(*it);
+    }
+    return tdl;
+}
+
 // Get charset to be used for transcoding to utf-8 if unspecified by doc
 // For document contents:
 //   If defcharset was set (from the config or a previous call), use it.

src/common/rclconfig.h

@@ -16,9 +16,14 @@
  */
 #ifndef _RCLCONFIG_H_INCLUDED_
 #define _RCLCONFIG_H_INCLUDED_
-/* @(#$Id: rclconfig.h,v 1.22 2006-10-11 14:16:25 dockes Exp $  (C) 2004 J.F.Dockes */
+/* @(#$Id: rclconfig.h,v 1.23 2006-10-16 15:33:08 dockes Exp $  (C) 2004 J.F.Dockes */
 
 #include <list>
+#include <string>
+#ifndef NO_NAMESPACES
+using std::list;
+using std::string;
+#endif
 
 #include "conftree.h"
 #include "smallut.h"
@@ -61,6 +66,11 @@ class RclConfig {
     /** Get guessCharset for current keydir (was set during setKeydir) */
     bool getGuessCharset() {return guesscharset;}
 
+    /** Get list of top directories. This is needed from a number of places
+     * and needs some cleaning-up code. An empty list is always an error, no
+     * need for other status */
+    list<string> getTopdirs();
+
     /** Get database directory */
     string getDbDir();
 
@@ -70,7 +80,7 @@ class RclConfig {
      * The list is initialized on first call, and not changed for subsequent
      * setKeydirs.
      */
-    bool getStopSuffixes(std::list<std::string>& sufflist);
+    bool getStopSuffixes(list<string>& sufflist);
 
     /** 
      * Check in mimeconf if input mime type is a compressed one, and
@@ -79,26 +89,26 @@ class RclConfig {
      * The returned command has substitutable places for input file name 
      * and temp dir name, and will return output name
      */
-    bool getUncompressor(const std::string &mtpe, std::list<std::string>& cmd);
+    bool getUncompressor(const string &mtpe, list<string>& cmd);
 
     /** Use mimemap to compute mimetype */
-    std::string getMimeTypeFromSuffix(const std::string &suffix);
+    string getMimeTypeFromSuffix(const string &suffix);
 
     /** Get input filter from mimeconf for mimetype */
-    std::string getMimeHandlerDef(const std::string &mimetype);
+    string getMimeHandlerDef(const string &mimetype);
 
     /** Get external viewer exec string from mimeconf for mimetype */
-    std::string getMimeViewerDef(const std::string &mimetype);
+    string getMimeViewerDef(const string &mimetype);
 
     /** Get icon name from mimeconf for mimetype */
     string getMimeIconName(const string &mtype, string *path = 0);
 
     /** Get a list of all indexable mime types defined in mimemap */
-    std::list<string> getAllMimeTypes();
+    list<string> getAllMimeTypes();
 
     /** Find exec file for external filter. cmd is the command name from the
      * command string returned by getMimeHandlerDef */
-    std::string findFilter(const std::string& cmd);
+    string findFilter(const string& cmd);
 
     ~RclConfig() {
 	freeAll();
@@ -114,7 +124,7 @@ class RclConfig {
 	}
 	return *this;
     }
-    std::list<string> getConfNames(const string &sk) {
+    list<string> getConfNames(const string &sk) {
 	return m_conf->getNames(sk);
     }
 
@@ -129,7 +139,7 @@ class RclConfig {
     ConfStack<ConfTree> *mimemap;  // The files don't change with keydir, but their
     ConfStack<ConfTree> *mimeconf; // content may depend on it.
 
-    std::list<std::string> *stopsuffixes;
+    list<string> *stopsuffixes;
 
     // Parameters auto-fetched on setkeydir
     string defcharset;   // These are stored locally to avoid 

src/index/Makefile

@@ -2,16 +2,20 @@ depth = ..
 include $(depth)/mk/sysconf
 
 PROGS = recollindex csguess mimetype
-SRCS = recollindex.cpp
+SRCS = recollindex.cpp rclmonrcv.cpp rclmonprc.cpp
 
 all: depend $(PROGS) $(BIGLIB)
 
-RECOLLINDEX_OBJS= recollindex.o $(BIGLIB) $(MIMELIB)
+RECOLLINDEX_OBJS= recollindex.o rclmonrcv.o rclmonprc.o $(BIGLIB) $(MIMELIB)
 recollindex : $(RECOLLINDEX_OBJS)
 	$(CXX) $(ALL_CXXFLAGS) -o recollindex $(RECOLLINDEX_OBJS) \
-	       $(BSTATIC) $(LIBXAPIAN) $(LIBICONV) $(BDYNAMIC) $(LIBSYS)
+	       $(BSTATIC) $(LIBXAPIAN) $(LIBICONV) $(BDYNAMIC) -lfam $(LIBSYS)
 recollindex.o : recollindex.cpp
 	$(CXX) $(ALL_CXXFLAGS) -c -o recollindex.o $<
+rclmonrcv.o : rclmonrcv.cpp 
+	$(CXX) $(ALL_CXXFLAGS) -c -o rclmonrcv.o $<
+rclmonprc.o : rclmonprc.cpp 
+	$(CXX) $(ALL_CXXFLAGS) -c -o rclmonprc.o $<
 
 CSGUESS_OBJS= trcsguess.o  $(BIGLIB) 
 csguess : $(CSGUESS_OBJS)

src/index/indexer.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: indexer.cpp,v 1.37 2006-10-12 14:46:02 dockes Exp $ (C) 2004 J.F.Dockes";
+static char rcsid[] = "@(#$Id: indexer.cpp,v 1.38 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes";
 #endif
 /*
  *   This program is free software; you can redistribute it and/or modify
@@ -26,6 +26,7 @@ static char rcsid[] = "@(#$Id: indexer.cpp,v 1.37 2006-10-12 14:46:02 dockes Exp
 #include <unistd.h>
 #include <errno.h>
 #include <strings.h>
+#include <fnmatch.h>
 
 #include <iostream>
 #include <list>
@@ -207,16 +208,17 @@ bool DbIndexer::createAspellDict()
 }
 
 /** 
- Index individual files, out of a full tree run. No database purging
+ * Index individual files, out of a full tree run. No database purging
-*/
+ */
 bool DbIndexer::indexFiles(const list<string> &filenames)
 {
     if (!init())
 	return false;
 
     list<string>::const_iterator it;
-    for (it = filenames.begin(); it != filenames.end();it++) {
+    for (it = filenames.begin(); it != filenames.end(); it++) {
-	m_config->setKeyDir(path_getfather(*it));
+	string dir = path_getfather(*it);
+	m_config->setKeyDir(dir);
 	int abslen;
 	if (m_config->getConfParam("idxabsmlen", &abslen))
 	    m_db.setAbstractParams(abslen, -1, -1);
@@ -231,12 +233,37 @@ bool DbIndexer::indexFiles(const list<string> &filenames)
 		    it->c_str()));
 	    continue;
 	}
+
+	static string lstdir;
+	static list<string> skpl;
+	if (lstdir.compare(dir)) {
+	    LOGDEB(("Recomputing list of skipped names\n"));
+	    string skipped; 
+	    if (m_config->getConfParam("skippedNames", skipped)) {
+		stringToStrings(skipped, skpl);
+		lstdir = dir;
+	    }
+	}
+	if (!skpl.empty()) {
+	    list<string>::const_iterator skit;
+	    string fn = path_getsimple(*it);
+	    for (skit = skpl.begin(); skit != skpl.end(); skit++) {
+		if (fnmatch(skit->c_str(), fn.c_str(), 0) == 0) {
+		    LOGDEB(("Skipping [%s] :matches skip list\n", fn.c_str()));
+		    goto skipped;
+		}
+	    }
+	}
+
 	if (processone(*it, &stb, FsTreeWalker::FtwRegular) != 
 	    FsTreeWalker::FtwOk) {
 	    LOGERR(("DbIndexer::indexFiles: Database error\n"));
 	    return false;
 	}
+    skipped: 
+	false; // Need a statement here to make compiler happy ??
     }
+
     // The close would be done in our destructor, but we want status here
     if (!m_db.close()) {
 	LOGERR(("DbIndexer::indexfiles: error closing database in %s\n", 
@@ -371,27 +398,9 @@ ConfIndexer::~ConfIndexer()
      deleteZ(m_dbindexer);
 }
 
-list<string> topdirsToList(RclConfig *conf)
-{
-    list<string> tdl;
-    // Retrieve the list of directories to be indexed.
-    string topdirs;
-    if (!conf->getConfParam("topdirs", topdirs)) {
-	LOGERR(("ConfIndexer::index: no top directories in configuration\n"));
-	return tdl;
-    }
-    if (!stringToStrings(topdirs, tdl)) {
-	LOGERR(("ConfIndexer::index: parse error for directory list\n"));
-    }
-    for (list<string>::iterator it = tdl.begin(); it != tdl.end(); it++) {
-	*it = path_tildexpand(*it);
-    }
-    return tdl;
-}
-
 bool ConfIndexer::index(bool resetbefore)
 {
-    list<string> tdl = topdirsToList(m_config);
+    list<string> tdl = m_config->getTopdirs();
     if (tdl.empty()) {
 	m_reason = "Top directory list (topdirs param.) not found in config"
 	    "or Directory list parse error";

src/index/indexer.h

@@ -16,7 +16,7 @@
  */
 #ifndef _INDEXER_H_INCLUDED_
 #define _INDEXER_H_INCLUDED_
-/* @(#$Id: indexer.h,v 1.18 2006-10-12 14:46:02 dockes Exp $  (C) 2004 J.F.Dockes */
+/* @(#$Id: indexer.h,v 1.19 2006-10-16 15:33:08 dockes Exp $  (C) 2004 J.F.Dockes */
 
 #include <string>
 #include <list>
@@ -127,18 +127,22 @@ class DbIndexer : public FsTreeWalkerCB {
 	processone(const string &, const struct stat *, 
 		   FsTreeWalker::CbFlag);
 
+    /** Return my db dir */
+    string getDbDir() {return m_dbdir;}
+
  private:
     FsTreeWalker m_walker;
-    RclConfig *m_config;
+    RclConfig   *m_config;
-    string m_dbdir;
+    string       m_dbdir;
-    Rcl::Db m_db;
+    Rcl::Db      m_db;
-    string m_tmpdir;
+    string       m_tmpdir;
     DbIxStatusUpdater *m_updater;
 
     bool init(bool rst = false);
 };
 
-/** utility function to turn topdirs into a proper list */
+/** Helper method in recollindex.cpp for initial checks/setup to index 
-list<string> topdirsToList(RclConfig *conf);
+ * a list of files (either from the monitor or the command line) */
+extern bool indexfiles(RclConfig *config, const list<string> &filenames);
 
 #endif /* _INDEXER_H_INCLUDED_ */

src/index/rclmon.h

@@ -0,0 +1,65 @@
+#ifndef _RCLMON_H_INCLUDED_
+#define _RCLMON_H_INCLUDED_
+/* @(#$Id: rclmon.h,v 1.1 2006-10-16 15:33:08 dockes Exp $  (C) 2006 J.F.Dockes */
+/**
+ * Definitions for the real-time monitoring recoll. 
+ * We're interested in file modifications, deletions and renaming.
+ * We use two threads, one to receive events from the source, the
+ * other to perform adequate processing.
+ *
+ * The two threads communicate through an event buffer which is
+ * actually a hash map indexed by file path for easy coalescing of
+ * multiple events to the same file.
+ */
+
+#include <string>
+#include <map>
+
+#include "rclconfig.h"
+
+#ifndef NO_NAMESPACES
+using std::string;
+using std::multimap;
+#endif
+
+class RclMonEvent {
+ public: 
+    enum EvType {RCLEVT_NONE, RCLEVT_MODIFY, RCLEVT_DELETE, RCLEVT_RENAME};
+    string m_path;
+    string m_opath;
+    EvType m_etyp;
+    RclMonEvent() : m_etyp(RCLEVT_NONE) {}
+};
+
+class RclEQData;
+
+class RclMonEventQueue {
+ public:
+    RclMonEventQueue();
+    ~RclMonEventQueue();
+    /** Unlock queue and wait until there are new events. 
+     *  Returns with the queue locked */
+    bool wait();
+    /** Unlock queue */
+    bool unlock();
+    /** Lock queue. */
+    bool lock();
+    /** Lock queue and add event. */
+    bool pushEvent(const RclMonEvent &ev);
+    void setTerminate(); /* To all threads: end processing */
+    bool ok();
+    bool empty();
+    RclMonEvent pop();
+
+    // Convenience function for initially communicating config to mon thr
+    void setConfig(RclConfig *conf);
+    RclConfig *getConfig();
+
+ private:
+    RclEQData *m_data;
+};
+
+extern RclMonEventQueue rclEQ;
+extern bool startMonitor(RclConfig *conf, bool nofork);
+
+#endif /* _RCLMON_H_INCLUDED_ */

src/index/rclmonprc.cpp

@@ -0,0 +1,197 @@
+#ifndef lint
+static char rcsid[] = "@(#$Id: rclmonprc.cpp,v 1.1 2006-10-16 15:33:08 dockes Exp $ (C) 2006 J.F.Dockes";
+#endif
+/*
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+
+/**
+ * Recoll real time monitor processing. This file has the code to retrieve
+ * event from the event queue and do the database-side processing, and the 
+ * initialization function.
+ */
+
+#include <pthread.h>
+
+#include "debuglog.h"
+#include "rclmon.h"
+#include "debuglog.h"
+#include "indexer.h"
+
+typedef map<string, RclMonEvent> queue_type;
+
+class RclEQData {
+public:
+    queue_type m_queue;
+    RclConfig *m_config;
+    bool       m_ok;
+    pthread_mutex_t m_mutex;
+    pthread_cond_t m_cond;
+    RclEQData() 
+	: m_config(0), m_ok(false)
+    {
+	if (!pthread_mutex_init(&m_mutex, 0) && !pthread_cond_init(&m_cond, 0))
+	    m_ok = true;
+    }
+};
+
+RclMonEventQueue rclEQ;
+
+RclMonEventQueue::RclMonEventQueue()
+{
+    m_data = new RclEQData;
+}
+RclMonEventQueue::~RclMonEventQueue()
+{
+    delete m_data;
+}
+bool RclMonEventQueue::empty()
+{
+    return m_data == 0 ? true : m_data->m_queue.empty();
+}
+
+RclMonEvent RclMonEventQueue::pop()
+{
+    RclMonEvent ev;
+    if (!empty()) {
+	ev = m_data->m_queue.begin()->second;
+	m_data->m_queue.erase(m_data->m_queue.begin());
+    }
+    return ev;
+}
+
+/** Wait until there is something to process on the queue.
+ *  Must be called with the queue locked 
+ */
+bool RclMonEventQueue::wait() 
+{
+    if (!empty())
+	return true;
+    if (pthread_cond_wait(&m_data->m_cond, &m_data->m_mutex)) {
+	LOGERR(("RclMonEventQueue::wait: pthread_cond_wait failed\n"));
+	return false;
+    }
+    return true;
+}
+
+bool RclMonEventQueue::lock()
+{
+    if (pthread_mutex_lock(&m_data->m_mutex)) {
+	LOGERR(("RclMonEventQueue::lock: pthread_mutex_lock failed\n"));
+	return false;
+    }
+    return true;
+}
+bool RclMonEventQueue::unlock()
+{
+    if (pthread_mutex_unlock(&m_data->m_mutex)) {
+	LOGERR(("RclMonEventQueue::lock: pthread_mutex_unlock failed\n"));
+	return false;
+    }
+    return true;
+}
+
+void RclMonEventQueue::setConfig(RclConfig *cnf)
+{
+    m_data->m_config = cnf;
+}
+
+RclConfig *RclMonEventQueue::getConfig()
+{
+    return m_data->m_config;
+}
+
+bool RclMonEventQueue::ok()
+{
+    if (m_data == 0)
+	return false;
+    return m_data->m_ok;
+}
+
+void RclMonEventQueue::setTerminate()
+{
+    lock();
+    m_data->m_ok = false;
+    pthread_cond_broadcast(&m_data->m_cond);
+    unlock();
+}
+
+bool RclMonEventQueue::pushEvent(const RclMonEvent &ev)
+{
+    LOGDEB2(("RclMonEventQueue::pushEvent for %s\n", ev.m_path.c_str()));
+    lock();
+    // It seems that a newer event always override any older. TBVerified ?
+    m_data->m_queue[ev.m_path] = ev;
+    pthread_cond_broadcast(&m_data->m_cond);
+    unlock();
+    return true;
+}
+
+
+pthread_t rcv_thrid;
+void *rcv_result;
+extern void *rclMonRcvRun(void *);
+extern int stopindexing;
+
+bool startMonitor(RclConfig *conf, bool nofork)
+{
+    rclEQ.setConfig(conf);
+    if (pthread_create(&rcv_thrid, 0, &rclMonRcvRun, &rclEQ) != 0) {
+	LOGERR(("start_monitoring: cant create event-receiving thread\n"));
+	return false;
+    }
+
+    if (!rclEQ.lock()) {
+	return false;
+    }
+    LOGDEB(("start_monitoring: entering main loop\n"));
+    while (rclEQ.wait()) {
+	LOGDEB2(("startMonitor: wait returned\n"));
+	if (stopindexing || !rclEQ.ok())
+	    break;
+	list<string> modified;
+	list<string> deleted;
+
+	// Process event queue
+	while (!rclEQ.empty()) {
+	    // Retrieve event
+	    RclMonEvent ev = rclEQ.pop();
+	    switch (ev.m_etyp) {
+	    case RclMonEvent::RCLEVT_MODIFY:
+		LOGDEB(("Monitor: Modify/Check on %s\n", ev.m_path.c_str()));
+		modified.push_back(ev.m_path);
+		break;
+	    case RclMonEvent::RCLEVT_DELETE:
+		LOGDEB(("Monitor: Delete on %s\n", ev.m_path.c_str()));
+		deleted.push_back(ev.m_path);
+		break;
+	    case RclMonEvent::RCLEVT_RENAME:
+		LOGDEB(("Monitor: Rename on %s\n", ev.m_path.c_str()));
+		break;
+	    default:
+		LOGDEB(("Monitor: got Other on %s\n", ev.m_path.c_str()));
+	    }
+	}
+	// Unlock queue before processing lists
+	rclEQ.unlock();
+	// Process
+	indexfiles(conf, modified);
+	// Lock queue before waiting again
+	rclEQ.lock();
+    }
+    LOGERR(("start_monitoring: rclEQ::wait() failed\n"));
+    return false;
+}

src/index/rclmonrcv.cpp

@@ -0,0 +1,284 @@
+#ifndef lint
+static char rcsid[] = "@(#$Id: rclmonrcv.cpp,v 1.1 2006-10-16 15:33:08 dockes Exp $ (C) 2006 J.F.Dockes";
+#endif
+/*
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+#include <unistd.h>
+#include <sys/stat.h>
+#include <errno.h>
+
+#include "debuglog.h"
+#include "rclmon.h"
+#include "fstreewalk.h"
+#include "indexer.h"
+#include "pathut.h"
+
+/**
+ * Recoll real time monitor event receiver. This file has code to interface 
+ * to FAM and place events on the event queue.
+ */
+
+
+/** A small virtual interface for monitors. Suitable to let either of
+    fam/gamin/ or raw imonitor hide behind */
+class RclMonitor {
+public:
+    RclMonitor(){}
+    virtual ~RclMonitor() {}
+    virtual bool addWatch(const string& path, const struct stat&) = 0;
+    virtual bool getEvent(RclMonEvent& ev) = 0;
+    virtual bool ok() = 0;
+};
+// Monitor factory
+static RclMonitor *makeMonitor();
+
+/** Class used to create the directory watches */
+class WalkCB : public FsTreeWalkerCB {
+public:
+    WalkCB(RclConfig *conf, RclMonitor *mon)
+	: m_conf(conf), m_mon(mon) 
+    {}
+    virtual ~WalkCB() 
+    {}
+
+    virtual FsTreeWalker::Status 
+    processone(const string &fn, const struct stat *st, 
+	       FsTreeWalker::CbFlag flg)
+    {
+	LOGDEB2(("rclMonRcvRun: processone %s m_mon %p m_mon->ok %d\n", 
+		 fn.c_str(), m_mon, m_mon?m_mon->ok():0));
+	if (flg == FsTreeWalker::FtwDirEnter) {
+	    if (!m_mon || !m_mon->ok() || !m_mon->addWatch(fn, *st))
+		return FsTreeWalker::FtwError;
+	}
+	return FsTreeWalker::FtwOk;
+    }
+private:
+    RclConfig  *m_conf;
+    RclMonitor *m_mon;
+};
+
+/** Main thread routine: create watches, then wait for events an queue them */
+void *rclMonRcvRun(void *q)
+{
+    RclMonEventQueue *queue = (RclMonEventQueue *)q;
+    RclMonitor *mon;
+
+    LOGDEB(("rclMonRcvRun: running\n"));
+
+    if ((mon = makeMonitor()) == 0) {
+	LOGERR(("rclMonRcvRun: makeMonitor failed\n"));
+	rclEQ.setTerminate();
+	return 0;
+    }
+
+    // Get top directories from config and walk trees to add watches
+    FsTreeWalker walker;
+    WalkCB walkcb(queue->getConfig(), mon);
+    list<string> tdl = queue->getConfig()->getTopdirs();
+    if (tdl.empty()) {
+	LOGERR(("rclMonRcvRun:: top directory list (topdirs param.) not"
+		"found in config or Directory list parse error"));
+	rclEQ.setTerminate();
+	return 0;
+    }
+    for (list<string>::iterator it = tdl.begin(); it != tdl.end(); it++) {
+	queue->getConfig()->setKeyDir(*it);
+	walker.clearSkippedNames();
+	string skipped; 
+	if (queue->getConfig()->getConfParam("skippedNames", skipped)) {
+	    list<string> skpl;
+	    stringToStrings(skipped, skpl);
+	    walker.setSkippedNames(skpl);
+	}
+	LOGDEB(("rclMonRcvRun: walking %s\n", it->c_str()));
+	walker.walk(*it, walkcb);
+    }
+
+    // Forever wait for monitoring events and add them to queue:
+    LOGDEB2(("rclMonRcvRun: waiting for events. rclEQ.ok() %d\n", rclEQ.ok()));
+    while (rclEQ.ok()) {
+	if (!mon->ok())
+	    break;
+	RclMonEvent ev;
+	if (mon->getEvent(ev)) {
+	    rclEQ.pushEvent(ev);
+	}
+	if (!mon->ok())
+	    break;
+    }
+    LOGDEB(("rclMonRcvRun: exiting\n"));
+    rclEQ.setTerminate();
+    return 0;
+}
+
+//////////////////////////////////////////////////////////////////////////
+/** Fam/gamin -based monitor class */
+#include <fam.h>
+#include <sys/select.h>
+
+static const char *event_name(int code)
+{
+    static const char *famevent[] = {
+        "",
+        "FAMChanged",
+        "FAMDeleted",
+        "FAMStartExecuting",
+        "FAMStopExecuting",
+        "FAMCreated",
+        "FAMMoved",
+        "FAMAcknowledge",
+        "FAMExists",
+        "FAMEndExist"
+    };
+    static char unknown_event[20];
+ 
+    if (code < FAMChanged || code > FAMEndExist)
+    {
+        sprintf(unknown_event, "unknown (%d)", code);
+        return unknown_event;
+    }
+    return famevent[code];
+}
+
+// FAM based monitor class
+class RclFAM : public RclMonitor {
+public:
+    RclFAM();
+    virtual ~RclFAM();
+    virtual bool addWatch(const string& path, const struct stat& st);
+    virtual bool getEvent(RclMonEvent& ev);
+    bool ok() {return m_ok;}
+
+private:
+    bool m_ok;
+    FAMConnection m_conn;
+    void close() {
+	FAMClose(&m_conn);
+	m_ok = false;
+    }
+    map<int,string> m_reqtodir;
+};
+
+RclFAM::RclFAM()
+    : m_ok(false)
+{
+    if (FAMOpen2(&m_conn, "Recoll")) {
+	LOGERR(("RclFAM::RclFAM: FAMOpen2 failed, errno %d\n", errno));
+	return;
+    }
+    m_ok = true;
+}
+
+RclFAM::~RclFAM()
+{
+    if (ok())
+	FAMClose(&m_conn);
+}
+
+bool RclFAM::addWatch(const string& path, const struct stat& st)
+{
+    if (!ok())
+	return false;
+    LOGDEB(("RclFAM::addWatch: adding %s\n", path.c_str()));
+    FAMRequest req;
+    if (S_ISDIR(st.st_mode)) {
+	if (FAMMonitorDirectory(&m_conn, path.c_str(), &req, 0) != 0) {
+	    LOGERR(("RclFAM::addWatch: FAMMonitorDirectory failed\n"));
+	    return false;
+	}
+	m_reqtodir[req.reqnum] = path;
+    } else if (S_ISREG(st.st_mode)) {
+	if (FAMMonitorFile(&m_conn, path.c_str(), &req, 0) != 0) {
+	    LOGERR(("RclFAM::addWatch: FAMMonitorFile failed\n"));
+	    return false;
+	}
+    }
+    return true;
+}
+
+bool RclFAM::getEvent(RclMonEvent& ev)
+{
+    if (!ok())
+	return false;
+    LOGDEB2(("RclFAM::getEvent:\n"));
+
+    fd_set readfds;
+    int fam_fd = FAMCONNECTION_GETFD(&m_conn);
+    FD_ZERO(&readfds);
+    FD_SET(fam_fd, &readfds);
+
+    // Note: can't see a reason to set a timeout. Only reason we might
+    // want out is signal which will break the select call anyway (I
+    // don't think that there is any system still using the old bsd-type
+    // syscall re-entrance after signal).
+    LOGDEB(("RclFAM::getEvent: select\n"));
+    if (select(fam_fd + 1, &readfds, 0, 0, 0) < 0) {
+	LOGERR(("RclFAM::getEvent: select failed, errno %d\n", errno));
+	close();
+	return false;
+    }
+    if (!FD_ISSET(fam_fd, &readfds))
+	return false;
+
+    FAMEvent fe;
+    if (FAMNextEvent(&m_conn, &fe) < 0) {
+	LOGERR(("RclFAM::getEvent: FAMNextEvent failed, errno %d\n", errno));
+	close();
+	return false;
+    }
+    map<int,string>::const_iterator it;
+    if ((it = m_reqtodir.find(fe.fr.reqnum)) != m_reqtodir.end()) {
+	ev.m_path = path_cat(it->second, fe.filename);
+    } else {
+	ev.m_path = fe.filename;
+    }
+    LOGDEB(("RclFAM::getEvent: %-12s %s\n", 
+	    event_name(fe.code), ev.m_path.c_str()));
+
+    switch (fe.code) {
+    case FAMChanged:
+    case FAMCreated:
+    case FAMExists:
+	// Let the other side sort out the status of this file vs the db
+	ev.m_etyp = RclMonEvent::RCLEVT_MODIFY;
+	break;
+
+    case FAMDeleted:
+	ev.m_etyp = RclMonEvent::RCLEVT_DELETE;
+	break;
+
+    case FAMMoved: /* Never generated it seems */
+	LOGDEB(("RclFAM::getEvent: got move event !\n"));
+	ev.m_etyp = RclMonEvent::RCLEVT_MODIFY;
+	break;
+
+    case FAMStartExecuting:
+    case FAMStopExecuting:
+    case FAMAcknowledge:
+    case FAMEndExist:
+    default:
+	return false;
+    }
+    return true;
+}
+
+// The monitor factory
+static RclMonitor *makeMonitor()
+{
+    return new RclFAM;
+}

src/index/recollindex.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.22 2006-10-12 14:46:02 dockes Exp $ (C) 2004 J.F.Dockes";
+static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.23 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes";
 #endif
 /*
  *   This program is free software; you can redistribute it and/or modify
@@ -36,7 +36,7 @@ using namespace std;
 #include "indexer.h"
 #include "smallut.h"
 #include "pathut.h"
-
+#include "rclmon.h"
 
 // Globals for exit cleanup
 ConfIndexer *confindexer;
@@ -44,34 +44,42 @@ DbIndexer *dbindexer;
 
 static bool makeDbIndexer(RclConfig *config)
 {
-    if (dbindexer) {
-	delete dbindexer;
-	dbindexer = 0;
-    }
     string dbdir = config->getDbDir();
     if (dbdir.empty()) {
 	fprintf(stderr, "makeDbIndexer: no database directory in "
 		"configuration for %s\n", config->getKeyDir().c_str());
 	return false;
     }
-    dbindexer = new DbIndexer(config, dbdir);
+    // Check if there is already an indexer for the right db
+    if (dbindexer && dbindexer->getDbDir().compare(dbdir)) {
+	delete dbindexer;
+	dbindexer = 0;
+    }
+
+    if (!dbindexer)
+	dbindexer = new DbIndexer(config, dbdir);
+
     return true;
 }
 
-// Index a list of files 
+// The list of top directories/files wont change during program run,
-static bool indexfiles(RclConfig *config, const list<string> &filenames)
+// let's cache it:
+static list<string> o_tdl;
+
+// Index a list of files. We just check that they belong to one of the topdirs
+// subtrees, and call the indexer method
+bool indexfiles(RclConfig *config, const list<string> &filenames)
 {
     if (filenames.empty())
 	return true;
     
-    list<string> tdl = topdirsToList(config);
+    if (o_tdl.empty()) {
-    if (tdl.empty()) {
+	o_tdl = config->getTopdirs();
-	fprintf(stderr, "Top directory list (topdirs param.) not found in"
+	if (o_tdl.empty()) {
-		"config or Directory list parse error");
+	    fprintf(stderr, "Top directory list (topdirs param.) "
-	return false;
+		    "not found in config or Directory list parse error");
-    }
+	    return false;
-    for (list<string>::iterator dit= tdl.begin(); dit!= tdl.end(); dit++) {
+	}
-	*dit = path_canon(*dit);
     }
 
     list<string> myfiles;
@@ -79,7 +87,9 @@ static bool indexfiles(RclConfig *config, const list<string> &filenames)
 	 it != filenames.end(); it++) {
 	string fn = path_canon(*it);
 	bool ok = false;
-	for (list<string>::iterator dit= tdl.begin(); dit!= tdl.end(); dit++) {
+	// Check that this file name belongs to one of our subtrees
+	for (list<string>::iterator dit = o_tdl.begin(); 
+	     dit != o_tdl.end(); dit++) {
 	    if (fn.find(*dit) == 0) {
 		myfiles.push_back(fn);
 		ok = true;
@@ -153,6 +163,8 @@ static int     op_flags;
 #define OPT_s     0x10
 #define OPT_c     0x20
 #define OPT_S     0x40
+#define OPT_m     0x80
+#define OPT_D     0x100
 
 static const char usage [] =
 "\n"
@@ -161,6 +173,8 @@ static const char usage [] =
 "recollindex [-z] \n"
 "    Index everything according to configuration file\n"
 "    -z : reset database before starting indexation\n"
+"recollindex -m [-D]\n"
+"    Perform real time indexation. Don't become a daemon if -D is set\n"
 "recollindex -i <filename [filename ...]>\n"
 "    Index individual files. No database purge or stem database updates\n"
 "recollindex -s <lang>\n"
@@ -181,7 +195,6 @@ Usage(void)
     exit((op_flags & OPT_h)==0);
 }
 
-
 int main(int argc, const char **argv)
 {
     string a_config;
@@ -197,8 +210,10 @@ int main(int argc, const char **argv)
 	    case 'c':	op_flags |= OPT_c; if (argc < 2)  Usage();
 		a_config = *(++argv);
 		argc--; goto b1;
+	    case 'D': op_flags |= OPT_D; break;
 	    case 'h': op_flags |= OPT_h; break;
 	    case 'i': op_flags |= OPT_i; break;
+	    case 'm': op_flags |= OPT_m; break;
 	    case 's': op_flags |= OPT_s; break;
 #ifdef RCL_USE_ASPELL
 	    case 'S': op_flags |= OPT_S; break;
@@ -241,6 +256,16 @@ int main(int argc, const char **argv)
 	    Usage();
 	string lang = *argv++; argc--;
 	exit(!createstemdb(config, lang));
+    } else if (op_flags & OPT_m) {
+	if (argc != 0) 
+	    Usage();
+	if (!(op_flags&OPT_D)) {
+	    LOGDEB(("Daemonizing\n"));
+	    daemon(0,0);
+	}
+	if (startMonitor(config, (op_flags&OPT_D)!=0))
+	    exit(0);
+	exit(1);
 #ifdef RCL_USE_ASPELL
     } else if (op_flags & OPT_S) {
 	makeDbIndexer(config);