From 01d96314c5176c5b829bb9dcf9bbf9decbd0eddb Mon Sep 17 00:00:00 2001 From: dockes Date: Mon, 16 Oct 2006 15:33:08 +0000 Subject: [PATCH] 1st version of real time monitor --- src/common/rclconfig.cpp | 22 ++- src/common/rclconfig.h | 30 ++-- src/index/Makefile | 10 +- src/index/indexer.cpp | 57 ++++---- src/index/indexer.h | 18 ++- src/index/rclmon.h | 65 +++++++++ src/index/rclmonprc.cpp | 197 ++++++++++++++++++++++++++ src/index/rclmonrcv.cpp | 284 ++++++++++++++++++++++++++++++++++++++ src/index/recollindex.cpp | 65 ++++++--- 9 files changed, 683 insertions(+), 65 deletions(-) create mode 100644 src/index/rclmon.h create mode 100644 src/index/rclmonprc.cpp create mode 100644 src/index/rclmonrcv.cpp diff --git a/src/common/rclconfig.cpp b/src/common/rclconfig.cpp index 90602227..a443baec 100644 --- a/src/common/rclconfig.cpp +++ b/src/common/rclconfig.cpp @@ -1,5 +1,5 @@ #ifndef lint -static char rcsid[] = "@(#$Id: rclconfig.cpp,v 1.30 2006-09-08 09:02:47 dockes Exp $ (C) 2004 J.F.Dockes"; +static char rcsid[] = "@(#$Id: rclconfig.cpp,v 1.31 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes"; #endif /* * This program is free software; you can redistribute it and/or modify @@ -150,6 +150,26 @@ bool RclConfig::getConfParam(const std::string &name, bool *bvp) return true; } +list RclConfig::getTopdirs() +{ + list tdl; + // Retrieve the list of directories to be indexed. + string topdirs; + if (!getConfParam("topdirs", topdirs)) { + LOGERR(("RclConfig::getTopdirs: no top directories in config\n")); + return tdl; + } + if (!stringToStrings(topdirs, tdl)) { + LOGERR(("RclConfig::getTopdirs: parse error for directory list\n")); + return tdl; + } + for (list::iterator it = tdl.begin(); it != tdl.end(); it++) { + *it = path_tildexpand(*it); + *it = path_canon(*it); + } + return tdl; +} + // Get charset to be used for transcoding to utf-8 if unspecified by doc // For document contents: // If defcharset was set (from the config or a previous call), use it. diff --git a/src/common/rclconfig.h b/src/common/rclconfig.h index 8f2ec0aa..8460d488 100644 --- a/src/common/rclconfig.h +++ b/src/common/rclconfig.h @@ -16,9 +16,14 @@ */ #ifndef _RCLCONFIG_H_INCLUDED_ #define _RCLCONFIG_H_INCLUDED_ -/* @(#$Id: rclconfig.h,v 1.22 2006-10-11 14:16:25 dockes Exp $ (C) 2004 J.F.Dockes */ +/* @(#$Id: rclconfig.h,v 1.23 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes */ #include +#include +#ifndef NO_NAMESPACES +using std::list; +using std::string; +#endif #include "conftree.h" #include "smallut.h" @@ -61,6 +66,11 @@ class RclConfig { /** Get guessCharset for current keydir (was set during setKeydir) */ bool getGuessCharset() {return guesscharset;} + /** Get list of top directories. This is needed from a number of places + * and needs some cleaning-up code. An empty list is always an error, no + * need for other status */ + list getTopdirs(); + /** Get database directory */ string getDbDir(); @@ -70,7 +80,7 @@ class RclConfig { * The list is initialized on first call, and not changed for subsequent * setKeydirs. */ - bool getStopSuffixes(std::list& sufflist); + bool getStopSuffixes(list& sufflist); /** * Check in mimeconf if input mime type is a compressed one, and @@ -79,26 +89,26 @@ class RclConfig { * The returned command has substitutable places for input file name * and temp dir name, and will return output name */ - bool getUncompressor(const std::string &mtpe, std::list& cmd); + bool getUncompressor(const string &mtpe, list& cmd); /** Use mimemap to compute mimetype */ - std::string getMimeTypeFromSuffix(const std::string &suffix); + string getMimeTypeFromSuffix(const string &suffix); /** Get input filter from mimeconf for mimetype */ - std::string getMimeHandlerDef(const std::string &mimetype); + string getMimeHandlerDef(const string &mimetype); /** Get external viewer exec string from mimeconf for mimetype */ - std::string getMimeViewerDef(const std::string &mimetype); + string getMimeViewerDef(const string &mimetype); /** Get icon name from mimeconf for mimetype */ string getMimeIconName(const string &mtype, string *path = 0); /** Get a list of all indexable mime types defined in mimemap */ - std::list getAllMimeTypes(); + list getAllMimeTypes(); /** Find exec file for external filter. cmd is the command name from the * command string returned by getMimeHandlerDef */ - std::string findFilter(const std::string& cmd); + string findFilter(const string& cmd); ~RclConfig() { freeAll(); @@ -114,7 +124,7 @@ class RclConfig { } return *this; } - std::list getConfNames(const string &sk) { + list getConfNames(const string &sk) { return m_conf->getNames(sk); } @@ -129,7 +139,7 @@ class RclConfig { ConfStack *mimemap; // The files don't change with keydir, but their ConfStack *mimeconf; // content may depend on it. - std::list *stopsuffixes; + list *stopsuffixes; // Parameters auto-fetched on setkeydir string defcharset; // These are stored locally to avoid diff --git a/src/index/Makefile b/src/index/Makefile index e46d182c..eb62ce8b 100644 --- a/src/index/Makefile +++ b/src/index/Makefile @@ -2,16 +2,20 @@ depth = .. include $(depth)/mk/sysconf PROGS = recollindex csguess mimetype -SRCS = recollindex.cpp +SRCS = recollindex.cpp rclmonrcv.cpp rclmonprc.cpp all: depend $(PROGS) $(BIGLIB) -RECOLLINDEX_OBJS= recollindex.o $(BIGLIB) $(MIMELIB) +RECOLLINDEX_OBJS= recollindex.o rclmonrcv.o rclmonprc.o $(BIGLIB) $(MIMELIB) recollindex : $(RECOLLINDEX_OBJS) $(CXX) $(ALL_CXXFLAGS) -o recollindex $(RECOLLINDEX_OBJS) \ - $(BSTATIC) $(LIBXAPIAN) $(LIBICONV) $(BDYNAMIC) $(LIBSYS) + $(BSTATIC) $(LIBXAPIAN) $(LIBICONV) $(BDYNAMIC) -lfam $(LIBSYS) recollindex.o : recollindex.cpp $(CXX) $(ALL_CXXFLAGS) -c -o recollindex.o $< +rclmonrcv.o : rclmonrcv.cpp + $(CXX) $(ALL_CXXFLAGS) -c -o rclmonrcv.o $< +rclmonprc.o : rclmonprc.cpp + $(CXX) $(ALL_CXXFLAGS) -c -o rclmonprc.o $< CSGUESS_OBJS= trcsguess.o $(BIGLIB) csguess : $(CSGUESS_OBJS) diff --git a/src/index/indexer.cpp b/src/index/indexer.cpp index 27b8a903..2d7e6b60 100644 --- a/src/index/indexer.cpp +++ b/src/index/indexer.cpp @@ -1,5 +1,5 @@ #ifndef lint -static char rcsid[] = "@(#$Id: indexer.cpp,v 1.37 2006-10-12 14:46:02 dockes Exp $ (C) 2004 J.F.Dockes"; +static char rcsid[] = "@(#$Id: indexer.cpp,v 1.38 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes"; #endif /* * This program is free software; you can redistribute it and/or modify @@ -26,6 +26,7 @@ static char rcsid[] = "@(#$Id: indexer.cpp,v 1.37 2006-10-12 14:46:02 dockes Exp #include #include #include +#include #include #include @@ -207,16 +208,17 @@ bool DbIndexer::createAspellDict() } /** - Index individual files, out of a full tree run. No database purging -*/ + * Index individual files, out of a full tree run. No database purging + */ bool DbIndexer::indexFiles(const list &filenames) { if (!init()) return false; list::const_iterator it; - for (it = filenames.begin(); it != filenames.end();it++) { - m_config->setKeyDir(path_getfather(*it)); + for (it = filenames.begin(); it != filenames.end(); it++) { + string dir = path_getfather(*it); + m_config->setKeyDir(dir); int abslen; if (m_config->getConfParam("idxabsmlen", &abslen)) m_db.setAbstractParams(abslen, -1, -1); @@ -231,12 +233,37 @@ bool DbIndexer::indexFiles(const list &filenames) it->c_str())); continue; } + + static string lstdir; + static list skpl; + if (lstdir.compare(dir)) { + LOGDEB(("Recomputing list of skipped names\n")); + string skipped; + if (m_config->getConfParam("skippedNames", skipped)) { + stringToStrings(skipped, skpl); + lstdir = dir; + } + } + if (!skpl.empty()) { + list::const_iterator skit; + string fn = path_getsimple(*it); + for (skit = skpl.begin(); skit != skpl.end(); skit++) { + if (fnmatch(skit->c_str(), fn.c_str(), 0) == 0) { + LOGDEB(("Skipping [%s] :matches skip list\n", fn.c_str())); + goto skipped; + } + } + } + if (processone(*it, &stb, FsTreeWalker::FtwRegular) != FsTreeWalker::FtwOk) { LOGERR(("DbIndexer::indexFiles: Database error\n")); return false; } + skipped: + false; // Need a statement here to make compiler happy ?? } + // The close would be done in our destructor, but we want status here if (!m_db.close()) { LOGERR(("DbIndexer::indexfiles: error closing database in %s\n", @@ -371,27 +398,9 @@ ConfIndexer::~ConfIndexer() deleteZ(m_dbindexer); } -list topdirsToList(RclConfig *conf) -{ - list tdl; - // Retrieve the list of directories to be indexed. - string topdirs; - if (!conf->getConfParam("topdirs", topdirs)) { - LOGERR(("ConfIndexer::index: no top directories in configuration\n")); - return tdl; - } - if (!stringToStrings(topdirs, tdl)) { - LOGERR(("ConfIndexer::index: parse error for directory list\n")); - } - for (list::iterator it = tdl.begin(); it != tdl.end(); it++) { - *it = path_tildexpand(*it); - } - return tdl; -} - bool ConfIndexer::index(bool resetbefore) { - list tdl = topdirsToList(m_config); + list tdl = m_config->getTopdirs(); if (tdl.empty()) { m_reason = "Top directory list (topdirs param.) not found in config" "or Directory list parse error"; diff --git a/src/index/indexer.h b/src/index/indexer.h index ccbc737a..5004d1e0 100644 --- a/src/index/indexer.h +++ b/src/index/indexer.h @@ -16,7 +16,7 @@ */ #ifndef _INDEXER_H_INCLUDED_ #define _INDEXER_H_INCLUDED_ -/* @(#$Id: indexer.h,v 1.18 2006-10-12 14:46:02 dockes Exp $ (C) 2004 J.F.Dockes */ +/* @(#$Id: indexer.h,v 1.19 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes */ #include #include @@ -127,18 +127,22 @@ class DbIndexer : public FsTreeWalkerCB { processone(const string &, const struct stat *, FsTreeWalker::CbFlag); + /** Return my db dir */ + string getDbDir() {return m_dbdir;} + private: FsTreeWalker m_walker; - RclConfig *m_config; - string m_dbdir; - Rcl::Db m_db; - string m_tmpdir; + RclConfig *m_config; + string m_dbdir; + Rcl::Db m_db; + string m_tmpdir; DbIxStatusUpdater *m_updater; bool init(bool rst = false); }; -/** utility function to turn topdirs into a proper list */ -list topdirsToList(RclConfig *conf); +/** Helper method in recollindex.cpp for initial checks/setup to index + * a list of files (either from the monitor or the command line) */ +extern bool indexfiles(RclConfig *config, const list &filenames); #endif /* _INDEXER_H_INCLUDED_ */ diff --git a/src/index/rclmon.h b/src/index/rclmon.h new file mode 100644 index 00000000..06e33e0d --- /dev/null +++ b/src/index/rclmon.h @@ -0,0 +1,65 @@ +#ifndef _RCLMON_H_INCLUDED_ +#define _RCLMON_H_INCLUDED_ +/* @(#$Id: rclmon.h,v 1.1 2006-10-16 15:33:08 dockes Exp $ (C) 2006 J.F.Dockes */ +/** + * Definitions for the real-time monitoring recoll. + * We're interested in file modifications, deletions and renaming. + * We use two threads, one to receive events from the source, the + * other to perform adequate processing. + * + * The two threads communicate through an event buffer which is + * actually a hash map indexed by file path for easy coalescing of + * multiple events to the same file. + */ + +#include +#include + +#include "rclconfig.h" + +#ifndef NO_NAMESPACES +using std::string; +using std::multimap; +#endif + +class RclMonEvent { + public: + enum EvType {RCLEVT_NONE, RCLEVT_MODIFY, RCLEVT_DELETE, RCLEVT_RENAME}; + string m_path; + string m_opath; + EvType m_etyp; + RclMonEvent() : m_etyp(RCLEVT_NONE) {} +}; + +class RclEQData; + +class RclMonEventQueue { + public: + RclMonEventQueue(); + ~RclMonEventQueue(); + /** Unlock queue and wait until there are new events. + * Returns with the queue locked */ + bool wait(); + /** Unlock queue */ + bool unlock(); + /** Lock queue. */ + bool lock(); + /** Lock queue and add event. */ + bool pushEvent(const RclMonEvent &ev); + void setTerminate(); /* To all threads: end processing */ + bool ok(); + bool empty(); + RclMonEvent pop(); + + // Convenience function for initially communicating config to mon thr + void setConfig(RclConfig *conf); + RclConfig *getConfig(); + + private: + RclEQData *m_data; +}; + +extern RclMonEventQueue rclEQ; +extern bool startMonitor(RclConfig *conf, bool nofork); + +#endif /* _RCLMON_H_INCLUDED_ */ diff --git a/src/index/rclmonprc.cpp b/src/index/rclmonprc.cpp new file mode 100644 index 00000000..0993b145 --- /dev/null +++ b/src/index/rclmonprc.cpp @@ -0,0 +1,197 @@ +#ifndef lint +static char rcsid[] = "@(#$Id: rclmonprc.cpp,v 1.1 2006-10-16 15:33:08 dockes Exp $ (C) 2006 J.F.Dockes"; +#endif +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +/** + * Recoll real time monitor processing. This file has the code to retrieve + * event from the event queue and do the database-side processing, and the + * initialization function. + */ + +#include + +#include "debuglog.h" +#include "rclmon.h" +#include "debuglog.h" +#include "indexer.h" + +typedef map queue_type; + +class RclEQData { +public: + queue_type m_queue; + RclConfig *m_config; + bool m_ok; + pthread_mutex_t m_mutex; + pthread_cond_t m_cond; + RclEQData() + : m_config(0), m_ok(false) + { + if (!pthread_mutex_init(&m_mutex, 0) && !pthread_cond_init(&m_cond, 0)) + m_ok = true; + } +}; + +RclMonEventQueue rclEQ; + +RclMonEventQueue::RclMonEventQueue() +{ + m_data = new RclEQData; +} +RclMonEventQueue::~RclMonEventQueue() +{ + delete m_data; +} +bool RclMonEventQueue::empty() +{ + return m_data == 0 ? true : m_data->m_queue.empty(); +} + +RclMonEvent RclMonEventQueue::pop() +{ + RclMonEvent ev; + if (!empty()) { + ev = m_data->m_queue.begin()->second; + m_data->m_queue.erase(m_data->m_queue.begin()); + } + return ev; +} + +/** Wait until there is something to process on the queue. + * Must be called with the queue locked + */ +bool RclMonEventQueue::wait() +{ + if (!empty()) + return true; + if (pthread_cond_wait(&m_data->m_cond, &m_data->m_mutex)) { + LOGERR(("RclMonEventQueue::wait: pthread_cond_wait failed\n")); + return false; + } + return true; +} + +bool RclMonEventQueue::lock() +{ + if (pthread_mutex_lock(&m_data->m_mutex)) { + LOGERR(("RclMonEventQueue::lock: pthread_mutex_lock failed\n")); + return false; + } + return true; +} +bool RclMonEventQueue::unlock() +{ + if (pthread_mutex_unlock(&m_data->m_mutex)) { + LOGERR(("RclMonEventQueue::lock: pthread_mutex_unlock failed\n")); + return false; + } + return true; +} + +void RclMonEventQueue::setConfig(RclConfig *cnf) +{ + m_data->m_config = cnf; +} + +RclConfig *RclMonEventQueue::getConfig() +{ + return m_data->m_config; +} + +bool RclMonEventQueue::ok() +{ + if (m_data == 0) + return false; + return m_data->m_ok; +} + +void RclMonEventQueue::setTerminate() +{ + lock(); + m_data->m_ok = false; + pthread_cond_broadcast(&m_data->m_cond); + unlock(); +} + +bool RclMonEventQueue::pushEvent(const RclMonEvent &ev) +{ + LOGDEB2(("RclMonEventQueue::pushEvent for %s\n", ev.m_path.c_str())); + lock(); + // It seems that a newer event always override any older. TBVerified ? + m_data->m_queue[ev.m_path] = ev; + pthread_cond_broadcast(&m_data->m_cond); + unlock(); + return true; +} + + +pthread_t rcv_thrid; +void *rcv_result; +extern void *rclMonRcvRun(void *); +extern int stopindexing; + +bool startMonitor(RclConfig *conf, bool nofork) +{ + rclEQ.setConfig(conf); + if (pthread_create(&rcv_thrid, 0, &rclMonRcvRun, &rclEQ) != 0) { + LOGERR(("start_monitoring: cant create event-receiving thread\n")); + return false; + } + + if (!rclEQ.lock()) { + return false; + } + LOGDEB(("start_monitoring: entering main loop\n")); + while (rclEQ.wait()) { + LOGDEB2(("startMonitor: wait returned\n")); + if (stopindexing || !rclEQ.ok()) + break; + list modified; + list deleted; + + // Process event queue + while (!rclEQ.empty()) { + // Retrieve event + RclMonEvent ev = rclEQ.pop(); + switch (ev.m_etyp) { + case RclMonEvent::RCLEVT_MODIFY: + LOGDEB(("Monitor: Modify/Check on %s\n", ev.m_path.c_str())); + modified.push_back(ev.m_path); + break; + case RclMonEvent::RCLEVT_DELETE: + LOGDEB(("Monitor: Delete on %s\n", ev.m_path.c_str())); + deleted.push_back(ev.m_path); + break; + case RclMonEvent::RCLEVT_RENAME: + LOGDEB(("Monitor: Rename on %s\n", ev.m_path.c_str())); + break; + default: + LOGDEB(("Monitor: got Other on %s\n", ev.m_path.c_str())); + } + } + // Unlock queue before processing lists + rclEQ.unlock(); + // Process + indexfiles(conf, modified); + // Lock queue before waiting again + rclEQ.lock(); + } + LOGERR(("start_monitoring: rclEQ::wait() failed\n")); + return false; +} diff --git a/src/index/rclmonrcv.cpp b/src/index/rclmonrcv.cpp new file mode 100644 index 00000000..601db83c --- /dev/null +++ b/src/index/rclmonrcv.cpp @@ -0,0 +1,284 @@ +#ifndef lint +static char rcsid[] = "@(#$Id: rclmonrcv.cpp,v 1.1 2006-10-16 15:33:08 dockes Exp $ (C) 2006 J.F.Dockes"; +#endif +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#include +#include +#include + +#include "debuglog.h" +#include "rclmon.h" +#include "fstreewalk.h" +#include "indexer.h" +#include "pathut.h" + +/** + * Recoll real time monitor event receiver. This file has code to interface + * to FAM and place events on the event queue. + */ + + +/** A small virtual interface for monitors. Suitable to let either of + fam/gamin/ or raw imonitor hide behind */ +class RclMonitor { +public: + RclMonitor(){} + virtual ~RclMonitor() {} + virtual bool addWatch(const string& path, const struct stat&) = 0; + virtual bool getEvent(RclMonEvent& ev) = 0; + virtual bool ok() = 0; +}; +// Monitor factory +static RclMonitor *makeMonitor(); + +/** Class used to create the directory watches */ +class WalkCB : public FsTreeWalkerCB { +public: + WalkCB(RclConfig *conf, RclMonitor *mon) + : m_conf(conf), m_mon(mon) + {} + virtual ~WalkCB() + {} + + virtual FsTreeWalker::Status + processone(const string &fn, const struct stat *st, + FsTreeWalker::CbFlag flg) + { + LOGDEB2(("rclMonRcvRun: processone %s m_mon %p m_mon->ok %d\n", + fn.c_str(), m_mon, m_mon?m_mon->ok():0)); + if (flg == FsTreeWalker::FtwDirEnter) { + if (!m_mon || !m_mon->ok() || !m_mon->addWatch(fn, *st)) + return FsTreeWalker::FtwError; + } + return FsTreeWalker::FtwOk; + } +private: + RclConfig *m_conf; + RclMonitor *m_mon; +}; + +/** Main thread routine: create watches, then wait for events an queue them */ +void *rclMonRcvRun(void *q) +{ + RclMonEventQueue *queue = (RclMonEventQueue *)q; + RclMonitor *mon; + + LOGDEB(("rclMonRcvRun: running\n")); + + if ((mon = makeMonitor()) == 0) { + LOGERR(("rclMonRcvRun: makeMonitor failed\n")); + rclEQ.setTerminate(); + return 0; + } + + // Get top directories from config and walk trees to add watches + FsTreeWalker walker; + WalkCB walkcb(queue->getConfig(), mon); + list tdl = queue->getConfig()->getTopdirs(); + if (tdl.empty()) { + LOGERR(("rclMonRcvRun:: top directory list (topdirs param.) not" + "found in config or Directory list parse error")); + rclEQ.setTerminate(); + return 0; + } + for (list::iterator it = tdl.begin(); it != tdl.end(); it++) { + queue->getConfig()->setKeyDir(*it); + walker.clearSkippedNames(); + string skipped; + if (queue->getConfig()->getConfParam("skippedNames", skipped)) { + list skpl; + stringToStrings(skipped, skpl); + walker.setSkippedNames(skpl); + } + LOGDEB(("rclMonRcvRun: walking %s\n", it->c_str())); + walker.walk(*it, walkcb); + } + + // Forever wait for monitoring events and add them to queue: + LOGDEB2(("rclMonRcvRun: waiting for events. rclEQ.ok() %d\n", rclEQ.ok())); + while (rclEQ.ok()) { + if (!mon->ok()) + break; + RclMonEvent ev; + if (mon->getEvent(ev)) { + rclEQ.pushEvent(ev); + } + if (!mon->ok()) + break; + } + LOGDEB(("rclMonRcvRun: exiting\n")); + rclEQ.setTerminate(); + return 0; +} + +////////////////////////////////////////////////////////////////////////// +/** Fam/gamin -based monitor class */ +#include +#include + +static const char *event_name(int code) +{ + static const char *famevent[] = { + "", + "FAMChanged", + "FAMDeleted", + "FAMStartExecuting", + "FAMStopExecuting", + "FAMCreated", + "FAMMoved", + "FAMAcknowledge", + "FAMExists", + "FAMEndExist" + }; + static char unknown_event[20]; + + if (code < FAMChanged || code > FAMEndExist) + { + sprintf(unknown_event, "unknown (%d)", code); + return unknown_event; + } + return famevent[code]; +} + +// FAM based monitor class +class RclFAM : public RclMonitor { +public: + RclFAM(); + virtual ~RclFAM(); + virtual bool addWatch(const string& path, const struct stat& st); + virtual bool getEvent(RclMonEvent& ev); + bool ok() {return m_ok;} + +private: + bool m_ok; + FAMConnection m_conn; + void close() { + FAMClose(&m_conn); + m_ok = false; + } + map m_reqtodir; +}; + +RclFAM::RclFAM() + : m_ok(false) +{ + if (FAMOpen2(&m_conn, "Recoll")) { + LOGERR(("RclFAM::RclFAM: FAMOpen2 failed, errno %d\n", errno)); + return; + } + m_ok = true; +} + +RclFAM::~RclFAM() +{ + if (ok()) + FAMClose(&m_conn); +} + +bool RclFAM::addWatch(const string& path, const struct stat& st) +{ + if (!ok()) + return false; + LOGDEB(("RclFAM::addWatch: adding %s\n", path.c_str())); + FAMRequest req; + if (S_ISDIR(st.st_mode)) { + if (FAMMonitorDirectory(&m_conn, path.c_str(), &req, 0) != 0) { + LOGERR(("RclFAM::addWatch: FAMMonitorDirectory failed\n")); + return false; + } + m_reqtodir[req.reqnum] = path; + } else if (S_ISREG(st.st_mode)) { + if (FAMMonitorFile(&m_conn, path.c_str(), &req, 0) != 0) { + LOGERR(("RclFAM::addWatch: FAMMonitorFile failed\n")); + return false; + } + } + return true; +} + +bool RclFAM::getEvent(RclMonEvent& ev) +{ + if (!ok()) + return false; + LOGDEB2(("RclFAM::getEvent:\n")); + + fd_set readfds; + int fam_fd = FAMCONNECTION_GETFD(&m_conn); + FD_ZERO(&readfds); + FD_SET(fam_fd, &readfds); + + // Note: can't see a reason to set a timeout. Only reason we might + // want out is signal which will break the select call anyway (I + // don't think that there is any system still using the old bsd-type + // syscall re-entrance after signal). + LOGDEB(("RclFAM::getEvent: select\n")); + if (select(fam_fd + 1, &readfds, 0, 0, 0) < 0) { + LOGERR(("RclFAM::getEvent: select failed, errno %d\n", errno)); + close(); + return false; + } + if (!FD_ISSET(fam_fd, &readfds)) + return false; + + FAMEvent fe; + if (FAMNextEvent(&m_conn, &fe) < 0) { + LOGERR(("RclFAM::getEvent: FAMNextEvent failed, errno %d\n", errno)); + close(); + return false; + } + map::const_iterator it; + if ((it = m_reqtodir.find(fe.fr.reqnum)) != m_reqtodir.end()) { + ev.m_path = path_cat(it->second, fe.filename); + } else { + ev.m_path = fe.filename; + } + LOGDEB(("RclFAM::getEvent: %-12s %s\n", + event_name(fe.code), ev.m_path.c_str())); + + switch (fe.code) { + case FAMChanged: + case FAMCreated: + case FAMExists: + // Let the other side sort out the status of this file vs the db + ev.m_etyp = RclMonEvent::RCLEVT_MODIFY; + break; + + case FAMDeleted: + ev.m_etyp = RclMonEvent::RCLEVT_DELETE; + break; + + case FAMMoved: /* Never generated it seems */ + LOGDEB(("RclFAM::getEvent: got move event !\n")); + ev.m_etyp = RclMonEvent::RCLEVT_MODIFY; + break; + + case FAMStartExecuting: + case FAMStopExecuting: + case FAMAcknowledge: + case FAMEndExist: + default: + return false; + } + return true; +} + +// The monitor factory +static RclMonitor *makeMonitor() +{ + return new RclFAM; +} diff --git a/src/index/recollindex.cpp b/src/index/recollindex.cpp index 07fc11c2..b61f5fdb 100644 --- a/src/index/recollindex.cpp +++ b/src/index/recollindex.cpp @@ -1,5 +1,5 @@ #ifndef lint -static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.22 2006-10-12 14:46:02 dockes Exp $ (C) 2004 J.F.Dockes"; +static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.23 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes"; #endif /* * This program is free software; you can redistribute it and/or modify @@ -36,7 +36,7 @@ using namespace std; #include "indexer.h" #include "smallut.h" #include "pathut.h" - +#include "rclmon.h" // Globals for exit cleanup ConfIndexer *confindexer; @@ -44,34 +44,42 @@ DbIndexer *dbindexer; static bool makeDbIndexer(RclConfig *config) { - if (dbindexer) { - delete dbindexer; - dbindexer = 0; - } string dbdir = config->getDbDir(); if (dbdir.empty()) { fprintf(stderr, "makeDbIndexer: no database directory in " "configuration for %s\n", config->getKeyDir().c_str()); return false; } - dbindexer = new DbIndexer(config, dbdir); + // Check if there is already an indexer for the right db + if (dbindexer && dbindexer->getDbDir().compare(dbdir)) { + delete dbindexer; + dbindexer = 0; + } + + if (!dbindexer) + dbindexer = new DbIndexer(config, dbdir); + return true; } -// Index a list of files -static bool indexfiles(RclConfig *config, const list &filenames) +// The list of top directories/files wont change during program run, +// let's cache it: +static list o_tdl; + +// Index a list of files. We just check that they belong to one of the topdirs +// subtrees, and call the indexer method +bool indexfiles(RclConfig *config, const list &filenames) { if (filenames.empty()) return true; - - list tdl = topdirsToList(config); - if (tdl.empty()) { - fprintf(stderr, "Top directory list (topdirs param.) not found in" - "config or Directory list parse error"); - return false; - } - for (list::iterator dit= tdl.begin(); dit!= tdl.end(); dit++) { - *dit = path_canon(*dit); + + if (o_tdl.empty()) { + o_tdl = config->getTopdirs(); + if (o_tdl.empty()) { + fprintf(stderr, "Top directory list (topdirs param.) " + "not found in config or Directory list parse error"); + return false; + } } list myfiles; @@ -79,7 +87,9 @@ static bool indexfiles(RclConfig *config, const list &filenames) it != filenames.end(); it++) { string fn = path_canon(*it); bool ok = false; - for (list::iterator dit= tdl.begin(); dit!= tdl.end(); dit++) { + // Check that this file name belongs to one of our subtrees + for (list::iterator dit = o_tdl.begin(); + dit != o_tdl.end(); dit++) { if (fn.find(*dit) == 0) { myfiles.push_back(fn); ok = true; @@ -153,6 +163,8 @@ static int op_flags; #define OPT_s 0x10 #define OPT_c 0x20 #define OPT_S 0x40 +#define OPT_m 0x80 +#define OPT_D 0x100 static const char usage [] = "\n" @@ -161,6 +173,8 @@ static const char usage [] = "recollindex [-z] \n" " Index everything according to configuration file\n" " -z : reset database before starting indexation\n" +"recollindex -m [-D]\n" +" Perform real time indexation. Don't become a daemon if -D is set\n" "recollindex -i \n" " Index individual files. No database purge or stem database updates\n" "recollindex -s \n" @@ -181,7 +195,6 @@ Usage(void) exit((op_flags & OPT_h)==0); } - int main(int argc, const char **argv) { string a_config; @@ -197,8 +210,10 @@ int main(int argc, const char **argv) case 'c': op_flags |= OPT_c; if (argc < 2) Usage(); a_config = *(++argv); argc--; goto b1; + case 'D': op_flags |= OPT_D; break; case 'h': op_flags |= OPT_h; break; case 'i': op_flags |= OPT_i; break; + case 'm': op_flags |= OPT_m; break; case 's': op_flags |= OPT_s; break; #ifdef RCL_USE_ASPELL case 'S': op_flags |= OPT_S; break; @@ -241,6 +256,16 @@ int main(int argc, const char **argv) Usage(); string lang = *argv++; argc--; exit(!createstemdb(config, lang)); + } else if (op_flags & OPT_m) { + if (argc != 0) + Usage(); + if (!(op_flags&OPT_D)) { + LOGDEB(("Daemonizing\n")); + daemon(0,0); + } + if (startMonitor(config, (op_flags&OPT_D)!=0)) + exit(0); + exit(1); #ifdef RCL_USE_ASPELL } else if (op_flags & OPT_S) { makeDbIndexer(config);