diff --git a/src/index/indexer.cpp b/src/index/indexer.cpp index 608dcafa..ccf9eb94 100644 --- a/src/index/indexer.cpp +++ b/src/index/indexer.cpp @@ -1,5 +1,5 @@ #ifndef lint -static char rcsid[] = "@(#$Id: indexer.cpp,v 1.40 2006-10-24 09:09:36 dockes Exp $ (C) 2004 J.F.Dockes"; +static char rcsid[] = "@(#$Id: indexer.cpp,v 1.41 2006-10-24 14:28:38 dockes Exp $ (C) 2004 J.F.Dockes"; #endif /* * This program is free software; you can redistribute it and/or modify @@ -115,8 +115,27 @@ bool DbIndexer::indexDb(bool resetbefore, list *topdirs) // filesystem anymore. m_db.purge(); - // Create stemming databases. We also remove those which are not - // configured. + createStemmingDatabases(); + createAspellDict(); + + // The close would be done in our destructor, but we want status here + if (m_updater) { + m_updater->status.phase = DbIxStatus::DBIXS_CLOSING; + m_updater->status.fn.erase(); + m_updater->update(); + } + if (!m_db.close()) { + LOGERR(("DbIndexer::index: error closing database in %s\n", + m_dbdir.c_str())); + return false; + } + return true; +} + +// Create stemming databases. We also remove those which are not +// configured. +bool DbIndexer::createStemmingDatabases() +{ string slangs; if (m_config->getConfParam("indexstemminglanguages", slangs)) { list langs; @@ -139,20 +158,6 @@ bool DbIndexer::indexDb(bool resetbefore, list *topdirs) m_db.createStemDb(*it); } } - - createAspellDict(); - - // The close would be done in our destructor, but we want status here - if (m_updater) { - m_updater->status.phase = DbIxStatus::DBIXS_CLOSING; - m_updater->status.fn.erase(); - m_updater->update(); - } - if (!m_db.close()) { - LOGERR(("DbIndexer::index: error closing database in %s\n", - m_dbdir.c_str())); - return false; - } return true; } diff --git a/src/index/indexer.h b/src/index/indexer.h index 351a9a8e..9cda62fa 100644 --- a/src/index/indexer.h +++ b/src/index/indexer.h @@ -16,7 +16,7 @@ */ #ifndef _INDEXER_H_INCLUDED_ #define _INDEXER_H_INCLUDED_ -/* @(#$Id: indexer.h,v 1.20 2006-10-22 14:47:13 dockes Exp $ (C) 2004 J.F.Dockes */ +/* @(#$Id: indexer.h,v 1.21 2006-10-24 14:28:38 dockes Exp $ (C) 2004 J.F.Dockes */ #include #include @@ -119,6 +119,9 @@ class DbIndexer : public FsTreeWalkerCB { /** Purge a list of files. */ bool purgeFiles(const std::list &files); + /** Stemming reset to config: create needed, delete unconfigured */ + bool createStemmingDatabases(); + /** Create stem database for given language */ bool createStemDb(const string &lang); @@ -148,5 +151,6 @@ class DbIndexer : public FsTreeWalkerCB { * a list of files (either from the monitor or the command line) */ extern bool indexfiles(RclConfig *config, const list &filenames); extern bool purgefiles(RclConfig *config, const list &filenames); +extern bool createAuxDbs(RclConfig *config); #endif /* _INDEXER_H_INCLUDED_ */ diff --git a/src/index/rclmon.h b/src/index/rclmon.h index 05474af8..8072bc25 100644 --- a/src/index/rclmon.h +++ b/src/index/rclmon.h @@ -2,7 +2,7 @@ #define _RCLMON_H_INCLUDED_ #include "autoconfig.h" #ifdef RCL_MONITOR -/* @(#$Id: rclmon.h,v 1.4 2006-10-24 12:48:08 dockes Exp $ (C) 2006 J.F.Dockes */ +/* @(#$Id: rclmon.h,v 1.5 2006-10-24 14:28:38 dockes Exp $ (C) 2006 J.F.Dockes */ /** * Definitions for the real-time monitoring recoll. * We're interested in file modifications, deletions and renaming. @@ -49,7 +49,7 @@ class RclMonEventQueue { ~RclMonEventQueue(); /** Unlock queue and wait until there are new events. * Returns with the queue locked */ - bool wait(); + bool wait(int secs = -1, bool *timedout = 0); /** Unlock queue */ bool unlock(); /** Lock queue. */ diff --git a/src/index/rclmonprc.cpp b/src/index/rclmonprc.cpp index bb53ecd7..07247ed0 100644 --- a/src/index/rclmonprc.cpp +++ b/src/index/rclmonprc.cpp @@ -2,7 +2,7 @@ #ifdef RCL_MONITOR #ifndef lint -static char rcsid[] = "@(#$Id: rclmonprc.cpp,v 1.4 2006-10-24 12:48:09 dockes Exp $ (C) 2006 J.F.Dockes"; +static char rcsid[] = "@(#$Id: rclmonprc.cpp,v 1.5 2006-10-24 14:28:38 dockes Exp $ (C) 2006 J.F.Dockes"; #endif /* * This program is free software; you can redistribute it and/or modify @@ -89,13 +89,34 @@ RclMonEvent RclMonEventQueue::pop() /** Wait until there is something to process on the queue. * Must be called with the queue locked */ -bool RclMonEventQueue::wait() +bool RclMonEventQueue::wait(int seconds, bool *top) { if (!empty()) return true; - if (pthread_cond_wait(&m_data->m_cond, &m_data->m_mutex)) { - LOGERR(("RclMonEventQueue::wait: pthread_cond_wait failed\n")); - return false; + + int err; + if (seconds > 0) { + struct timespec to; + to.tv_sec = time(0L) + seconds; + to.tv_nsec = 0; + if (top) + *top = false; + if ((err = + pthread_cond_timedwait(&m_data->m_cond, &m_data->m_mutex, &to))) { + if (err == ETIMEDOUT) { + *top = true; + return true; + } + LOGERR(("RclMonEventQueue::wait:pthread_cond_timedwait failed" + "with err %d\n", err)); + return false; + } + } else { + if ((err = pthread_cond_wait(&m_data->m_cond, &m_data->m_mutex))) { + LOGERR(("RclMonEventQueue::wait: pthread_cond_wait failed" + "with err %d\n", err)); + return false; + } } return true; } @@ -240,7 +261,13 @@ bool startMonitor(RclConfig *conf, bool nofork) return false; } LOGDEB(("start_monitoring: entering main loop\n")); - while (rclEQ.wait()) { + bool timedout; + bool didsomething = false; + + // We set a timeout of 10mn. If we do timeout, and there have been some + // indexing activity since the last such operation, we'll update the + // auxiliary data (stemming and spelling) + while (rclEQ.wait(10 * 60, &timedout)) { LOGDEB2(("startMonitor: wait returned\n")); if (!rclEQ.ok()) break; @@ -270,10 +297,29 @@ bool startMonitor(RclConfig *conf, bool nofork) // Unlock queue before processing lists rclEQ.unlock(); // Process - if (!indexfiles(conf, modified)) - break; - if (!purgefiles(conf, deleted)) - break; + if (!modified.empty()) { + if (!indexfiles(conf, modified)) + break; + didsomething = true; + } + if (!deleted.empty()) { + if (!purgefiles(conf, deleted)) + break; + didsomething = true; + } + + if (timedout) { + LOGDEB2(("Monitor: queue wait timed out\n")); + // Timed out. there must not be much activity around here. + // If anything was modified, process the end-of-indexing + // tasks: stemming and spelling database creations. + if (didsomething) { + didsomething = false; + if (!createAuxDbs(conf)) + break; + } + } + // Lock queue before waiting again rclEQ.lock(); } diff --git a/src/index/recollindex.cpp b/src/index/recollindex.cpp index 571cba17..23f4f94b 100644 --- a/src/index/recollindex.cpp +++ b/src/index/recollindex.cpp @@ -1,5 +1,5 @@ #ifndef lint -static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.25 2006-10-22 14:47:14 dockes Exp $ (C) 2004 J.F.Dockes"; +static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.26 2006-10-24 14:28:38 dockes Exp $ (C) 2004 J.F.Dockes"; #endif /* * This program is free software; you can redistribute it and/or modify @@ -171,6 +171,21 @@ bool purgefiles(RclConfig *config, const list &filenames) return dbindexer->purgeFiles(myfiles); } +// Create stemming and spelling databases +bool createAuxDbs(RclConfig *config) +{ + if (!makeDbIndexer(config) || !dbindexer) + return false; + + if (!dbindexer->createStemmingDatabases()) + return false; + + if (!dbindexer->createAspellDict()) + return false; + + return true; +} + // Create additional stem database static bool createstemdb(RclConfig *config, const string &lang) {