create stemming db on queue timeout if needed
This commit is contained in:
dockes
parent
d01c087204
commit
f43b0ee348
@ -1,5 +1,5 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.40 2006-10-24 09:09:36 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.41 2006-10-24 14:28:38 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
@ -115,8 +115,27 @@ bool DbIndexer::indexDb(bool resetbefore, list<string> *topdirs)
|
||||
// filesystem anymore.
|
||||
m_db.purge();
|
||||
|
||||
// Create stemming databases. We also remove those which are not
|
||||
// configured.
|
||||
createStemmingDatabases();
|
||||
createAspellDict();
|
||||
|
||||
// The close would be done in our destructor, but we want status here
|
||||
if (m_updater) {
|
||||
m_updater->status.phase = DbIxStatus::DBIXS_CLOSING;
|
||||
m_updater->status.fn.erase();
|
||||
m_updater->update();
|
||||
}
|
||||
if (!m_db.close()) {
|
||||
LOGERR(("DbIndexer::index: error closing database in %s\n",
|
||||
m_dbdir.c_str()));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Create stemming databases. We also remove those which are not
|
||||
// configured.
|
||||
bool DbIndexer::createStemmingDatabases()
|
||||
{
|
||||
string slangs;
|
||||
if (m_config->getConfParam("indexstemminglanguages", slangs)) {
|
||||
list<string> langs;
|
||||
@ -139,20 +158,6 @@ bool DbIndexer::indexDb(bool resetbefore, list<string> *topdirs)
|
||||
m_db.createStemDb(*it);
|
||||
}
|
||||
}
|
||||
|
||||
createAspellDict();
|
||||
|
||||
// The close would be done in our destructor, but we want status here
|
||||
if (m_updater) {
|
||||
m_updater->status.phase = DbIxStatus::DBIXS_CLOSING;
|
||||
m_updater->status.fn.erase();
|
||||
m_updater->update();
|
||||
}
|
||||
if (!m_db.close()) {
|
||||
LOGERR(("DbIndexer::index: error closing database in %s\n",
|
||||
m_dbdir.c_str()));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
*/
|
||||
#ifndef _INDEXER_H_INCLUDED_
|
||||
#define _INDEXER_H_INCLUDED_
|
||||
/* @(#$Id: indexer.h,v 1.20 2006-10-22 14:47:13 dockes Exp $ (C) 2004 J.F.Dockes */
|
||||
/* @(#$Id: indexer.h,v 1.21 2006-10-24 14:28:38 dockes Exp $ (C) 2004 J.F.Dockes */
|
||||
|
||||
#include <string>
|
||||
#include <list>
|
||||
@ -119,6 +119,9 @@ class DbIndexer : public FsTreeWalkerCB {
|
||||
/** Purge a list of files. */
|
||||
bool purgeFiles(const std::list<string> &files);
|
||||
|
||||
/** Stemming reset to config: create needed, delete unconfigured */
|
||||
bool createStemmingDatabases();
|
||||
|
||||
/** Create stem database for given language */
|
||||
bool createStemDb(const string &lang);
|
||||
|
||||
@ -148,5 +151,6 @@ class DbIndexer : public FsTreeWalkerCB {
|
||||
* a list of files (either from the monitor or the command line) */
|
||||
extern bool indexfiles(RclConfig *config, const list<string> &filenames);
|
||||
extern bool purgefiles(RclConfig *config, const list<string> &filenames);
|
||||
extern bool createAuxDbs(RclConfig *config);
|
||||
|
||||
#endif /* _INDEXER_H_INCLUDED_ */
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
#define _RCLMON_H_INCLUDED_
|
||||
#include "autoconfig.h"
|
||||
#ifdef RCL_MONITOR
|
||||
/* @(#$Id: rclmon.h,v 1.4 2006-10-24 12:48:08 dockes Exp $ (C) 2006 J.F.Dockes */
|
||||
/* @(#$Id: rclmon.h,v 1.5 2006-10-24 14:28:38 dockes Exp $ (C) 2006 J.F.Dockes */
|
||||
/**
|
||||
* Definitions for the real-time monitoring recoll.
|
||||
* We're interested in file modifications, deletions and renaming.
|
||||
@ -49,7 +49,7 @@ class RclMonEventQueue {
|
||||
~RclMonEventQueue();
|
||||
/** Unlock queue and wait until there are new events.
|
||||
* Returns with the queue locked */
|
||||
bool wait();
|
||||
bool wait(int secs = -1, bool *timedout = 0);
|
||||
/** Unlock queue */
|
||||
bool unlock();
|
||||
/** Lock queue. */
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
#ifdef RCL_MONITOR
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: rclmonprc.cpp,v 1.4 2006-10-24 12:48:09 dockes Exp $ (C) 2006 J.F.Dockes";
|
||||
static char rcsid[] = "@(#$Id: rclmonprc.cpp,v 1.5 2006-10-24 14:28:38 dockes Exp $ (C) 2006 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
@ -89,13 +89,34 @@ RclMonEvent RclMonEventQueue::pop()
|
||||
/** Wait until there is something to process on the queue.
|
||||
* Must be called with the queue locked
|
||||
*/
|
||||
bool RclMonEventQueue::wait()
|
||||
bool RclMonEventQueue::wait(int seconds, bool *top)
|
||||
{
|
||||
if (!empty())
|
||||
return true;
|
||||
if (pthread_cond_wait(&m_data->m_cond, &m_data->m_mutex)) {
|
||||
LOGERR(("RclMonEventQueue::wait: pthread_cond_wait failed\n"));
|
||||
return false;
|
||||
|
||||
int err;
|
||||
if (seconds > 0) {
|
||||
struct timespec to;
|
||||
to.tv_sec = time(0L) + seconds;
|
||||
to.tv_nsec = 0;
|
||||
if (top)
|
||||
*top = false;
|
||||
if ((err =
|
||||
pthread_cond_timedwait(&m_data->m_cond, &m_data->m_mutex, &to))) {
|
||||
if (err == ETIMEDOUT) {
|
||||
*top = true;
|
||||
return true;
|
||||
}
|
||||
LOGERR(("RclMonEventQueue::wait:pthread_cond_timedwait failed"
|
||||
"with err %d\n", err));
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
if ((err = pthread_cond_wait(&m_data->m_cond, &m_data->m_mutex))) {
|
||||
LOGERR(("RclMonEventQueue::wait: pthread_cond_wait failed"
|
||||
"with err %d\n", err));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@ -240,7 +261,13 @@ bool startMonitor(RclConfig *conf, bool nofork)
|
||||
return false;
|
||||
}
|
||||
LOGDEB(("start_monitoring: entering main loop\n"));
|
||||
while (rclEQ.wait()) {
|
||||
bool timedout;
|
||||
bool didsomething = false;
|
||||
|
||||
// We set a timeout of 10mn. If we do timeout, and there have been some
|
||||
// indexing activity since the last such operation, we'll update the
|
||||
// auxiliary data (stemming and spelling)
|
||||
while (rclEQ.wait(10 * 60, &timedout)) {
|
||||
LOGDEB2(("startMonitor: wait returned\n"));
|
||||
if (!rclEQ.ok())
|
||||
break;
|
||||
@ -270,10 +297,29 @@ bool startMonitor(RclConfig *conf, bool nofork)
|
||||
// Unlock queue before processing lists
|
||||
rclEQ.unlock();
|
||||
// Process
|
||||
if (!indexfiles(conf, modified))
|
||||
break;
|
||||
if (!purgefiles(conf, deleted))
|
||||
break;
|
||||
if (!modified.empty()) {
|
||||
if (!indexfiles(conf, modified))
|
||||
break;
|
||||
didsomething = true;
|
||||
}
|
||||
if (!deleted.empty()) {
|
||||
if (!purgefiles(conf, deleted))
|
||||
break;
|
||||
didsomething = true;
|
||||
}
|
||||
|
||||
if (timedout) {
|
||||
LOGDEB2(("Monitor: queue wait timed out\n"));
|
||||
// Timed out. there must not be much activity around here.
|
||||
// If anything was modified, process the end-of-indexing
|
||||
// tasks: stemming and spelling database creations.
|
||||
if (didsomething) {
|
||||
didsomething = false;
|
||||
if (!createAuxDbs(conf))
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Lock queue before waiting again
|
||||
rclEQ.lock();
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.25 2006-10-22 14:47:14 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.26 2006-10-24 14:28:38 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
@ -171,6 +171,21 @@ bool purgefiles(RclConfig *config, const list<string> &filenames)
|
||||
return dbindexer->purgeFiles(myfiles);
|
||||
}
|
||||
|
||||
// Create stemming and spelling databases
|
||||
bool createAuxDbs(RclConfig *config)
|
||||
{
|
||||
if (!makeDbIndexer(config) || !dbindexer)
|
||||
return false;
|
||||
|
||||
if (!dbindexer->createStemmingDatabases())
|
||||
return false;
|
||||
|
||||
if (!dbindexer->createAspellDict())
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Create additional stem database
|
||||
static bool createstemdb(RclConfig *config, const string &lang)
|
||||
{
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user