tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

monitor: purge docs for deleted files from db

Browse Source
This commit is contained in:
dockes
2006-10-22 14:47:14 +00:00
parent 4269a149b2
commit 4e0d1e2483
11 changed files with 314 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/index/Makefile
+2 -1
View File
@@ -9,7 +9,8 @@ all: depend $(PROGS) $(BIGLIB)
RECOLLINDEX_OBJS= recollindex.o rclmonrcv.o rclmonprc.o $(BIGLIB) $(MIMELIB)
recollindex : $(RECOLLINDEX_OBJS)
$(CXX) $(ALL_CXXFLAGS) -o recollindex $(RECOLLINDEX_OBJS) \
$(BSTATIC) $(LIBXAPIAN) $(LIBICONV) $(BDYNAMIC) -lfam $(LIBSYS)
$(BSTATIC) $(LIBXAPIAN) $(LIBICONV) $(BDYNAMIC) \
$(LIBFAM) $(LIBSYS)
recollindex.o : recollindex.cpp
$(CXX) $(ALL_CXXFLAGS) -c -o recollindex.o $<
rclmonrcv.o : rclmonrcv.cpp
src/index/indexer.cpp
+31 -6
View File
@@ -1,5 +1,5 @@
#ifndef lint
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.38 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes";
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.39 2006-10-22 14:47:13 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
@@ -223,13 +223,13 @@ bool DbIndexer::indexFiles(const list<string> &filenames)
if (m_config->getConfParam("idxabsmlen", &abslen))
m_db.setAbstractParams(abslen, -1, -1);
struct stat stb;
if (stat(it->c_str(), &stb) != 0) {
LOGERR(("DbIndexer::indexFiles: stat(%s): %s", it->c_str(),
if (lstat(it->c_str(), &stb) != 0) {
LOGERR(("DbIndexer::indexFiles: lstat(%s): %s", it->c_str(),
strerror(errno)));
continue;
}
if (!S_ISREG(stb.st_mode)) {
LOGERR(("DbIndexer::indexFiles: %s: not a regular file\n",
LOGDEB2(("DbIndexer::indexFiles: %s: not a regular file\n",
it->c_str()));
continue;
}
@@ -257,7 +257,7 @@ bool DbIndexer::indexFiles(const list<string> &filenames)
if (processone(*it, &stb, FsTreeWalker::FtwRegular) !=
FsTreeWalker::FtwOk) {
LOGERR(("DbIndexer::indexFiles: Database error\n"));
LOGERR(("DbIndexer::indexFiles: processone failed\n"));
return false;
}
skipped:
@@ -273,6 +273,31 @@ bool DbIndexer::indexFiles(const list<string> &filenames)
return true;
}
/** Purge docs for given files out of the database */
bool DbIndexer::purgeFiles(const list<string> &filenames)
{
if (!init())
return false;
list<string>::const_iterator it;
for (it = filenames.begin(); it != filenames.end(); it++) {
if (!m_db.purgeFile(*it)) {
LOGERR(("DbIndexer::purgeFiles: Database error\n"));
return false;
}
}
// The close would be done in our destructor, but we want status here
if (!m_db.close()) {
LOGERR(("DbIndexer::purgefiles: error closing database in %s\n",
m_dbdir.c_str()));
return false;
}
return true;
}
/// This method gets called for every file and directory found by the
/// tree walker.
///
@@ -308,7 +333,7 @@ DbIndexer::processone(const std::string &fn, const struct stat *stp,
// without mime type will not be purged from the db, resulting
// in possible 'cannot intern file' messages at query time...
if (!m_db.needUpdate(fn, stp)) {
LOGDEB(("indexfile: up to date: %s\n", fn.c_str()));
LOGDEB(("processone: up to date: %s\n", fn.c_str()));
if (m_updater) {
m_updater->status.fn = fn;
if (!m_updater->update()) {
src/index/indexer.h
+6 -2
View File
@@ -16,7 +16,7 @@
*/
#ifndef _INDEXER_H_INCLUDED_
#define _INDEXER_H_INCLUDED_
/* @(#$Id: indexer.h,v 1.19 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes */
/* @(#$Id: indexer.h,v 1.20 2006-10-22 14:47:13 dockes Exp $ (C) 2004 J.F.Dockes */
#include <string>
#include <list>
@@ -116,6 +116,9 @@ class DbIndexer : public FsTreeWalkerCB {
/** Index a list of files. No db cleaning or stemdb updating */
bool indexFiles(const std::list<string> &files);
/** Purge a list of files. */
bool purgeFiles(const std::list<string> &files);
/** Create stem database for given language */
bool createStemDb(const string &lang);
@@ -141,8 +144,9 @@ class DbIndexer : public FsTreeWalkerCB {
bool init(bool rst = false);
};
/** Helper method in recollindex.cpp for initial checks/setup to index
/** Helper methods in recollindex.cpp for initial checks/setup to index
* a list of files (either from the monitor or the command line) */
extern bool indexfiles(RclConfig *config, const list<string> &filenames);
extern bool purgefiles(RclConfig *config, const list<string> &filenames);
#endif /* _INDEXER_H_INCLUDED_ */
src/index/rclmonprc.cpp
+10 -8
View File
@@ -2,7 +2,7 @@
#ifdef RCL_MONITOR
#ifndef lint
static char rcsid[] = "@(#$Id: rclmonprc.cpp,v 1.2 2006-10-17 14:41:59 dockes Exp $ (C) 2006 J.F.Dockes";
static char rcsid[] = "@(#$Id: rclmonprc.cpp,v 1.3 2006-10-22 14:47:13 dockes Exp $ (C) 2006 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
@@ -117,11 +117,13 @@ RclConfig *RclMonEventQueue::getConfig()
return m_data->m_config;
}
extern int stopindexing;
bool RclMonEventQueue::ok()
{
if (m_data == 0)
return false;
return m_data->m_ok;
return !stopindexing && m_data->m_ok;
}
void RclMonEventQueue::setTerminate()
@@ -143,11 +145,9 @@ bool RclMonEventQueue::pushEvent(const RclMonEvent &ev)
return true;
}
pthread_t rcv_thrid;
void *rcv_result;
extern void *rclMonRcvRun(void *);
extern int stopindexing;
bool startMonitor(RclConfig *conf, bool nofork)
{
@@ -163,7 +163,7 @@ bool startMonitor(RclConfig *conf, bool nofork)
LOGDEB(("start_monitoring: entering main loop\n"));
while (rclEQ.wait()) {
LOGDEB2(("startMonitor: wait returned\n"));
if (stopindexing || !rclEQ.ok())
if (!rclEQ.ok())
break;
list<string> modified;
list<string> deleted;
@@ -191,11 +191,13 @@ bool startMonitor(RclConfig *conf, bool nofork)
// Unlock queue before processing lists
rclEQ.unlock();
// Process
indexfiles(conf, modified);
if (!indexfiles(conf, modified))
break;
if (!purgefiles(conf, deleted))
break;
// Lock queue before waiting again
rclEQ.lock();
}
LOGERR(("start_monitoring: rclEQ::wait() failed\n"));
return false;
return true;
}
#endif // RCL_MONITOR
src/index/rclmonrcv.cpp
+67 -39
View File
@@ -1,7 +1,7 @@
#include "autoconfig.h"
#ifdef RCL_MONITOR
#ifndef lint
static char rcsid[] = "@(#$Id: rclmonrcv.cpp,v 1.2 2006-10-17 14:41:59 dockes Exp $ (C) 2006 J.F.Dockes";
static char rcsid[] = "@(#$Id: rclmonrcv.cpp,v 1.3 2006-10-22 14:47:13 dockes Exp $ (C) 2006 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
@@ -35,71 +35,92 @@ static char rcsid[] = "@(#$Id: rclmonrcv.cpp,v 1.2 2006-10-17 14:41:59 dockes Ex
*/
/** A small virtual interface for monitors. Suitable to let either of
fam/gamin/ or raw imonitor hide behind */
/** A small virtual interface for monitors. Probably suitable to let
either of fam/gamin or raw imonitor hide behind */
class RclMonitor {
public:
RclMonitor(){}
virtual ~RclMonitor() {}
virtual bool addWatch(const string& path, const struct stat&) = 0;
virtual bool getEvent(RclMonEvent& ev) = 0;
virtual bool getEvent(RclMonEvent& ev, int secs = -1) = 0;
virtual bool ok() = 0;
};
// Monitor factory
// Monitor factory. We only have one compiled-in kind at a time, no
// need for a 'kind' parameter
static RclMonitor *makeMonitor();
/** Class used to create the directory watches */
/** This class is a callback for the file system tree walker
class. The callback method alternatively creates the directory
watches and flushes the event queue (to avoid a possible overflow
while we create the watches)*/
class WalkCB : public FsTreeWalkerCB {
public:
WalkCB(RclConfig *conf, RclMonitor *mon)
: m_conf(conf), m_mon(mon)
WalkCB(RclConfig *conf, RclMonitor *mon, RclMonEventQueue *queue)
: m_conf(conf), m_mon(mon), m_queue(queue)
{}
virtual ~WalkCB()
{}
virtual FsTreeWalker::Status
processone(const string &fn, const struct stat *st,
FsTreeWalker::CbFlag flg)
processone(const string &fn, const struct stat *st, FsTreeWalker::CbFlag flg)
{
LOGDEB2(("rclMonRcvRun: processone %s m_mon %p m_mon->ok %d\n",
fn.c_str(), m_mon, m_mon?m_mon->ok():0));
// Create watch when entering directory
if (flg == FsTreeWalker::FtwDirEnter) {
// Empty whatever events we may already have on queue
while (m_queue->ok() && m_mon->ok()) {
RclMonEvent ev;
if (m_mon->getEvent(ev, 0)) {
m_queue->pushEvent(ev);
} else {
break;
}
}
if (!m_mon || !m_mon->ok() || !m_mon->addWatch(fn, *st))
return FsTreeWalker::FtwError;
}
return FsTreeWalker::FtwOk;
}
private:
RclConfig *m_conf;
RclMonitor *m_mon;
RclConfig *m_conf;
RclMonitor *m_mon;
RclMonEventQueue *m_queue;
};
/** Main thread routine: create watches, then wait for events an queue them */
/** Main thread routine: create watches, then forever wait for and queue events */
void *rclMonRcvRun(void *q)
{
RclMonEventQueue *queue = (RclMonEventQueue *)q;
RclMonitor *mon;
LOGDEB(("rclMonRcvRun: running\n"));
// Create the fam/whatever interface object
RclMonitor *mon;
if ((mon = makeMonitor()) == 0) {
LOGERR(("rclMonRcvRun: makeMonitor failed\n"));
rclEQ.setTerminate();
queue->setTerminate();
return 0;
}
// Get top directories from config and walk trees to add watches
FsTreeWalker walker;
WalkCB walkcb(queue->getConfig(), mon);
// Get top directories from config
list<string> tdl = queue->getConfig()->getTopdirs();
if (tdl.empty()) {
LOGERR(("rclMonRcvRun:: top directory list (topdirs param.) not"
"found in config or Directory list parse error"));
rclEQ.setTerminate();
queue->setTerminate();
return 0;
}
// Walk the directory trees to add watches
FsTreeWalker walker;
WalkCB walkcb(queue->getConfig(), mon, queue);
for (list<string>::iterator it = tdl.begin(); it != tdl.end(); it++) {
queue->getConfig()->setKeyDir(*it);
// Adjust the skipped names according to config
walker.clearSkippedNames();
string skipped;
if (queue->getConfig()->getConfParam("skippedNames", skipped)) {
@@ -112,19 +133,16 @@ void *rclMonRcvRun(void *q)
}
// Forever wait for monitoring events and add them to queue:
LOGDEB2(("rclMonRcvRun: waiting for events. rclEQ.ok() %d\n", rclEQ.ok()));
while (rclEQ.ok()) {
if (!mon->ok())
break;
LOGDEB2(("rclMonRcvRun: waiting for events. queue->ok() %d\n", queue->ok()));
while (queue->ok() && mon->ok()) {
RclMonEvent ev;
if (mon->getEvent(ev)) {
rclEQ.pushEvent(ev);
queue->pushEvent(ev);
}
if (!mon->ok())
break;
}
LOGDEB(("rclMonRcvRun: exiting\n"));
rclEQ.setTerminate();
queue->setTerminate();
return 0;
}
@@ -133,6 +151,7 @@ void *rclMonRcvRun(void *q)
#include <fam.h>
#include <sys/select.h>
// Translate event code to string (debug)
static const char *event_name(int code)
{
static const char *famevent[] = {
@@ -149,21 +168,22 @@ static const char *event_name(int code)
};
static char unknown_event[20];
if (code < FAMChanged || code > FAMEndExist)
{
if (code < FAMChanged || code > FAMEndExist) {
sprintf(unknown_event, "unknown (%d)", code);
return unknown_event;
}
return famevent[code];
}
// FAM based monitor class
/** FAM based monitor class. We have to keep a record of FAM watch
request numbers to directory names as the event only contain the
request number and file name, not the full path */
class RclFAM : public RclMonitor {
public:
RclFAM();
virtual ~RclFAM();
virtual bool addWatch(const string& path, const struct stat& st);
virtual bool getEvent(RclMonEvent& ev);
virtual bool getEvent(RclMonEvent& ev, int secs = -1);
bool ok() {return m_ok;}
private:
@@ -213,7 +233,7 @@ bool RclFAM::addWatch(const string& path, const struct stat& st)
return true;
}
bool RclFAM::getEvent(RclMonEvent& ev)
bool RclFAM::getEvent(RclMonEvent& ev, int secs)
{
if (!ok())
return false;
@@ -224,16 +244,22 @@ bool RclFAM::getEvent(RclMonEvent& ev)
FD_ZERO(&readfds);
FD_SET(fam_fd, &readfds);
// Note: can't see a reason to set a timeout. Only reason we might
// want out is signal which will break the select call anyway (I
// don't think that there is any system still using the old bsd-type
// syscall re-entrance after signal).
LOGDEB(("RclFAM::getEvent: select\n"));
if (select(fam_fd + 1, &readfds, 0, 0, 0) < 0) {
struct timeval timeout;
if (secs >= 0) {
memset(&timeout, 0, sizeof(timeout));
timeout.tv_sec = secs;
}
int ret;
if ((ret=select(fam_fd + 1, &readfds, 0, 0, secs >= 0 ? &timeout : 0)) < 0) {
LOGERR(("RclFAM::getEvent: select failed, errno %d\n", errno));
close();
return false;
} else if (ret == 0) {
// timeout
return false;
}
if (!FD_ISSET(fam_fd, &readfds))
return false;
@@ -243,8 +269,10 @@ bool RclFAM::getEvent(RclMonEvent& ev)
close();
return false;
}
map<int,string>::const_iterator it;
if ((it = m_reqtodir.find(fe.fr.reqnum)) != m_reqtodir.end()) {
if ((fe.filename[0] != '/') &&
(it = m_reqtodir.find(fe.fr.reqnum)) != m_reqtodir.end()) {
ev.m_path = path_cat(it->second, fe.filename);
} else {
ev.m_path = fe.filename;
@@ -279,7 +307,7 @@ bool RclFAM::getEvent(RclMonEvent& ev)
return true;
}
// The monitor factory
// The monitor 'factory'
static RclMonitor *makeMonitor()
{
return new RclFAM;
src/index/recollindex.cpp
+57 -22
View File
@@ -1,5 +1,5 @@
#ifndef lint
static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.24 2006-10-17 14:41:59 dockes Exp $ (C) 2004 J.F.Dockes";
static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.25 2006-10-22 14:47:14 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
@@ -42,6 +42,27 @@ using namespace std;
ConfIndexer *confindexer;
DbIndexer *dbindexer;
int stopindexing;
// Mainly used to request indexing stop, we currently do not use the
// current file name
class MyUpdater : public DbIxStatusUpdater {
public:
virtual bool update() {
if (stopindexing) {
return false;
}
return true;
}
};
MyUpdater updater;
static void sigcleanup(int sig)
{
fprintf(stderr, "sigcleanup\n");
LOGDEB(("sigcleanup\n"));
stopindexing = 1;
}
static bool makeDbIndexer(RclConfig *config)
{
string dbdir = config->getDbDir();
@@ -57,7 +78,7 @@ static bool makeDbIndexer(RclConfig *config)
}
if (!dbindexer)
dbindexer = new DbIndexer(config, dbdir);
dbindexer = new DbIndexer(config, dbdir, &updater);
return true;
}
@@ -116,6 +137,40 @@ bool indexfiles(RclConfig *config, const list<string> &filenames)
return dbindexer->indexFiles(myfiles);
}
// Delete a list of files.
bool purgefiles(RclConfig *config, const list<string> &filenames)
{
if (filenames.empty())
return true;
if (o_tdl.empty()) {
o_tdl = config->getTopdirs();
if (o_tdl.empty()) {
fprintf(stderr, "Top directory list (topdirs param.) "
"not found in config or Directory list parse error");
return false;
}
}
list<string> myfiles;
for (list<string>::const_iterator it = filenames.begin();
it != filenames.end(); it++) {
myfiles.push_back(path_canon(*it));
}
// Note: we should sort the file names against the topdirs here
// and check for different databases. But we can for now only have
// one database per config, so we set the keydir from the first
// file (which is not really needed...), create the indexer/db and
// go:
config->setKeyDir(path_getfather(*myfiles.begin()));
if (!makeDbIndexer(config) || !dbindexer)
return false;
else
return dbindexer->purgeFiles(myfiles);
}
// Create additional stem database
static bool createstemdb(RclConfig *config, const string &lang)
{
@@ -134,26 +189,6 @@ static void cleanup()
dbindexer = 0;
}
int stopindexing;
// Mainly used to request indexing stop, we currently do not use the
// current file name
class MyUpdater : public DbIxStatusUpdater {
public:
virtual bool update() {
if (stopindexing) {
return false;
}
return true;
}
};
MyUpdater updater;
static void sigcleanup(int sig)
{
fprintf(stderr, "sigcleanup\n");
stopindexing = 1;
}
static const char *thisprog;
static int op_flags;
#define OPT_MOINS 0x1