1st version of real time monitor
This commit is contained in:
dockes
parent
cca7060385
commit
01d96314c5
@ -1,5 +1,5 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: rclconfig.cpp,v 1.30 2006-09-08 09:02:47 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
static char rcsid[] = "@(#$Id: rclconfig.cpp,v 1.31 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
@ -150,6 +150,26 @@ bool RclConfig::getConfParam(const std::string &name, bool *bvp)
|
||||
return true;
|
||||
}
|
||||
|
||||
list<string> RclConfig::getTopdirs()
|
||||
{
|
||||
list<string> tdl;
|
||||
// Retrieve the list of directories to be indexed.
|
||||
string topdirs;
|
||||
if (!getConfParam("topdirs", topdirs)) {
|
||||
LOGERR(("RclConfig::getTopdirs: no top directories in config\n"));
|
||||
return tdl;
|
||||
}
|
||||
if (!stringToStrings(topdirs, tdl)) {
|
||||
LOGERR(("RclConfig::getTopdirs: parse error for directory list\n"));
|
||||
return tdl;
|
||||
}
|
||||
for (list<string>::iterator it = tdl.begin(); it != tdl.end(); it++) {
|
||||
*it = path_tildexpand(*it);
|
||||
*it = path_canon(*it);
|
||||
}
|
||||
return tdl;
|
||||
}
|
||||
|
||||
// Get charset to be used for transcoding to utf-8 if unspecified by doc
|
||||
// For document contents:
|
||||
// If defcharset was set (from the config or a previous call), use it.
|
||||
|
||||
@ -16,9 +16,14 @@
|
||||
*/
|
||||
#ifndef _RCLCONFIG_H_INCLUDED_
|
||||
#define _RCLCONFIG_H_INCLUDED_
|
||||
/* @(#$Id: rclconfig.h,v 1.22 2006-10-11 14:16:25 dockes Exp $ (C) 2004 J.F.Dockes */
|
||||
/* @(#$Id: rclconfig.h,v 1.23 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes */
|
||||
|
||||
#include <list>
|
||||
#include <string>
|
||||
#ifndef NO_NAMESPACES
|
||||
using std::list;
|
||||
using std::string;
|
||||
#endif
|
||||
|
||||
#include "conftree.h"
|
||||
#include "smallut.h"
|
||||
@ -61,6 +66,11 @@ class RclConfig {
|
||||
/** Get guessCharset for current keydir (was set during setKeydir) */
|
||||
bool getGuessCharset() {return guesscharset;}
|
||||
|
||||
/** Get list of top directories. This is needed from a number of places
|
||||
* and needs some cleaning-up code. An empty list is always an error, no
|
||||
* need for other status */
|
||||
list<string> getTopdirs();
|
||||
|
||||
/** Get database directory */
|
||||
string getDbDir();
|
||||
|
||||
@ -70,7 +80,7 @@ class RclConfig {
|
||||
* The list is initialized on first call, and not changed for subsequent
|
||||
* setKeydirs.
|
||||
*/
|
||||
bool getStopSuffixes(std::list<std::string>& sufflist);
|
||||
bool getStopSuffixes(list<string>& sufflist);
|
||||
|
||||
/**
|
||||
* Check in mimeconf if input mime type is a compressed one, and
|
||||
@ -79,26 +89,26 @@ class RclConfig {
|
||||
* The returned command has substitutable places for input file name
|
||||
* and temp dir name, and will return output name
|
||||
*/
|
||||
bool getUncompressor(const std::string &mtpe, std::list<std::string>& cmd);
|
||||
bool getUncompressor(const string &mtpe, list<string>& cmd);
|
||||
|
||||
/** Use mimemap to compute mimetype */
|
||||
std::string getMimeTypeFromSuffix(const std::string &suffix);
|
||||
string getMimeTypeFromSuffix(const string &suffix);
|
||||
|
||||
/** Get input filter from mimeconf for mimetype */
|
||||
std::string getMimeHandlerDef(const std::string &mimetype);
|
||||
string getMimeHandlerDef(const string &mimetype);
|
||||
|
||||
/** Get external viewer exec string from mimeconf for mimetype */
|
||||
std::string getMimeViewerDef(const std::string &mimetype);
|
||||
string getMimeViewerDef(const string &mimetype);
|
||||
|
||||
/** Get icon name from mimeconf for mimetype */
|
||||
string getMimeIconName(const string &mtype, string *path = 0);
|
||||
|
||||
/** Get a list of all indexable mime types defined in mimemap */
|
||||
std::list<string> getAllMimeTypes();
|
||||
list<string> getAllMimeTypes();
|
||||
|
||||
/** Find exec file for external filter. cmd is the command name from the
|
||||
* command string returned by getMimeHandlerDef */
|
||||
std::string findFilter(const std::string& cmd);
|
||||
string findFilter(const string& cmd);
|
||||
|
||||
~RclConfig() {
|
||||
freeAll();
|
||||
@ -114,7 +124,7 @@ class RclConfig {
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
std::list<string> getConfNames(const string &sk) {
|
||||
list<string> getConfNames(const string &sk) {
|
||||
return m_conf->getNames(sk);
|
||||
}
|
||||
|
||||
@ -129,7 +139,7 @@ class RclConfig {
|
||||
ConfStack<ConfTree> *mimemap; // The files don't change with keydir, but their
|
||||
ConfStack<ConfTree> *mimeconf; // content may depend on it.
|
||||
|
||||
std::list<std::string> *stopsuffixes;
|
||||
list<string> *stopsuffixes;
|
||||
|
||||
// Parameters auto-fetched on setkeydir
|
||||
string defcharset; // These are stored locally to avoid
|
||||
|
||||
@ -2,16 +2,20 @@ depth = ..
|
||||
include $(depth)/mk/sysconf
|
||||
|
||||
PROGS = recollindex csguess mimetype
|
||||
SRCS = recollindex.cpp
|
||||
SRCS = recollindex.cpp rclmonrcv.cpp rclmonprc.cpp
|
||||
|
||||
all: depend $(PROGS) $(BIGLIB)
|
||||
|
||||
RECOLLINDEX_OBJS= recollindex.o $(BIGLIB) $(MIMELIB)
|
||||
RECOLLINDEX_OBJS= recollindex.o rclmonrcv.o rclmonprc.o $(BIGLIB) $(MIMELIB)
|
||||
recollindex : $(RECOLLINDEX_OBJS)
|
||||
$(CXX) $(ALL_CXXFLAGS) -o recollindex $(RECOLLINDEX_OBJS) \
|
||||
$(BSTATIC) $(LIBXAPIAN) $(LIBICONV) $(BDYNAMIC) $(LIBSYS)
|
||||
$(BSTATIC) $(LIBXAPIAN) $(LIBICONV) $(BDYNAMIC) -lfam $(LIBSYS)
|
||||
recollindex.o : recollindex.cpp
|
||||
$(CXX) $(ALL_CXXFLAGS) -c -o recollindex.o $<
|
||||
rclmonrcv.o : rclmonrcv.cpp
|
||||
$(CXX) $(ALL_CXXFLAGS) -c -o rclmonrcv.o $<
|
||||
rclmonprc.o : rclmonprc.cpp
|
||||
$(CXX) $(ALL_CXXFLAGS) -c -o rclmonprc.o $<
|
||||
|
||||
CSGUESS_OBJS= trcsguess.o $(BIGLIB)
|
||||
csguess : $(CSGUESS_OBJS)
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.37 2006-10-12 14:46:02 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.38 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
@ -26,6 +26,7 @@ static char rcsid[] = "@(#$Id: indexer.cpp,v 1.37 2006-10-12 14:46:02 dockes Exp
|
||||
#include <unistd.h>
|
||||
#include <errno.h>
|
||||
#include <strings.h>
|
||||
#include <fnmatch.h>
|
||||
|
||||
#include <iostream>
|
||||
#include <list>
|
||||
@ -207,16 +208,17 @@ bool DbIndexer::createAspellDict()
|
||||
}
|
||||
|
||||
/**
|
||||
Index individual files, out of a full tree run. No database purging
|
||||
*/
|
||||
* Index individual files, out of a full tree run. No database purging
|
||||
*/
|
||||
bool DbIndexer::indexFiles(const list<string> &filenames)
|
||||
{
|
||||
if (!init())
|
||||
return false;
|
||||
|
||||
list<string>::const_iterator it;
|
||||
for (it = filenames.begin(); it != filenames.end();it++) {
|
||||
m_config->setKeyDir(path_getfather(*it));
|
||||
for (it = filenames.begin(); it != filenames.end(); it++) {
|
||||
string dir = path_getfather(*it);
|
||||
m_config->setKeyDir(dir);
|
||||
int abslen;
|
||||
if (m_config->getConfParam("idxabsmlen", &abslen))
|
||||
m_db.setAbstractParams(abslen, -1, -1);
|
||||
@ -231,12 +233,37 @@ bool DbIndexer::indexFiles(const list<string> &filenames)
|
||||
it->c_str()));
|
||||
continue;
|
||||
}
|
||||
|
||||
static string lstdir;
|
||||
static list<string> skpl;
|
||||
if (lstdir.compare(dir)) {
|
||||
LOGDEB(("Recomputing list of skipped names\n"));
|
||||
string skipped;
|
||||
if (m_config->getConfParam("skippedNames", skipped)) {
|
||||
stringToStrings(skipped, skpl);
|
||||
lstdir = dir;
|
||||
}
|
||||
}
|
||||
if (!skpl.empty()) {
|
||||
list<string>::const_iterator skit;
|
||||
string fn = path_getsimple(*it);
|
||||
for (skit = skpl.begin(); skit != skpl.end(); skit++) {
|
||||
if (fnmatch(skit->c_str(), fn.c_str(), 0) == 0) {
|
||||
LOGDEB(("Skipping [%s] :matches skip list\n", fn.c_str()));
|
||||
goto skipped;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (processone(*it, &stb, FsTreeWalker::FtwRegular) !=
|
||||
FsTreeWalker::FtwOk) {
|
||||
LOGERR(("DbIndexer::indexFiles: Database error\n"));
|
||||
return false;
|
||||
}
|
||||
skipped:
|
||||
false; // Need a statement here to make compiler happy ??
|
||||
}
|
||||
|
||||
// The close would be done in our destructor, but we want status here
|
||||
if (!m_db.close()) {
|
||||
LOGERR(("DbIndexer::indexfiles: error closing database in %s\n",
|
||||
@ -371,27 +398,9 @@ ConfIndexer::~ConfIndexer()
|
||||
deleteZ(m_dbindexer);
|
||||
}
|
||||
|
||||
list<string> topdirsToList(RclConfig *conf)
|
||||
{
|
||||
list<string> tdl;
|
||||
// Retrieve the list of directories to be indexed.
|
||||
string topdirs;
|
||||
if (!conf->getConfParam("topdirs", topdirs)) {
|
||||
LOGERR(("ConfIndexer::index: no top directories in configuration\n"));
|
||||
return tdl;
|
||||
}
|
||||
if (!stringToStrings(topdirs, tdl)) {
|
||||
LOGERR(("ConfIndexer::index: parse error for directory list\n"));
|
||||
}
|
||||
for (list<string>::iterator it = tdl.begin(); it != tdl.end(); it++) {
|
||||
*it = path_tildexpand(*it);
|
||||
}
|
||||
return tdl;
|
||||
}
|
||||
|
||||
bool ConfIndexer::index(bool resetbefore)
|
||||
{
|
||||
list<string> tdl = topdirsToList(m_config);
|
||||
list<string> tdl = m_config->getTopdirs();
|
||||
if (tdl.empty()) {
|
||||
m_reason = "Top directory list (topdirs param.) not found in config"
|
||||
"or Directory list parse error";
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
*/
|
||||
#ifndef _INDEXER_H_INCLUDED_
|
||||
#define _INDEXER_H_INCLUDED_
|
||||
/* @(#$Id: indexer.h,v 1.18 2006-10-12 14:46:02 dockes Exp $ (C) 2004 J.F.Dockes */
|
||||
/* @(#$Id: indexer.h,v 1.19 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes */
|
||||
|
||||
#include <string>
|
||||
#include <list>
|
||||
@ -127,18 +127,22 @@ class DbIndexer : public FsTreeWalkerCB {
|
||||
processone(const string &, const struct stat *,
|
||||
FsTreeWalker::CbFlag);
|
||||
|
||||
/** Return my db dir */
|
||||
string getDbDir() {return m_dbdir;}
|
||||
|
||||
private:
|
||||
FsTreeWalker m_walker;
|
||||
RclConfig *m_config;
|
||||
string m_dbdir;
|
||||
Rcl::Db m_db;
|
||||
string m_tmpdir;
|
||||
RclConfig *m_config;
|
||||
string m_dbdir;
|
||||
Rcl::Db m_db;
|
||||
string m_tmpdir;
|
||||
DbIxStatusUpdater *m_updater;
|
||||
|
||||
bool init(bool rst = false);
|
||||
};
|
||||
|
||||
/** utility function to turn topdirs into a proper list */
|
||||
list<string> topdirsToList(RclConfig *conf);
|
||||
/** Helper method in recollindex.cpp for initial checks/setup to index
|
||||
* a list of files (either from the monitor or the command line) */
|
||||
extern bool indexfiles(RclConfig *config, const list<string> &filenames);
|
||||
|
||||
#endif /* _INDEXER_H_INCLUDED_ */
|
||||
|
||||
65
src/index/rclmon.h
Normal file
65
src/index/rclmon.h
Normal file
@ -0,0 +1,65 @@
|
||||
#ifndef _RCLMON_H_INCLUDED_
|
||||
#define _RCLMON_H_INCLUDED_
|
||||
/* @(#$Id: rclmon.h,v 1.1 2006-10-16 15:33:08 dockes Exp $ (C) 2006 J.F.Dockes */
|
||||
/**
|
||||
* Definitions for the real-time monitoring recoll.
|
||||
* We're interested in file modifications, deletions and renaming.
|
||||
* We use two threads, one to receive events from the source, the
|
||||
* other to perform adequate processing.
|
||||
*
|
||||
* The two threads communicate through an event buffer which is
|
||||
* actually a hash map indexed by file path for easy coalescing of
|
||||
* multiple events to the same file.
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
#include <map>
|
||||
|
||||
#include "rclconfig.h"
|
||||
|
||||
#ifndef NO_NAMESPACES
|
||||
using std::string;
|
||||
using std::multimap;
|
||||
#endif
|
||||
|
||||
class RclMonEvent {
|
||||
public:
|
||||
enum EvType {RCLEVT_NONE, RCLEVT_MODIFY, RCLEVT_DELETE, RCLEVT_RENAME};
|
||||
string m_path;
|
||||
string m_opath;
|
||||
EvType m_etyp;
|
||||
RclMonEvent() : m_etyp(RCLEVT_NONE) {}
|
||||
};
|
||||
|
||||
class RclEQData;
|
||||
|
||||
class RclMonEventQueue {
|
||||
public:
|
||||
RclMonEventQueue();
|
||||
~RclMonEventQueue();
|
||||
/** Unlock queue and wait until there are new events.
|
||||
* Returns with the queue locked */
|
||||
bool wait();
|
||||
/** Unlock queue */
|
||||
bool unlock();
|
||||
/** Lock queue. */
|
||||
bool lock();
|
||||
/** Lock queue and add event. */
|
||||
bool pushEvent(const RclMonEvent &ev);
|
||||
void setTerminate(); /* To all threads: end processing */
|
||||
bool ok();
|
||||
bool empty();
|
||||
RclMonEvent pop();
|
||||
|
||||
// Convenience function for initially communicating config to mon thr
|
||||
void setConfig(RclConfig *conf);
|
||||
RclConfig *getConfig();
|
||||
|
||||
private:
|
||||
RclEQData *m_data;
|
||||
};
|
||||
|
||||
extern RclMonEventQueue rclEQ;
|
||||
extern bool startMonitor(RclConfig *conf, bool nofork);
|
||||
|
||||
#endif /* _RCLMON_H_INCLUDED_ */
|
||||
197
src/index/rclmonprc.cpp
Normal file
197
src/index/rclmonprc.cpp
Normal file
@ -0,0 +1,197 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: rclmonprc.cpp,v 1.1 2006-10-16 15:33:08 dockes Exp $ (C) 2006 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Recoll real time monitor processing. This file has the code to retrieve
|
||||
* event from the event queue and do the database-side processing, and the
|
||||
* initialization function.
|
||||
*/
|
||||
|
||||
#include <pthread.h>
|
||||
|
||||
#include "debuglog.h"
|
||||
#include "rclmon.h"
|
||||
#include "debuglog.h"
|
||||
#include "indexer.h"
|
||||
|
||||
typedef map<string, RclMonEvent> queue_type;
|
||||
|
||||
class RclEQData {
|
||||
public:
|
||||
queue_type m_queue;
|
||||
RclConfig *m_config;
|
||||
bool m_ok;
|
||||
pthread_mutex_t m_mutex;
|
||||
pthread_cond_t m_cond;
|
||||
RclEQData()
|
||||
: m_config(0), m_ok(false)
|
||||
{
|
||||
if (!pthread_mutex_init(&m_mutex, 0) && !pthread_cond_init(&m_cond, 0))
|
||||
m_ok = true;
|
||||
}
|
||||
};
|
||||
|
||||
RclMonEventQueue rclEQ;
|
||||
|
||||
RclMonEventQueue::RclMonEventQueue()
|
||||
{
|
||||
m_data = new RclEQData;
|
||||
}
|
||||
RclMonEventQueue::~RclMonEventQueue()
|
||||
{
|
||||
delete m_data;
|
||||
}
|
||||
bool RclMonEventQueue::empty()
|
||||
{
|
||||
return m_data == 0 ? true : m_data->m_queue.empty();
|
||||
}
|
||||
|
||||
RclMonEvent RclMonEventQueue::pop()
|
||||
{
|
||||
RclMonEvent ev;
|
||||
if (!empty()) {
|
||||
ev = m_data->m_queue.begin()->second;
|
||||
m_data->m_queue.erase(m_data->m_queue.begin());
|
||||
}
|
||||
return ev;
|
||||
}
|
||||
|
||||
/** Wait until there is something to process on the queue.
|
||||
* Must be called with the queue locked
|
||||
*/
|
||||
bool RclMonEventQueue::wait()
|
||||
{
|
||||
if (!empty())
|
||||
return true;
|
||||
if (pthread_cond_wait(&m_data->m_cond, &m_data->m_mutex)) {
|
||||
LOGERR(("RclMonEventQueue::wait: pthread_cond_wait failed\n"));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool RclMonEventQueue::lock()
|
||||
{
|
||||
if (pthread_mutex_lock(&m_data->m_mutex)) {
|
||||
LOGERR(("RclMonEventQueue::lock: pthread_mutex_lock failed\n"));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
bool RclMonEventQueue::unlock()
|
||||
{
|
||||
if (pthread_mutex_unlock(&m_data->m_mutex)) {
|
||||
LOGERR(("RclMonEventQueue::lock: pthread_mutex_unlock failed\n"));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void RclMonEventQueue::setConfig(RclConfig *cnf)
|
||||
{
|
||||
m_data->m_config = cnf;
|
||||
}
|
||||
|
||||
RclConfig *RclMonEventQueue::getConfig()
|
||||
{
|
||||
return m_data->m_config;
|
||||
}
|
||||
|
||||
bool RclMonEventQueue::ok()
|
||||
{
|
||||
if (m_data == 0)
|
||||
return false;
|
||||
return m_data->m_ok;
|
||||
}
|
||||
|
||||
void RclMonEventQueue::setTerminate()
|
||||
{
|
||||
lock();
|
||||
m_data->m_ok = false;
|
||||
pthread_cond_broadcast(&m_data->m_cond);
|
||||
unlock();
|
||||
}
|
||||
|
||||
bool RclMonEventQueue::pushEvent(const RclMonEvent &ev)
|
||||
{
|
||||
LOGDEB2(("RclMonEventQueue::pushEvent for %s\n", ev.m_path.c_str()));
|
||||
lock();
|
||||
// It seems that a newer event always override any older. TBVerified ?
|
||||
m_data->m_queue[ev.m_path] = ev;
|
||||
pthread_cond_broadcast(&m_data->m_cond);
|
||||
unlock();
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
pthread_t rcv_thrid;
|
||||
void *rcv_result;
|
||||
extern void *rclMonRcvRun(void *);
|
||||
extern int stopindexing;
|
||||
|
||||
bool startMonitor(RclConfig *conf, bool nofork)
|
||||
{
|
||||
rclEQ.setConfig(conf);
|
||||
if (pthread_create(&rcv_thrid, 0, &rclMonRcvRun, &rclEQ) != 0) {
|
||||
LOGERR(("start_monitoring: cant create event-receiving thread\n"));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!rclEQ.lock()) {
|
||||
return false;
|
||||
}
|
||||
LOGDEB(("start_monitoring: entering main loop\n"));
|
||||
while (rclEQ.wait()) {
|
||||
LOGDEB2(("startMonitor: wait returned\n"));
|
||||
if (stopindexing || !rclEQ.ok())
|
||||
break;
|
||||
list<string> modified;
|
||||
list<string> deleted;
|
||||
|
||||
// Process event queue
|
||||
while (!rclEQ.empty()) {
|
||||
// Retrieve event
|
||||
RclMonEvent ev = rclEQ.pop();
|
||||
switch (ev.m_etyp) {
|
||||
case RclMonEvent::RCLEVT_MODIFY:
|
||||
LOGDEB(("Monitor: Modify/Check on %s\n", ev.m_path.c_str()));
|
||||
modified.push_back(ev.m_path);
|
||||
break;
|
||||
case RclMonEvent::RCLEVT_DELETE:
|
||||
LOGDEB(("Monitor: Delete on %s\n", ev.m_path.c_str()));
|
||||
deleted.push_back(ev.m_path);
|
||||
break;
|
||||
case RclMonEvent::RCLEVT_RENAME:
|
||||
LOGDEB(("Monitor: Rename on %s\n", ev.m_path.c_str()));
|
||||
break;
|
||||
default:
|
||||
LOGDEB(("Monitor: got Other on %s\n", ev.m_path.c_str()));
|
||||
}
|
||||
}
|
||||
// Unlock queue before processing lists
|
||||
rclEQ.unlock();
|
||||
// Process
|
||||
indexfiles(conf, modified);
|
||||
// Lock queue before waiting again
|
||||
rclEQ.lock();
|
||||
}
|
||||
LOGERR(("start_monitoring: rclEQ::wait() failed\n"));
|
||||
return false;
|
||||
}
|
||||
284
src/index/rclmonrcv.cpp
Normal file
284
src/index/rclmonrcv.cpp
Normal file
@ -0,0 +1,284 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: rclmonrcv.cpp,v 1.1 2006-10-16 15:33:08 dockes Exp $ (C) 2006 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
#include <unistd.h>
|
||||
#include <sys/stat.h>
|
||||
#include <errno.h>
|
||||
|
||||
#include "debuglog.h"
|
||||
#include "rclmon.h"
|
||||
#include "fstreewalk.h"
|
||||
#include "indexer.h"
|
||||
#include "pathut.h"
|
||||
|
||||
/**
|
||||
* Recoll real time monitor event receiver. This file has code to interface
|
||||
* to FAM and place events on the event queue.
|
||||
*/
|
||||
|
||||
|
||||
/** A small virtual interface for monitors. Suitable to let either of
|
||||
fam/gamin/ or raw imonitor hide behind */
|
||||
class RclMonitor {
|
||||
public:
|
||||
RclMonitor(){}
|
||||
virtual ~RclMonitor() {}
|
||||
virtual bool addWatch(const string& path, const struct stat&) = 0;
|
||||
virtual bool getEvent(RclMonEvent& ev) = 0;
|
||||
virtual bool ok() = 0;
|
||||
};
|
||||
// Monitor factory
|
||||
static RclMonitor *makeMonitor();
|
||||
|
||||
/** Class used to create the directory watches */
|
||||
class WalkCB : public FsTreeWalkerCB {
|
||||
public:
|
||||
WalkCB(RclConfig *conf, RclMonitor *mon)
|
||||
: m_conf(conf), m_mon(mon)
|
||||
{}
|
||||
virtual ~WalkCB()
|
||||
{}
|
||||
|
||||
virtual FsTreeWalker::Status
|
||||
processone(const string &fn, const struct stat *st,
|
||||
FsTreeWalker::CbFlag flg)
|
||||
{
|
||||
LOGDEB2(("rclMonRcvRun: processone %s m_mon %p m_mon->ok %d\n",
|
||||
fn.c_str(), m_mon, m_mon?m_mon->ok():0));
|
||||
if (flg == FsTreeWalker::FtwDirEnter) {
|
||||
if (!m_mon || !m_mon->ok() || !m_mon->addWatch(fn, *st))
|
||||
return FsTreeWalker::FtwError;
|
||||
}
|
||||
return FsTreeWalker::FtwOk;
|
||||
}
|
||||
private:
|
||||
RclConfig *m_conf;
|
||||
RclMonitor *m_mon;
|
||||
};
|
||||
|
||||
/** Main thread routine: create watches, then wait for events an queue them */
|
||||
void *rclMonRcvRun(void *q)
|
||||
{
|
||||
RclMonEventQueue *queue = (RclMonEventQueue *)q;
|
||||
RclMonitor *mon;
|
||||
|
||||
LOGDEB(("rclMonRcvRun: running\n"));
|
||||
|
||||
if ((mon = makeMonitor()) == 0) {
|
||||
LOGERR(("rclMonRcvRun: makeMonitor failed\n"));
|
||||
rclEQ.setTerminate();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Get top directories from config and walk trees to add watches
|
||||
FsTreeWalker walker;
|
||||
WalkCB walkcb(queue->getConfig(), mon);
|
||||
list<string> tdl = queue->getConfig()->getTopdirs();
|
||||
if (tdl.empty()) {
|
||||
LOGERR(("rclMonRcvRun:: top directory list (topdirs param.) not"
|
||||
"found in config or Directory list parse error"));
|
||||
rclEQ.setTerminate();
|
||||
return 0;
|
||||
}
|
||||
for (list<string>::iterator it = tdl.begin(); it != tdl.end(); it++) {
|
||||
queue->getConfig()->setKeyDir(*it);
|
||||
walker.clearSkippedNames();
|
||||
string skipped;
|
||||
if (queue->getConfig()->getConfParam("skippedNames", skipped)) {
|
||||
list<string> skpl;
|
||||
stringToStrings(skipped, skpl);
|
||||
walker.setSkippedNames(skpl);
|
||||
}
|
||||
LOGDEB(("rclMonRcvRun: walking %s\n", it->c_str()));
|
||||
walker.walk(*it, walkcb);
|
||||
}
|
||||
|
||||
// Forever wait for monitoring events and add them to queue:
|
||||
LOGDEB2(("rclMonRcvRun: waiting for events. rclEQ.ok() %d\n", rclEQ.ok()));
|
||||
while (rclEQ.ok()) {
|
||||
if (!mon->ok())
|
||||
break;
|
||||
RclMonEvent ev;
|
||||
if (mon->getEvent(ev)) {
|
||||
rclEQ.pushEvent(ev);
|
||||
}
|
||||
if (!mon->ok())
|
||||
break;
|
||||
}
|
||||
LOGDEB(("rclMonRcvRun: exiting\n"));
|
||||
rclEQ.setTerminate();
|
||||
return 0;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
/** Fam/gamin -based monitor class */
|
||||
#include <fam.h>
|
||||
#include <sys/select.h>
|
||||
|
||||
static const char *event_name(int code)
|
||||
{
|
||||
static const char *famevent[] = {
|
||||
"",
|
||||
"FAMChanged",
|
||||
"FAMDeleted",
|
||||
"FAMStartExecuting",
|
||||
"FAMStopExecuting",
|
||||
"FAMCreated",
|
||||
"FAMMoved",
|
||||
"FAMAcknowledge",
|
||||
"FAMExists",
|
||||
"FAMEndExist"
|
||||
};
|
||||
static char unknown_event[20];
|
||||
|
||||
if (code < FAMChanged || code > FAMEndExist)
|
||||
{
|
||||
sprintf(unknown_event, "unknown (%d)", code);
|
||||
return unknown_event;
|
||||
}
|
||||
return famevent[code];
|
||||
}
|
||||
|
||||
// FAM based monitor class
|
||||
class RclFAM : public RclMonitor {
|
||||
public:
|
||||
RclFAM();
|
||||
virtual ~RclFAM();
|
||||
virtual bool addWatch(const string& path, const struct stat& st);
|
||||
virtual bool getEvent(RclMonEvent& ev);
|
||||
bool ok() {return m_ok;}
|
||||
|
||||
private:
|
||||
bool m_ok;
|
||||
FAMConnection m_conn;
|
||||
void close() {
|
||||
FAMClose(&m_conn);
|
||||
m_ok = false;
|
||||
}
|
||||
map<int,string> m_reqtodir;
|
||||
};
|
||||
|
||||
RclFAM::RclFAM()
|
||||
: m_ok(false)
|
||||
{
|
||||
if (FAMOpen2(&m_conn, "Recoll")) {
|
||||
LOGERR(("RclFAM::RclFAM: FAMOpen2 failed, errno %d\n", errno));
|
||||
return;
|
||||
}
|
||||
m_ok = true;
|
||||
}
|
||||
|
||||
RclFAM::~RclFAM()
|
||||
{
|
||||
if (ok())
|
||||
FAMClose(&m_conn);
|
||||
}
|
||||
|
||||
bool RclFAM::addWatch(const string& path, const struct stat& st)
|
||||
{
|
||||
if (!ok())
|
||||
return false;
|
||||
LOGDEB(("RclFAM::addWatch: adding %s\n", path.c_str()));
|
||||
FAMRequest req;
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
if (FAMMonitorDirectory(&m_conn, path.c_str(), &req, 0) != 0) {
|
||||
LOGERR(("RclFAM::addWatch: FAMMonitorDirectory failed\n"));
|
||||
return false;
|
||||
}
|
||||
m_reqtodir[req.reqnum] = path;
|
||||
} else if (S_ISREG(st.st_mode)) {
|
||||
if (FAMMonitorFile(&m_conn, path.c_str(), &req, 0) != 0) {
|
||||
LOGERR(("RclFAM::addWatch: FAMMonitorFile failed\n"));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool RclFAM::getEvent(RclMonEvent& ev)
|
||||
{
|
||||
if (!ok())
|
||||
return false;
|
||||
LOGDEB2(("RclFAM::getEvent:\n"));
|
||||
|
||||
fd_set readfds;
|
||||
int fam_fd = FAMCONNECTION_GETFD(&m_conn);
|
||||
FD_ZERO(&readfds);
|
||||
FD_SET(fam_fd, &readfds);
|
||||
|
||||
// Note: can't see a reason to set a timeout. Only reason we might
|
||||
// want out is signal which will break the select call anyway (I
|
||||
// don't think that there is any system still using the old bsd-type
|
||||
// syscall re-entrance after signal).
|
||||
LOGDEB(("RclFAM::getEvent: select\n"));
|
||||
if (select(fam_fd + 1, &readfds, 0, 0, 0) < 0) {
|
||||
LOGERR(("RclFAM::getEvent: select failed, errno %d\n", errno));
|
||||
close();
|
||||
return false;
|
||||
}
|
||||
if (!FD_ISSET(fam_fd, &readfds))
|
||||
return false;
|
||||
|
||||
FAMEvent fe;
|
||||
if (FAMNextEvent(&m_conn, &fe) < 0) {
|
||||
LOGERR(("RclFAM::getEvent: FAMNextEvent failed, errno %d\n", errno));
|
||||
close();
|
||||
return false;
|
||||
}
|
||||
map<int,string>::const_iterator it;
|
||||
if ((it = m_reqtodir.find(fe.fr.reqnum)) != m_reqtodir.end()) {
|
||||
ev.m_path = path_cat(it->second, fe.filename);
|
||||
} else {
|
||||
ev.m_path = fe.filename;
|
||||
}
|
||||
LOGDEB(("RclFAM::getEvent: %-12s %s\n",
|
||||
event_name(fe.code), ev.m_path.c_str()));
|
||||
|
||||
switch (fe.code) {
|
||||
case FAMChanged:
|
||||
case FAMCreated:
|
||||
case FAMExists:
|
||||
// Let the other side sort out the status of this file vs the db
|
||||
ev.m_etyp = RclMonEvent::RCLEVT_MODIFY;
|
||||
break;
|
||||
|
||||
case FAMDeleted:
|
||||
ev.m_etyp = RclMonEvent::RCLEVT_DELETE;
|
||||
break;
|
||||
|
||||
case FAMMoved: /* Never generated it seems */
|
||||
LOGDEB(("RclFAM::getEvent: got move event !\n"));
|
||||
ev.m_etyp = RclMonEvent::RCLEVT_MODIFY;
|
||||
break;
|
||||
|
||||
case FAMStartExecuting:
|
||||
case FAMStopExecuting:
|
||||
case FAMAcknowledge:
|
||||
case FAMEndExist:
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// The monitor factory
|
||||
static RclMonitor *makeMonitor()
|
||||
{
|
||||
return new RclFAM;
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.22 2006-10-12 14:46:02 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.23 2006-10-16 15:33:08 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
@ -36,7 +36,7 @@ using namespace std;
|
||||
#include "indexer.h"
|
||||
#include "smallut.h"
|
||||
#include "pathut.h"
|
||||
|
||||
#include "rclmon.h"
|
||||
|
||||
// Globals for exit cleanup
|
||||
ConfIndexer *confindexer;
|
||||
@ -44,34 +44,42 @@ DbIndexer *dbindexer;
|
||||
|
||||
static bool makeDbIndexer(RclConfig *config)
|
||||
{
|
||||
if (dbindexer) {
|
||||
delete dbindexer;
|
||||
dbindexer = 0;
|
||||
}
|
||||
string dbdir = config->getDbDir();
|
||||
if (dbdir.empty()) {
|
||||
fprintf(stderr, "makeDbIndexer: no database directory in "
|
||||
"configuration for %s\n", config->getKeyDir().c_str());
|
||||
return false;
|
||||
}
|
||||
dbindexer = new DbIndexer(config, dbdir);
|
||||
// Check if there is already an indexer for the right db
|
||||
if (dbindexer && dbindexer->getDbDir().compare(dbdir)) {
|
||||
delete dbindexer;
|
||||
dbindexer = 0;
|
||||
}
|
||||
|
||||
if (!dbindexer)
|
||||
dbindexer = new DbIndexer(config, dbdir);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Index a list of files
|
||||
static bool indexfiles(RclConfig *config, const list<string> &filenames)
|
||||
// The list of top directories/files wont change during program run,
|
||||
// let's cache it:
|
||||
static list<string> o_tdl;
|
||||
|
||||
// Index a list of files. We just check that they belong to one of the topdirs
|
||||
// subtrees, and call the indexer method
|
||||
bool indexfiles(RclConfig *config, const list<string> &filenames)
|
||||
{
|
||||
if (filenames.empty())
|
||||
return true;
|
||||
|
||||
list<string> tdl = topdirsToList(config);
|
||||
if (tdl.empty()) {
|
||||
fprintf(stderr, "Top directory list (topdirs param.) not found in"
|
||||
"config or Directory list parse error");
|
||||
return false;
|
||||
}
|
||||
for (list<string>::iterator dit= tdl.begin(); dit!= tdl.end(); dit++) {
|
||||
*dit = path_canon(*dit);
|
||||
|
||||
if (o_tdl.empty()) {
|
||||
o_tdl = config->getTopdirs();
|
||||
if (o_tdl.empty()) {
|
||||
fprintf(stderr, "Top directory list (topdirs param.) "
|
||||
"not found in config or Directory list parse error");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
list<string> myfiles;
|
||||
@ -79,7 +87,9 @@ static bool indexfiles(RclConfig *config, const list<string> &filenames)
|
||||
it != filenames.end(); it++) {
|
||||
string fn = path_canon(*it);
|
||||
bool ok = false;
|
||||
for (list<string>::iterator dit= tdl.begin(); dit!= tdl.end(); dit++) {
|
||||
// Check that this file name belongs to one of our subtrees
|
||||
for (list<string>::iterator dit = o_tdl.begin();
|
||||
dit != o_tdl.end(); dit++) {
|
||||
if (fn.find(*dit) == 0) {
|
||||
myfiles.push_back(fn);
|
||||
ok = true;
|
||||
@ -153,6 +163,8 @@ static int op_flags;
|
||||
#define OPT_s 0x10
|
||||
#define OPT_c 0x20
|
||||
#define OPT_S 0x40
|
||||
#define OPT_m 0x80
|
||||
#define OPT_D 0x100
|
||||
|
||||
static const char usage [] =
|
||||
"\n"
|
||||
@ -161,6 +173,8 @@ static const char usage [] =
|
||||
"recollindex [-z] \n"
|
||||
" Index everything according to configuration file\n"
|
||||
" -z : reset database before starting indexation\n"
|
||||
"recollindex -m [-D]\n"
|
||||
" Perform real time indexation. Don't become a daemon if -D is set\n"
|
||||
"recollindex -i <filename [filename ...]>\n"
|
||||
" Index individual files. No database purge or stem database updates\n"
|
||||
"recollindex -s <lang>\n"
|
||||
@ -181,7 +195,6 @@ Usage(void)
|
||||
exit((op_flags & OPT_h)==0);
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, const char **argv)
|
||||
{
|
||||
string a_config;
|
||||
@ -197,8 +210,10 @@ int main(int argc, const char **argv)
|
||||
case 'c': op_flags |= OPT_c; if (argc < 2) Usage();
|
||||
a_config = *(++argv);
|
||||
argc--; goto b1;
|
||||
case 'D': op_flags |= OPT_D; break;
|
||||
case 'h': op_flags |= OPT_h; break;
|
||||
case 'i': op_flags |= OPT_i; break;
|
||||
case 'm': op_flags |= OPT_m; break;
|
||||
case 's': op_flags |= OPT_s; break;
|
||||
#ifdef RCL_USE_ASPELL
|
||||
case 'S': op_flags |= OPT_S; break;
|
||||
@ -241,6 +256,16 @@ int main(int argc, const char **argv)
|
||||
Usage();
|
||||
string lang = *argv++; argc--;
|
||||
exit(!createstemdb(config, lang));
|
||||
} else if (op_flags & OPT_m) {
|
||||
if (argc != 0)
|
||||
Usage();
|
||||
if (!(op_flags&OPT_D)) {
|
||||
LOGDEB(("Daemonizing\n"));
|
||||
daemon(0,0);
|
||||
}
|
||||
if (startMonitor(config, (op_flags&OPT_D)!=0))
|
||||
exit(0);
|
||||
exit(1);
|
||||
#ifdef RCL_USE_ASPELL
|
||||
} else if (op_flags & OPT_S) {
|
||||
makeDbIndexer(config);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user