tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

rationalized how we recompute things on setkeydir. recoll_noindex and skippedNames can now be changed at any point in the tree

Browse Source
This commit is contained in:
dockes
2009-11-15 08:38:43 +00:00
parent 1406bca35b
commit 0ccf8fccd9
8 changed files with 225 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/index/fsindexer.cpp
+40 -59
View File
@@ -130,10 +130,6 @@ bool FsIndexer::index()
if (m_config->getConfParam("idxabsmlen", &abslen))
m_db->setAbstractParams(abslen, -1, -1);
// Set up skipped patterns for this subtree. This probably should be
// done in the directory change code in processone() instead.
m_walker.setSkippedNames(m_config->getSkippedNames());
// Walk the directory tree
if (m_walker.walk(*it, *this) != FsTreeWalker::FtwOk) {
LOGERR(("FsIndexer::index: error while indexing %s: %s\n",
@@ -153,8 +149,7 @@ bool FsIndexer::index()
}
static bool matchesSkipped(const list<string>& tdl,
const list<string>& skpnl,
const list<string>& skppl,
FsTreeWalker& walker,
const string& path)
{
// First check what (if any) topdir this is in:
@@ -170,46 +165,33 @@ static bool matchesSkipped(const list<string>& tdl,
return true;
}
// Check path against skippedPaths. If we find a system where
// FNM_LEADING_DIR is undefined (its unposixy), will have to do this for
// all ascendant paths up to the topdir
for (list<string>::const_iterator it = skppl.begin();
it != skppl.end(); it++) {
if (fnmatch(it->c_str(), path.c_str(), FNM_PATHNAME|FNM_LEADING_DIR)
== 0) {
LOGDEB(("FsIndexer::indexFiles: skipping [%s] (skpp)\n",
path.c_str()));
return true;
}
// Check path against skippedPaths.
if (walker.inSkippedPaths(path)) {
LOGDEB(("FsIndexer::indexFiles: skipping [%s] (skpp)\n", path.c_str()));
return true;
}
// Then check all path components up to the topdir against skippedNames
if (!skpnl.empty()) {
string mpath = path;
while (mpath.length() >= td.length() && mpath.length() > 1) {
string fn = path_getsimple(mpath);
for (list<string>::const_iterator it = skpnl.begin();
it != skpnl.end(); it++) {
LOGDEB2(("Checking [%s] against [%s]\n",
fn.c_str(), it->c_str()));
if (fnmatch(it->c_str(), fn.c_str(), 0) == 0) {
LOGDEB(("FsIndexer::indexFiles: skipping [%s] (skpn)\n",
path.c_str()));
return true;
}
}
string::size_type len = mpath.length();
mpath = path_getfather(mpath);
// getfather normally returns a path ending with /, getsimple
// would then return ''
if (!mpath.empty() && mpath[mpath.size()-1] == '/')
mpath.erase(mpath.size()-1);
// should not be necessary, but lets be prudent. If the
// path did not shorten, something is seriously amiss
// (could be an assert actually)
if (mpath.length() >= len)
return true;
string mpath = path;
while (mpath.length() >= td.length() && mpath.length() > 1) {
string fn = path_getsimple(mpath);
if (walker.inSkippedNames(fn)) {
LOGDEB(("FsIndexer::indexFiles: skipping [%s] (skpn)\n",
path.c_str()));
return true;
}
string::size_type len = mpath.length();
mpath = path_getfather(mpath);
// getfather normally returns a path ending with /, getsimple
// would then return ''
if (!mpath.empty() && mpath[mpath.size()-1] == '/')
mpath.erase(mpath.size()-1);
// should not be necessary, but lets be prudent. If the
// path did not shorten, something is seriously amiss
// (could be an assert actually)
if (mpath.length() >= len)
return true;
}
return false;
}
@@ -222,8 +204,21 @@ bool FsIndexer::indexFiles(list<string>& files)
if (!init())
return false;
// We use an FsTreeWalker just for handling the skipped path/name lists
FsTreeWalker walker;
walker.setSkippedPaths(m_config->getSkippedPaths());
for (list<string>::iterator it = files.begin(); it != files.end(); ) {
LOGDEB2(("FsIndexer::indexFiles: [%s]\n", it->c_str()));
m_config->setKeyDir(path_getfather(*it));
walker.setSkippedNames(m_config->getSkippedNames());
// Check path against indexed areas and skipped names/paths
if (matchesSkipped(m_tdl, walker, *it)) {
it++; continue;
}
struct stat stb;
if (lstat(it->c_str(), &stb) != 0) {
LOGERR(("FsIndexer::indexFiles: lstat(%s): %s", it->c_str(),
@@ -238,23 +233,6 @@ bool FsIndexer::indexFiles(list<string>& files)
it++; continue;
}
string dir = path_getfather(*it);
m_config->setKeyDir(dir);
static string lstdir;
static list<string> skpnl;
static list<string> skppl;
if (lstdir.compare(dir)) {
LOGDEB(("Recomputing list of skipped names\n"));
skpnl = m_config->getSkippedNames();
skppl = m_config->getSkippedPaths();
lstdir = dir;
}
// Check path against indexed areas and skipped names/paths
if (matchesSkipped(m_tdl, skpnl, skppl, *it)) {
it++; continue;
}
int abslen;
if (m_config->getConfParam("idxabsmlen", &abslen))
m_db->setAbstractParams(abslen, -1, -1);
@@ -363,6 +341,9 @@ FsIndexer::processone(const std::string &fn, const struct stat *stp,
flg == FsTreeWalker::FtwDirReturn) {
m_config->setKeyDir(fn);
// Set up skipped patterns for this subtree.
m_walker.setSkippedNames(m_config->getSkippedNames());
int abslen;
if (m_config->getConfParam("idxabsmlen", &abslen))
m_db->setAbstractParams(abslen, -1, -1);
src/index/rclmonrcv.cpp
+15 -14
View File
@@ -59,18 +59,27 @@ static RclMonitor *makeMonitor();
while we create the watches)*/
class WalkCB : public FsTreeWalkerCB {
public:
WalkCB(RclConfig *conf, RclMonitor *mon, RclMonEventQueue *queue)
: m_conf(conf), m_mon(mon), m_queue(queue)
WalkCB(RclConfig *conf, RclMonitor *mon, RclMonEventQueue *queue
FsTreeWalker& walker)
: m_config(conf), m_mon(mon), m_queue(queue), m_walker(walker)
{}
virtual ~WalkCB()
{}
virtual FsTreeWalker::Status
processone(const string &fn, const struct stat *st, FsTreeWalker::CbFlag flg)
processone(const string &fn, const struct stat *st,
FsTreeWalker::CbFlag flg)
{
LOGDEB2(("rclMonRcvRun: processone %s m_mon %p m_mon->ok %d\n",
fn.c_str(), m_mon, m_mon?m_mon->ok():0));
if (flg == FsTreeWalker::FtwDirEnter ||
flg == FsTreeWalker::FtwDirReturn) {
m_config->setKeyDir(fn);
// Set up skipped patterns for this subtree.
m_walker.setSkippedNames(m_config->getSkippedNames());
}
if (flg == FsTreeWalker::FtwDirEnter) {
// Create watch when entering directory, but first empty
// whatever events we may already have on queue
@@ -99,9 +108,10 @@ public:
}
private:
RclConfig *m_conf;
RclConfig *m_config;
RclMonitor *m_mon;
RclMonEventQueue *m_queue;
FsTreeWalker& m_walker;
};
/** Main thread routine: create watches, then forever wait for and queue events */
@@ -133,7 +143,7 @@ void *rclMonRcvRun(void *q)
// Walk the directory trees to add watches
FsTreeWalker walker;
walker.setSkippedPaths(queue->getConfig()->getDaemSkippedPaths());
WalkCB walkcb(queue->getConfig(), mon, queue);
WalkCB walkcb(queue->getConfig(), mon, queue, walker);
for (list<string>::iterator it = tdl.begin(); it != tdl.end(); it++) {
queue->getConfig()->setKeyDir(*it);
// Adjust the follow symlinks options
@@ -144,15 +154,6 @@ void *rclMonRcvRun(void *q)
} else {
walker.setOpts(FsTreeWalker::FtwOptNone);
}
// Adjust the skipped names according to config
walker.setSkippedNames(queue->getConfig()->getSkippedNames());
// Add the dbdir to skipped paths. Note that adding the dbdir
// is probably not useful as we'll probably never have
// multiple dbs per config file, and the global dbdir is
// included by
// config->getSkippedPaths(). Still, better to be safe here as
// config->including dbdir in the walk will get us into a loop
walker.addSkippedPath(queue->getConfig()->getDbDir());
LOGDEB(("rclMonRcvRun: walking %s\n", it->c_str()));
walker.walk(*it, walkcb);
}