tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

monitor: properly handle cleanup on directory moves

Browse Source
This commit is contained in:
Jean-Francois Dockes
2011-09-30 08:56:29 +02:00
parent 702fb88a1e
commit 0c5f41c41c
8 changed files with 309 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/index/Makefile
+7
View File
@@ -21,6 +21,13 @@ rclmonrcv.o : rclmonrcv.cpp
rclmonprc.o : rclmonprc.cpp
$(CXX) $(ALL_CXXFLAGS) -c -o rclmonprc.o $<
SUBTREELIST_OBJS= subtreelist.o $(BIGLIB)
subtreelist : $(SUBTREELIST_OBJS)
$(CXX) $(ALL_CXXFLAGS) -o subtreelist $(SUBTREELIST_OBJS) \
$(LIBICONV) $(LIBXAPIAN) $(LIBSYS)
subtreelist.o : subtreelist.cpp
$(CXX) $(ALL_CXXFLAGS) -DTEST_SUBTREELIST -c subtreelist.cpp
CSGUESS_OBJS= trcsguess.o $(BIGLIB)
csguess : $(CSGUESS_OBJS)
$(CXX) $(ALL_CXXFLAGS) -o csguess $(CSGUESS_OBJS) \
src/index/rclmon.h
+7 -4
View File
@@ -45,10 +45,11 @@ using std::multimap;
*/
class RclMonEvent {
public:
enum EvType {RCLEVT_NONE, RCLEVT_MODIFY, RCLEVT_DELETE,
RCLEVT_DIRCREATE};
enum EvType {RCLEVT_NONE= 0, RCLEVT_MODIFY=1, RCLEVT_DELETE=2,
RCLEVT_DIRCREATE=3, RCLEVT_ISDIR=0x10};
string m_path;
EvType m_etyp;
// Type and flags
int m_etyp;
///// For fast changing files: minimum time interval before reindex
// Minimum interval (from config)
@@ -58,8 +59,10 @@ class RclMonEvent {
// Changed since put in purgatory after reindex
bool m_needidx;
RclMonEvent() : m_etyp(RCLEVT_NONE),
RclMonEvent() : m_etyp(RCLEVT_NONE),
m_itvsecs(0), m_minclock(0), m_needidx(false) {}
EvType evtype() {return EvType(m_etyp & 0xf);}
int evflags() {return m_etyp & 0xf0;}
};
enum RclMonitorOption {RCLMON_NONE=0, RCLMON_NOFORK=1, RCLMON_NOX11=2};
src/index/rclmonprc.cpp
+18 -1
View File
@@ -43,6 +43,7 @@
#include "pathut.h"
#include "x11mon.h"
#include "rclionice.h"
#include "subtreelist.h"
typedef unsigned long mttcast;
@@ -482,14 +483,30 @@ bool startMonitor(RclConfig *conf, int opts)
RclMonEvent ev = rclEQ.pop();
if (ev.m_path.empty())
break;
switch (ev.m_etyp) {
switch (ev.evtype()) {
case RclMonEvent::RCLEVT_MODIFY:
case RclMonEvent::RCLEVT_DIRCREATE:
LOGDEB(("Monitor: Modify/Check on %s\n", ev.m_path.c_str()));
modified.push_back(ev.m_path);
break;
case RclMonEvent::RCLEVT_DELETE:
LOGDEB(("Monitor: Delete on %s\n", ev.m_path.c_str()));
// If this is for a directory (which the caller should
// tell us because he knows), we should purge the db
// of all the subtree, because on a directory rename,
// inotify will only generate one event for the
// renamed top, not the subentries. This is relatively
// complicated to do though, and we currently do not
// do it, and just wait for a restart to do a full run and
// purge.
deleted.push_back(ev.m_path);
if (ev.evflags() & RclMonEvent::RCLEVT_ISDIR) {
list<string> paths;
if (subtreelist(conf, ev.m_path, paths)) {
deleted.insert(deleted.end(),
paths.begin(), paths.end());
}
}
break;
default:
LOGDEB(("Monitor: got Other on [%s]\n", ev.m_path.c_str()));
src/index/rclmonrcv.cpp
+106 -49
View File
@@ -41,9 +41,9 @@ public:
virtual ~RclMonitor() {}
virtual bool addWatch(const string& path, bool isDir) = 0;
virtual bool getEvent(RclMonEvent& ev, int secs = -1) = 0;
virtual bool ok() = 0;
virtual bool ok() const = 0;
// Does this monitor generate 'exist' events at startup?
virtual bool generatesExist() = 0;
virtual bool generatesExist() const = 0;
};
// Monitor factory. We only have one compiled-in kind at a time, no
@@ -96,6 +96,16 @@ public:
flg == FsTreeWalker::FtwRegular) {
// Have to synthetize events for regular files existence
// at startup because the monitor does not do it
// Note 2011-09-29: no sure this is actually needed. We just ran
// an incremental indexing pass (before starting the
// monitor). Why go over the files once more ? The only
// reason I can see would be to catch modifications that
// happen between the incremental and the start of
// monitoring ? There should be another way: maybe start
// monitoring without actually handling events (just
// queue), then run incremental then start handling
// events ? But we also have to do it on a directory
// move! So keep it
RclMonEvent ev;
ev.m_path = fn;
ev.m_etyp = RclMonEvent::RCLEVT_MODIFY;
@@ -180,13 +190,21 @@ void *rclMonRcvRun(void *q)
// timeout so that an intr will be detected
if (mon->getEvent(ev, 2)) {
if (ev.m_etyp == RclMonEvent::RCLEVT_DIRCREATE) {
// Add watch after checking that this doesn't match
// ignored files or paths
string name = path_getsimple(ev.m_path);
if (!walker.inSkippedNames(name) &&
!walker.inSkippedPaths(ev.m_path))
mon->addWatch(ev.m_path, true);
// Recursive addwatch: there may already be stuff
// inside this directory. Ie: files were quickly
// created, or this is actually the target of a
// directory move. This is necessary for inotify, but
// it seems that fam/gamin is doing the job for us so
// that we are generating double events here (no big
// deal as prc will sort/merge).
if (!walker.inSkippedNames(path_getsimple(ev.m_path)) &&
!walker.inSkippedPaths(ev.m_path)) {
LOGDEB(("rclMonRcvRun: walking new dir %s\n",
ev.m_path.c_str()));
walker.walk(ev.m_path, walkcb);
}
}
if (ev.m_etyp != RclMonEvent::RCLEVT_NONE)
queue->pushEvent(ev);
}
@@ -197,6 +215,24 @@ void *rclMonRcvRun(void *q)
return 0;
}
// Utility routine used by both the fam/gamin and inotify versions to get
// rid of the id-path translation for a moved dir
bool eraseWatchSubTree(map<int, string>& idtopath, const string& top)
{
bool found = false;
LOGDEB0(("Clearing map for [%s]\n", top.c_str()));
map<int,string>::iterator it = idtopath.begin();
while (it != idtopath.end()) {
if (it->second.find(top) == 0) {
found = true;
idtopath.erase(it++);
} else {
it++;
}
}
return found;
}
// We dont compile both the inotify and the fam interface and inotify
// has preference
#ifndef RCL_USE_INOTIFY
@@ -215,8 +251,8 @@ public:
virtual ~RclFAM();
virtual bool addWatch(const string& path, bool isdir);
virtual bool getEvent(RclMonEvent& ev, int secs = -1);
bool ok() {return m_ok;}
virtual bool generatesExist() {return true;}
bool ok() const {return m_ok;}
virtual bool generatesExist() const {return true;}
private:
bool m_ok;
@@ -225,7 +261,7 @@ private:
FAMClose(&m_conn);
m_ok = false;
}
map<int,string> m_reqtopath;
map<int,string> m_idtopath;
const char *event_name(int code);
};
@@ -286,7 +322,7 @@ bool RclFAM::addWatch(const string& path, bool isdir)
return false;
}
}
m_reqtopath[req.reqnum] = path;
m_idtopath[req.reqnum] = path;
return true;
}
@@ -346,12 +382,13 @@ bool RclFAM::getEvent(RclMonEvent& ev, int secs)
map<int,string>::const_iterator it;
if ((fe.filename[0] != '/') &&
(it = m_reqtopath.find(fe.fr.reqnum)) != m_reqtopath.end()) {
(it = m_idtopath.find(fe.fr.reqnum)) != m_idtopath.end()) {
ev.m_path = path_cat(it->second, fe.filename);
} else {
ev.m_path = fe.filename;
}
MONDEB(("RclFAM::getEvent: %-12s %s\n",
LOGDEB(("RclFAM::getEvent: %-12s %s\n",
event_name(fe.code), ev.m_path.c_str()));
switch (fe.code) {
@@ -367,13 +404,15 @@ bool RclFAM::getEvent(RclMonEvent& ev, int secs)
ev.m_etyp = RclMonEvent::RCLEVT_MODIFY;
break;
case FAMMoved:
case FAMDeleted:
ev.m_etyp = RclMonEvent::RCLEVT_DELETE;
break;
case FAMMoved: /* Never generated it seems */
LOGINFO(("RclFAM::getEvent: got move event !\n"));
ev.m_etyp = RclMonEvent::RCLEVT_MODIFY;
// We would like to signal a directory here to enable cleaning
// the subtree (on a dir move), but can't test the actual file
// which is gone. Let's rely on the fact that a directory
// should be watched
if (eraseWatchSubTree(m_idtopath, ev.m_path))
ev.m_etyp |= RclMonEvent::RCLEVT_ISDIR;
break;
case FAMStartExecuting:
@@ -401,17 +440,29 @@ bool RclFAM::getEvent(RclMonEvent& ev, int secs)
class RclIntf : public RclMonitor {
public:
RclIntf();
virtual ~RclIntf();
RclIntf()
: m_ok(false), m_fd(-1), m_evp(0), m_ep(0)
{
if ((m_fd = inotify_init()) < 0) {
LOGERR(("RclIntf:: inotify_init failed, errno %d\n", errno));
return;
}
m_ok = true;
}
virtual ~RclIntf()
{
close();
}
virtual bool addWatch(const string& path, bool isdir);
virtual bool getEvent(RclMonEvent& ev, int secs = -1);
bool ok() {return m_ok;}
virtual bool generatesExist() {return false;}
bool ok() const {return m_ok;}
virtual bool generatesExist() const {return false;}
private:
bool m_ok;
int m_fd;
map<int,string> m_wdtopath; // Watch descriptor to name
map<int,string> m_idtopath; // Watch descriptor to name
#define EVBUFSIZE (32*1024)
char m_evbuf[EVBUFSIZE]; // Event buffer
char *m_evp; // Pointer to next event or 0
@@ -455,40 +506,27 @@ const char *RclIntf::event_name(int code)
};
}
RclIntf::RclIntf()
: m_ok(false), m_fd(-1), m_evp(0), m_ep(0)
{
if ((m_fd = inotify_init()) < 0) {
LOGERR(("RclIntf::RclIntf: inotify_init failed, errno %d\n", errno));
return;
}
m_ok = true;
}
RclIntf::~RclIntf()
{
close();
}
bool RclIntf::addWatch(const string& path, bool)
{
if (!ok())
return false;
return false;
MONDEB(("RclIntf::addWatch: adding %s\n", path.c_str()));
// CLOSE_WRITE is covered through MODIFY. CREATE is needed for mkdirs
uint32_t mask = IN_MODIFY | IN_CREATE
| IN_MOVED_FROM | IN_MOVED_TO
| IN_DELETE
| IN_MOVED_FROM | IN_MOVED_TO | IN_DELETE
#ifdef IN_DONT_FOLLOW
| IN_DONT_FOLLOW
#endif
;
#ifdef IN_EXCL_UNLINK
| IN_EXCL_UNLINK
#endif
;
int wd;
if ((wd = inotify_add_watch(m_fd, path.c_str(), mask)) < 0) {
LOGERR(("RclIntf::addWatch: inotify_add_watch failed\n"));
return false;
}
m_wdtopath[wd] = path;
m_idtopath[wd] = path;
return true;
}
@@ -544,8 +582,8 @@ bool RclIntf::getEvent(RclMonEvent& ev, int secs)
m_evp = m_ep = 0;
map<int,string>::const_iterator it;
if ((it = m_wdtopath.find(evp->wd)) == m_wdtopath.end()) {
LOGERR(("RclIntf::getEvent: unknown wd\n"));
if ((it = m_idtopath.find(evp->wd)) == m_idtopath.end()) {
LOGERR(("RclIntf::getEvent: unknown wd %d\n", evp->wd));
return true;
}
ev.m_path = it->second;
@@ -557,17 +595,36 @@ bool RclIntf::getEvent(RclMonEvent& ev, int secs)
MONDEB(("RclIntf::getEvent: %-12s %s\n",
event_name(evp->mask), ev.m_path.c_str()));
if (evp->mask & (IN_MODIFY | IN_MOVED_TO)) {
if ((evp->mask & IN_MOVED_FROM) && (evp->mask & IN_ISDIR)) {
// We get this when a directory is renamed. Erase the subtree
// entries in the map. The subsequent MOVED_TO will recreate
// them. This is probably not needed because the watches
// actually still exist in the kernel, so that the wds
// returned by future addwatches will be the old ones, and the
// map will be updated in place. But still, this feels safer
eraseWatchSubTree(m_idtopath, ev.m_path);
}
if (evp->mask & (IN_MODIFY)) {
ev.m_etyp = RclMonEvent::RCLEVT_MODIFY;
} else if (evp->mask & (IN_DELETE | IN_MOVED_FROM)) {
ev.m_etyp = RclMonEvent::RCLEVT_DELETE;
} else if (evp->mask & (IN_CREATE)) {
if (path_isdir(ev.m_path)) {
if (evp->mask & IN_ISDIR)
ev.m_etyp |= RclMonEvent::RCLEVT_ISDIR;
} else if (evp->mask & (IN_CREATE | IN_MOVED_TO)) {
if (evp->mask & IN_ISDIR) {
ev.m_etyp = RclMonEvent::RCLEVT_DIRCREATE;
} else {
// Return null event. Will get modify event later
return true;
}
} else if (evp->mask & (IN_IGNORED)) {
if (!m_idtopath.erase(evp->wd)) {
LOGDEB0(("Got IGNORE event for unknown watch\n"));
} else {
eraseWatchSubTree(m_idtopath, ev.m_path);
}
} else {
LOGDEB(("RclIntf::getEvent: unhandled event %s 0x%x %s\n",
event_name(evp->mask), evp->mask, ev.m_path.c_str()));
src/index/subtreelist.cpp
+129
View File
@@ -0,0 +1,129 @@
/* Copyright (C) 2007 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef TEST_SUBTREELIST
#include "refcntr.h"
#include "rcldb.h"
#include "searchdata.h"
#include "rclquery.h"
#include "subtreelist.h"
#include "debuglog.h"
bool subtreelist(RclConfig *config, const string& top,
list<string>& paths)
{
LOGDEB(("subtreelist: top: [%s]\n", top.c_str()));
Rcl::Db rcldb(config);
if (!rcldb.open(Rcl::Db::DbRO)) {
LOGERR(("subtreelist: can't open database in [%s]: %s\n",
config->getDbDir().c_str(), rcldb.getReason().c_str()));
return false;
}
Rcl::SearchData *sd = new Rcl::SearchData(Rcl::SCLT_OR);
RefCntr<Rcl::SearchData> rq(sd);
rq->setTopdir(top);
Rcl::Query query(&rcldb);
query.setQuery(rq);
int cnt = query.getResCnt();
for (int i = 0; i < cnt; i++) {
Rcl::Doc doc;
if (!query.getDoc(i, doc))
break;
string path = fileurltolocalpath(doc.url);
if (!path.empty())
paths.push_back(path);
}
return true;
}
#else // TEST
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <iostream>
#include <list>
#include <string>
using namespace std;
#include "subtreelist.h"
#include "rclconfig.h"
#include "rclinit.h"
static char *thisprog;
static char usage [] =
" <path> : list document paths in this tree\n"
;
static void
Usage(void)
{
cerr << thisprog << ": usage:" << endl << usage;
exit(1);
}
static int op_flags;
#define OPT_o 0x2
int main(int argc, char **argv)
{
string top;
thisprog = argv[0];
argc--; argv++;
while (argc > 0 && **argv == '-') {
(*argv)++;
if (!(**argv))
/* Cas du "adb - core" */
Usage();
while (**argv)
switch (*(*argv)++) {
default: Usage(); break;
}
argc--; argv++;
}
if (argc < 1)
Usage();
top = *argv++;argc--;
string reason;
RclConfig *config = recollinit(0, 0, reason, 0);
if (!config || !config->ok()) {
fprintf(stderr, "Recoll init failed: %s\n", reason.c_str());
exit(1);
}
list<string> paths;
if (!subtreelist(config, top, paths)) {
cerr << "subtreelist failed" << endl;
exit(1);
}
for (list<string>::const_iterator it = paths.begin();
it != paths.end(); it++) {
cout << *it << endl;
}
exit(0);
}
#endif
src/index/subtreelist.h
+33
View File
@@ -0,0 +1,33 @@
/* Copyright (C) 2007 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef _SUBTREELIST_H_INCLUDED_
#define _SUBTREELIST_H_INCLUDED_
#include <list>
using std::list;
class RclConfig;
// This queries the database with a pure directory-filter query, to
// retrieve all the entries beside the specified path. This is used by
// the real time indexer to purge entries when a top directory is
// renamed. This is really convoluted, I'd like a better way.
extern bool subtreelist(RclConfig *config, const string& top,
list<string>& paths);
#endif /* _SUBTREELIST_H_INCLUDED_ */