tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

real-time indexer: regular files which were part of the topdirs list were not monitored

Browse Source
This commit is contained in:
Jean-Francois Dockes 2016-08-13 09:17:29 +02:00
parent 306fa42f17
commit 054554576a
1 changed files with 61 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
src/index/rclmonrcv.cpp
View File

@ -76,15 +76,13 @@ public:
FsTreeWalker& walker) FsTreeWalker& walker)
: m_config(conf), m_mon(mon), m_queue(queue), m_walker(walker) : m_config(conf), m_mon(mon), m_queue(queue), m_walker(walker)
{} {}
virtual ~WalkCB() virtual ~WalkCB() {}
{}
virtual FsTreeWalker::Status virtual FsTreeWalker::Status
processone(const string &fn, const struct stat *st, processone(const string &fn, const struct stat *st,
FsTreeWalker::CbFlag flg) FsTreeWalker::CbFlag flg) {
{ MONDEB("rclMonRcvRun: processone " << fn << " m_mon " << m_mon <<
MONDEB(("rclMonRcvRun: processone %s m_mon %p m_mon->ok %d\n", " m_mon->ok " << (m_mon ? m_mon->ok() : false) << std::endl);
fn.c_str(), m_mon, m_mon?m_mon->ok():0));
if (flg == FsTreeWalker::FtwDirEnter || if (flg == FsTreeWalker::FtwDirEnter ||
flg == FsTreeWalker::FtwDirReturn) { flg == FsTreeWalker::FtwDirReturn) {
@ -143,12 +141,12 @@ private:
FsTreeWalker& m_walker; FsTreeWalker& m_walker;
}; };
/** Main thread routine: create watches, then forever wait for and queue events */ // Main thread routine: create watches, then forever wait for and queue events
void *rclMonRcvRun(void *q) void *rclMonRcvRun(void *q)
{ {
RclMonEventQueue *queue = (RclMonEventQueue *)q; RclMonEventQueue *queue = (RclMonEventQueue *)q;
LOGDEB("rclMonRcvRun: running\n" ); LOGDEB("rclMonRcvRun: running\n");
recoll_threadinit(); recoll_threadinit();
// Make a local copy of the configuration as it doesn't like // Make a local copy of the configuration as it doesn't like
// concurrent accesses. It's ok to copy it here as the other // concurrent accesses. It's ok to copy it here as the other
@ -158,16 +156,16 @@ void *rclMonRcvRun(void *q)
// Create the fam/whatever interface object // Create the fam/whatever interface object
RclMonitor *mon; RclMonitor *mon;
if ((mon = makeMonitor()) == 0) { if ((mon = makeMonitor()) == 0) {
LOGERR("rclMonRcvRun: makeMonitor failed\n" ); LOGERR("rclMonRcvRun: makeMonitor failed\n");
queue->setTerminate(); queue->setTerminate();
return 0; return 0;
} }
// Get top directories from config // Get top directories from config
vector<string> tdl = lconfig.getTopdirs(); vector<string> tdl = lconfig.getTopdirs();
if (tdl.empty()) { if (tdl.empty()) {
LOGERR("rclMonRcvRun:: top directory list (topdirs param.) notfound in config or Directory list parse error" ); LOGERR("rclMonRcvRun:: top directory list (topdirs param.) not found "
"in configuration or topdirs list parse error");
queue->setTerminate(); queue->setTerminate();
return 0; return 0;
} }
@ -176,7 +174,7 @@ void *rclMonRcvRun(void *q)
FsTreeWalker walker; FsTreeWalker walker;
walker.setSkippedPaths(lconfig.getDaemSkippedPaths()); walker.setSkippedPaths(lconfig.getDaemSkippedPaths());
WalkCB walkcb(&lconfig, mon, queue, walker); WalkCB walkcb(&lconfig, mon, queue, walker);
for (vector<string>::iterator it = tdl.begin(); it != tdl.end(); it++) { for (auto it = tdl.begin(); it != tdl.end(); it++) {
lconfig.setKeyDir(*it); lconfig.setKeyDir(*it);
// Adjust the follow symlinks options // Adjust the follow symlinks options
bool follow; bool follow;
@ -186,14 +184,29 @@ void *rclMonRcvRun(void *q)
} else { } else {
walker.setOpts(FsTreeWalker::FtwOptNone); walker.setOpts(FsTreeWalker::FtwOptNone);
} }
LOGDEB("rclMonRcvRun: walking " << *it << "\n" ); // We have to special-case regular files which are part of the topdirs
if (walker.walk(*it, walkcb) != FsTreeWalker::FtwOk) { // list because we the tree walker only adds watches for directories
LOGERR("rclMonRcvRun: tree walk failed\n" ); struct stat st;
goto terminate; if (path_fileprops(*it, &st, follow) != 0) {
} LOGERR("rclMonRcvRun: stat failed for " << *it << "\n");
if (walker.getErrCnt() > 0) { continue;
LOGINFO("rclMonRcvRun: fs walker errors: " << (walker.getReason()) << "\n" ); }
} if (S_ISDIR(st.st_mode)) {
LOGDEB("rclMonRcvRun: walking " << *it << "\n");
if (walker.walk(*it, walkcb) != FsTreeWalker::FtwOk) {
LOGERR("rclMonRcvRun: tree walk failed\n");
goto terminate;
}
if (walker.getErrCnt() > 0) {
LOGINFO("rclMonRcvRun: fs walker errors: " <<
walker.getReason() << "\n");
}
} else {
if (!mon->addWatch(*it, false)) {
LOGERR("rclMonRcvRun: addWatch failed for " << *it <<
" errno " << mon->saved_errno << std::endl);
}
}
} }
{ {
@ -202,7 +215,7 @@ void *rclMonRcvRun(void *q)
if (doweb) { if (doweb) {
string webqueuedir = lconfig.getWebQueueDir(); string webqueuedir = lconfig.getWebQueueDir();
if (!mon->addWatch(webqueuedir, true)) { if (!mon->addWatch(webqueuedir, true)) {
LOGERR("rclMonRcvRun: addwatch (webqueuedir) failed\n" ); LOGERR("rclMonRcvRun: addwatch (webqueuedir) failed\n");
if (mon->saved_errno != EACCES && mon->saved_errno != ENOENT) if (mon->saved_errno != EACCES && mon->saved_errno != ENOENT)
goto terminate; goto terminate;
} }
@ -239,13 +252,15 @@ void *rclMonRcvRun(void *q)
// it seems that fam/gamin is doing the job for us so // it seems that fam/gamin is doing the job for us so
// that we are generating double events here (no big // that we are generating double events here (no big
// deal as prc will sort/merge). // deal as prc will sort/merge).
LOGDEB("rclMonRcvRun: walking new dir " << (ev.m_path) << "\n" ); LOGDEB("rclMonRcvRun: walking new dir " << ev.m_path << "\n");
if (walker.walk(ev.m_path, walkcb) != FsTreeWalker::FtwOk) { if (walker.walk(ev.m_path, walkcb) != FsTreeWalker::FtwOk) {
LOGERR("rclMonRcvRun: walking new dir " << (ev.m_path) << ": " << (walker.getReason()) << "\n" ); LOGERR("rclMonRcvRun: walking new dir " << ev.m_path <<
" : " << walker.getReason() << "\n");
goto terminate; goto terminate;
} }
if (walker.getErrCnt() > 0) { if (walker.getErrCnt() > 0) {
LOGINFO("rclMonRcvRun: fs walker errors: " << (walker.getReason()) << "\n" ); LOGINFO("rclMonRcvRun: fs walker errors: " <<
walker.getReason() << "\n");
} }
} }
@ -256,7 +271,7 @@ void *rclMonRcvRun(void *q)
terminate: terminate:
queue->setTerminate(); queue->setTerminate();
LOGINFO("rclMonRcvRun: monrcv thread routine returning\n" ); LOGINFO("rclMonRcvRun: monrcv thread routine returning\n");
return 0; return 0;
} }
@ -340,7 +355,7 @@ RclFAM::RclFAM()
: m_ok(false) : m_ok(false)
{ {
if (FAMOpen2(&m_conn, "Recoll")) { if (FAMOpen2(&m_conn, "Recoll")) {
LOGERR("RclFAM::RclFAM: FAMOpen2 failed, errno " << (errno) << "\n" ); LOGERR("RclFAM::RclFAM: FAMOpen2 failed, errno " << errno << "\n");
return; return;
} }
m_ok = true; m_ok = true;
@ -371,7 +386,7 @@ bool RclFAM::addWatch(const string& path, bool isdir)
// to unblock signals. SIGALRM is not used by the main thread, so at least // to unblock signals. SIGALRM is not used by the main thread, so at least
// ensure that we exit after gamin gets stuck. // ensure that we exit after gamin gets stuck.
if (setjmp(jbuf)) { if (setjmp(jbuf)) {
LOGERR("RclFAM::addWatch: timeout talking to FAM\n" ); LOGERR("RclFAM::addWatch: timeout talking to FAM\n");
return false; return false;
} }
signal(SIGALRM, onalrm); signal(SIGALRM, onalrm);
@ -379,12 +394,12 @@ bool RclFAM::addWatch(const string& path, bool isdir)
FAMRequest req; FAMRequest req;
if (isdir) { if (isdir) {
if (FAMMonitorDirectory(&m_conn, path.c_str(), &req, 0) != 0) { if (FAMMonitorDirectory(&m_conn, path.c_str(), &req, 0) != 0) {
LOGERR("RclFAM::addWatch: FAMMonitorDirectory failed\n" ); LOGERR("RclFAM::addWatch: FAMMonitorDirectory failed\n");
goto out; goto out;
} }
} else { } else {
if (FAMMonitorFile(&m_conn, path.c_str(), &req, 0) != 0) { if (FAMMonitorFile(&m_conn, path.c_str(), &req, 0) != 0) {
LOGERR("RclFAM::addWatch: FAMMonitorFile failed\n" ); LOGERR("RclFAM::addWatch: FAMMonitorFile failed\n");
goto out; goto out;
} }
} }
@ -422,7 +437,7 @@ bool RclFAM::getEvent(RclMonEvent& ev, int msecs)
} }
int ret; int ret;
if ((ret=select(fam_fd+1, &readfds, 0, 0, msecs >= 0 ? &timeout : 0)) < 0) { if ((ret=select(fam_fd+1, &readfds, 0, 0, msecs >= 0 ? &timeout : 0)) < 0) {
LOGERR("RclFAM::getEvent: select failed, errno " << (errno) << "\n" ); LOGERR("RclFAM::getEvent: select failed, errno " << errno << "\n");
close(); close();
return false; return false;
} else if (ret == 0) { } else if (ret == 0) {
@ -449,7 +464,7 @@ bool RclFAM::getEvent(RclMonEvent& ev, int msecs)
MONDEB("RclFAM::getEvent: call FAMNextEvent\n"); MONDEB("RclFAM::getEvent: call FAMNextEvent\n");
FAMEvent fe; FAMEvent fe;
if (FAMNextEvent(&m_conn, &fe) < 0) { if (FAMNextEvent(&m_conn, &fe) < 0) {
LOGERR("RclFAM::getEvent: FAMNextEvent failed, errno " << (errno) << "\n" ); LOGERR("RclFAM::getEvent: FAMNextEvent: errno " << errno << "\n");
close(); close();
return false; return false;
} }
@ -498,7 +513,7 @@ bool RclFAM::getEvent(RclMonEvent& ev, int msecs)
// Have to return something, this is different from an empty queue, // Have to return something, this is different from an empty queue,
// esp if we are trying to empty it... // esp if we are trying to empty it...
if (fe.code != FAMEndExist) if (fe.code != FAMEndExist)
LOGDEB("RclFAM::getEvent: got other event " << (fe.code) << "!\n" ); LOGDEB("RclFAM::getEvent: got other event " << fe.code << "!\n");
ev.m_etyp = RclMonEvent::RCLEVT_NONE; ev.m_etyp = RclMonEvent::RCLEVT_NONE;
break; break;
} }
@ -519,7 +534,7 @@ public:
: m_ok(false), m_fd(-1), m_evp(0), m_ep(0) : m_ok(false), m_fd(-1), m_evp(0), m_ep(0)
{ {
if ((m_fd = inotify_init()) < 0) { if ((m_fd = inotify_init()) < 0) {
LOGERR("RclIntf:: inotify_init failed, errno " << (errno) << "\n" ); LOGERR("RclIntf:: inotify_init failed, errno " << errno << "\n");
return; return;
} }
m_ok = true; m_ok = true;
@ -603,9 +618,11 @@ bool RclIntf::addWatch(const string& path, bool)
int wd; int wd;
if ((wd = inotify_add_watch(m_fd, path.c_str(), mask)) < 0) { if ((wd = inotify_add_watch(m_fd, path.c_str(), mask)) < 0) {
saved_errno = errno; saved_errno = errno;
LOGERR("RclIntf::addWatch: inotify_add_watch failed. errno " << (saved_errno) << "\n" ); LOGERR("RclIntf::addWatch: inotify_add_watch failed. errno " <<
saved_errno << "\n");
if (errno == ENOSPC) { if (errno == ENOSPC) {
LOGERR("RclIntf::addWatch: ENOSPC error may mean that you needincrease the inotify kernel constants. See inotify(7)\n" ); LOGERR("RclIntf::addWatch: ENOSPC error may mean that you should "
"increase the inotify kernel constants. See inotify(7)\n");
} }
return false; return false;
} }
@ -633,8 +650,9 @@ bool RclIntf::getEvent(RclMonEvent& ev, int msecs)
} }
int ret; int ret;
MONDEB("RclIntf::getEvent: select\n"); MONDEB("RclIntf::getEvent: select\n");
if ((ret=select(m_fd + 1, &readfds, 0, 0, msecs >= 0 ? &timeout : 0)) < 0) { if ((ret = select(m_fd + 1, &readfds, 0, 0, msecs >= 0 ? &timeout : 0))
LOGERR("RclIntf::getEvent: select failed, errno " << (errno) << "\n" ); < 0) {
LOGERR("RclIntf::getEvent: select failed, errno " << errno << "\n");
close(); close();
return false; return false;
} else if (ret == 0) { } else if (ret == 0) {
@ -666,7 +684,7 @@ bool RclIntf::getEvent(RclMonEvent& ev, int msecs)
map<int,string>::const_iterator it; map<int,string>::const_iterator it;
if ((it = m_idtopath.find(evp->wd)) == m_idtopath.end()) { if ((it = m_idtopath.find(evp->wd)) == m_idtopath.end()) {
LOGERR("RclIntf::getEvent: unknown wd " << (evp->wd) << "\n" ); LOGERR("RclIntf::getEvent: unknown wd " << evp->wd << "\n");
return true; return true;
} }
ev.m_path = it->second; ev.m_path = it->second;
@ -706,12 +724,13 @@ bool RclIntf::getEvent(RclMonEvent& ev, int msecs)
} }
} else if (evp->mask & (IN_IGNORED)) { } else if (evp->mask & (IN_IGNORED)) {
if (!m_idtopath.erase(evp->wd)) { if (!m_idtopath.erase(evp->wd)) {
LOGDEB0("Got IGNORE event for unknown watch\n" ); LOGDEB0("Got IGNORE event for unknown watch\n");
} else { } else {
eraseWatchSubTree(m_idtopath, ev.m_path); eraseWatchSubTree(m_idtopath, ev.m_path);
} }
} else { } else {
LOGDEB("RclIntf::getEvent: unhandled event " << (event_name(evp->mask)) << " 0x" << (evp->mask) << " " << (ev.m_path) << "\n" ); LOGDEB("RclIntf::getEvent: unhandled event " << event_name(evp->mask) <<
" " << evp->mask << " " << ev.m_path << "\n");
return true; return true;
} }
return true; return true;
@ -732,7 +751,8 @@ static RclMonitor *makeMonitor()
return new RclFAM; return new RclFAM;
#endif #endif
#endif #endif
LOGINFO("RclMonitor: neither Inotify nor Fam was compiled as file system change notification interface\n" ); LOGINFO("RclMonitor: neither Inotify nor Fam was compiled as file system "
"change notification interface\n");
return 0; return 0;
} }
#endif // RCL_MONITOR #endif // RCL_MONITOR