tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

recollindex: do not retry files which previously failed to be indexed, except if they were changed since, or option -k is set

Browse Source
This commit is contained in:
Jean-Francois Dockes
2015-04-23 10:37:37 +02:00
parent 906312519c
commit e6c0ca403d
9 changed files with 209 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/index/fsindexer.cpp
+35 -8
View File
@@ -99,7 +99,8 @@ public:
FsIndexer::FsIndexer(RclConfig *cnf, Rcl::Db *db, DbIxStatusUpdater *updfunc)
: m_config(cnf), m_db(db), m_updater(updfunc),
m_missing(new FSIFIMissingStore), m_detectxattronly(false)
m_missing(new FSIFIMissingStore), m_detectxattronly(false),
m_noretryfailed(false)
#ifdef IDX_THREADS
, m_iwqueue("Internfile", cnf->getThrConf(RclConfig::ThrIntern).first),
m_dwqueue("Split", cnf->getThrConf(RclConfig::ThrSplit).first)
@@ -172,8 +173,10 @@ bool FsIndexer::init()
}
// Recursively index each directory in the topdirs:
bool FsIndexer::index(bool quickshallow)
bool FsIndexer::index(int flags)
{
bool quickshallow = (flags & ConfIndexer::IxFQuickShallow) != 0;
m_noretryfailed = (flags & ConfIndexer::IxFNoRetryFailed) != 0;
Chrono chron;
if (!init())
return false;
@@ -261,7 +264,7 @@ static bool matchesSkipped(const vector<string>& tdl,
for (vector<string>::const_iterator it = tdl.begin();
it != tdl.end(); it++) {
// the topdirs members are already canonized.
LOGDEB2(("indexfiles:matchesskpp: comparing ancestor [%s] to "
LOGDEB2(("matchesSkipped: comparing ancestor [%s] to "
"topdir [%s]\n", mpath.c_str(), it->c_str()));
if (!mpath.compare(*it)) {
topdir = *it;
@@ -324,9 +327,10 @@ goodpath:
/**
* Index individual files, out of a full tree run. No database purging
*/
bool FsIndexer::indexFiles(list<string>& files, ConfIndexer::IxFlag flag)
bool FsIndexer::indexFiles(list<string>& files, int flags)
{
LOGDEB(("FsIndexer::indexFiles\n"));
m_noretryfailed = (flags & ConfIndexer::IxFNoRetryFailed) != 0;
int ret = false;
if (!init())
@@ -354,7 +358,7 @@ bool FsIndexer::indexFiles(list<string>& files, ConfIndexer::IxFlag flag)
walker.setSkippedNames(m_config->getSkippedNames());
// Check path against indexed areas and skipped names/paths
if (!(flag&ConfIndexer::IxFIgnoreSkip) &&
if (!(flags & ConfIndexer::IxFIgnoreSkip) &&
matchesSkipped(m_tdl, walker, *it)) {
it++;
continue;
@@ -648,8 +652,14 @@ FsIndexer::processonefile(RclConfig *config,
makesig(stp, sig);
string udi;
make_udi(fn, cstr_null, udi);
bool existingDoc;
bool needupdate = m_db->needUpdate(udi, sig, &existingDoc);
unsigned int existingDoc;
string oldsig;
bool needupdate;
if (m_noretryfailed) {
needupdate = m_db->needUpdate(udi, sig, &existingDoc, &oldsig);
} else {
needupdate = m_db->needUpdate(udi, sig, &existingDoc, 0);
}
// If ctime (which we use for the sig) differs from mtime, then at most
// the extended attributes were changed, no need to index content.
@@ -659,7 +669,24 @@ FsIndexer::processonefile(RclConfig *config,
// the ctime to avoid this
bool xattronly = m_detectxattronly && !m_db->inFullReset() &&
existingDoc && needupdate && (stp->st_mtime < stp->st_ctime);
LOGDEB(("processone: needupdate %d noretry %d existing %d oldsig [%s]\n",
needupdate, m_noretryfailed, existingDoc, oldsig.c_str()));
// If noretryfailed is set, check for a file which previously
// failed to index, and avoid re-processing it
if (needupdate && m_noretryfailed && existingDoc &&
!oldsig.empty() && *oldsig.rbegin() == '+') {
// Check that the sigs are the same except for the '+'. If the file
// actually changed, we always retry (maybe it was fixed)
string nold = oldsig.substr(0, oldsig.size()-1);
if (!nold.compare(sig)) {
LOGDEB(("processone: not retrying previously failed file\n"));
m_db->setExistingFlags(udi, existingDoc);
needupdate = false;
}
}
if (!needupdate) {
LOGDEB0(("processone: up to date: %s\n", fn.c_str()));
if (m_updater) {
src/index/fsindexer.h
+6 -3
View File
@@ -60,11 +60,11 @@ class FsIndexer : public FsTreeWalkerCB {
* We open the database,
* then call a file system walk for each top-level directory.
*/
bool index(bool quickshallow = 0);
bool index(int flags);
/** Index a list of files. No db cleaning or stemdb updating */
bool indexFiles(std::list<std::string> &files, ConfIndexer::IxFlag f =
ConfIndexer::IxFNone);
bool indexFiles(std::list<std::string> &files,
int f = ConfIndexer::IxFNone);
/** Purge a list of files. */
bool purgeFiles(std::list<std::string> &files);
@@ -136,6 +136,9 @@ class FsIndexer : public FsTreeWalkerCB {
// needs a config option
bool m_detectxattronly;
// No retry of previously failed files
bool m_noretryfailed;
#ifdef IDX_THREADS
friend void *FsIndexerDbUpdWorker(void*);
friend void *FsIndexerInternfileWorker(void*);
src/index/indexer.cpp
+5 -5
View File
@@ -84,13 +84,13 @@ bool ConfIndexer::firstFsIndexingSequence()
}
int flushmb = m_db.getFlushMb();
m_db.setFlushMb(2);
m_fsindexer->index(true);
m_fsindexer->index(IxFQuickShallow);
m_db.doFlush();
m_db.setFlushMb(flushmb);
return true;
}
bool ConfIndexer::index(bool resetbefore, ixType typestorun)
bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
{
Rcl::Db::OpenMode mode = resetbefore ? Rcl::Db::DbTrunc : Rcl::Db::DbUpd;
if (!m_db.open(mode)) {
@@ -106,7 +106,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun)
}
deleteZ(m_fsindexer);
m_fsindexer = new FsIndexer(m_config, &m_db, m_updater);
if (!m_fsindexer || !m_fsindexer->index()) {
if (!m_fsindexer || !m_fsindexer->index(flags)) {
m_db.close();
return false;
}
@@ -154,7 +154,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun)
return true;
}
bool ConfIndexer::indexFiles(list<string>& ifiles, IxFlag flag)
bool ConfIndexer::indexFiles(list<string>& ifiles, int flag)
{
list<string> myfiles;
string origcwd = m_config->getOrigCwd();
@@ -237,7 +237,7 @@ bool ConfIndexer::updateDocs(std::vector<Rcl::Doc> &docs, IxFlag flag)
return true;
}
bool ConfIndexer::purgeFiles(std::list<string> &files, IxFlag flag)
bool ConfIndexer::purgeFiles(std::list<string> &files, int flag)
{
list<string> myfiles;
string origcwd = m_config->getOrigCwd();
src/index/indexer.h
+8 -3
View File
@@ -102,10 +102,15 @@ class ConfIndexer {
enum IxFlag {IxFNone = 0,
IxFIgnoreSkip = 1, // Ignore skipped lists
IxFNoWeb = 2, // Do not process the web queue.
// First pass: just do the top files so that the user can
// try searching asap.
IxFQuickShallow = 4,
// Do not retry files which previously failed ('+' sigs)
IxFNoRetryFailed = 8,
};
/** Run indexers */
bool index(bool resetbefore, ixType typestorun);
bool index(bool resetbefore, ixType typestorun, int f = IxFNone);
const string &getReason() {return m_reason;}
@@ -122,14 +127,14 @@ class ConfIndexer {
static vector<string> getStemmerNames();
/** Index a list of files. No db cleaning or stemdb updating */
bool indexFiles(list<string> &files, IxFlag f = IxFNone);
bool indexFiles(list<string> &files, int f = IxFNone);
/** Update index for list of documents given as list of docs (out of query)
*/
bool updateDocs(vector<Rcl::Doc> &docs, IxFlag f = IxFNone);
static bool docsToPaths(vector<Rcl::Doc> &docs, vector<string> &paths);
/** Purge a list of files. */
bool purgeFiles(list<string> &files, IxFlag f = IxFNone);
bool purgeFiles(list<string> &files, int f = IxFNone);
/** Set in place reset mode */
void setInPlaceReset() {m_db.setInPlaceReset();}
src/index/recollindex.cpp
+15 -7
View File
@@ -68,6 +68,7 @@ static int op_flags;
#define OPT_Z 0x10000
#define OPT_n 0x20000
#define OPT_r 0x40000
#define OPT_k 0x80000
ReExec *o_reexec;
@@ -182,7 +183,7 @@ public:
{
}
virtual FsTreeWalker::Status
processone(const string & fn, const struct stat *, FsTreeWalker::CbFlag flg)
processone(const string& fn, const struct stat *, FsTreeWalker::CbFlag flg)
{
if (flg == FsTreeWalker::FtwDirEnter || flg == FsTreeWalker::FtwRegular)
m_files.push_back(fn);
@@ -255,11 +256,12 @@ static const char usage [] =
"\n"
"recollindex [-h] \n"
" Print help\n"
"recollindex [-z|-Z] \n"
"recollindex [-z|-Z] [-k]\n"
" Index everything according to configuration file\n"
" -z : reset database before starting indexing\n"
" -Z : in place reset: consider all documents as changed. Can also\n"
" be combined with -i or -r but not -m\n"
" -k : retry files on which we previously failed\n"
#ifdef RCL_MONITOR
"recollindex -m [-w <secs>] -x [-D] [-C]\n"
" Perform real time indexing. Don't become a daemon if -D is set.\n"
@@ -282,8 +284,8 @@ static const char usage [] =
"recollindex -s <lang>\n"
" Build stem database for additional language <lang>\n"
#ifdef FUTURE_IMPROVEMENT
"recollindex -b\n"
" Process the Beagle queue\n"
"recollindex -W\n"
" Process the Web queue\n"
#endif
#ifdef RCL_USE_ASPELL
"recollindex -S\n"
@@ -351,6 +353,7 @@ int main(int argc, char **argv)
case 'f': op_flags |= OPT_f; break;
case 'h': op_flags |= OPT_h; break;
case 'i': op_flags |= OPT_i; break;
case 'k': op_flags |= OPT_k; break;
case 'l': op_flags |= OPT_l; break;
case 'm': op_flags |= OPT_m; break;
case 'n': op_flags |= OPT_n; break;
@@ -415,6 +418,10 @@ int main(int argc, char **argv)
bool rezero((op_flags & OPT_z) != 0);
bool inPlaceReset((op_flags & OPT_Z) != 0);
int indexerFlags = ConfIndexer::IxFNone;
if (!(op_flags & OPT_k))
indexerFlags |= ConfIndexer::IxFNoRetryFailed;
Pidfile pidfile(config->getPidfile());
updater = new MyUpdater(config);
@@ -526,8 +533,8 @@ int main(int argc, char **argv)
if (!(op_flags & OPT_n)) {
makeIndexerOrExit(config, inPlaceReset);
LOGDEB(("Recollindex: initial indexing pass before monitoring\n"));
if (!confindexer->index(rezero, ConfIndexer::IxTAll) ||
stopindexing) {
if (!confindexer->index(rezero, ConfIndexer::IxTAll, indexerFlags)
|| stopindexing) {
LOGERR(("recollindex, initial indexing pass failed, "
"not going into monitor mode\n"));
exit(1);
@@ -564,7 +571,8 @@ int main(int argc, char **argv)
} else {
lockorexit(&pidfile);
makeIndexerOrExit(config, inPlaceReset);
bool status = confindexer->index(rezero, ConfIndexer::IxTAll);
bool status = confindexer->index(rezero, ConfIndexer::IxTAll,
indexerFlags);
if (!status)
cerr << "Indexing failed" << endl;
if (!confindexer->getReason().empty())