tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Avoid purging existing subdocuments on file indexing error (e.g.: maybe a file lock issue that will go away)

Browse Source
This commit is contained in:
Jean-Francois Dockes 2019-06-21 17:18:15 +02:00
parent db9fd248f3
commit ee8c5410bd
3 changed files with 336 additions and 327 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
src/index/fsindexer.cpp
View File

@ -586,18 +586,14 @@ FsIndexer::processone(const std::string &fn, const struct stat *stp,
// If we're changing directories, possibly adjust parameters (set
// the current directory in configuration object)
if (flg == FsTreeWalker::FtwDirEnter ||
flg == FsTreeWalker::FtwDirReturn) {
if (flg == FsTreeWalker::FtwDirEnter || flg == FsTreeWalker::FtwDirReturn) {
m_config->setKeyDir(fn);
// Set up filter/skipped patterns for this subtree.
m_walker.setOnlyNames(m_config->getOnlyNames());
m_walker.setSkippedNames(m_config->getSkippedNames());
// Adjust local fields from config for this subtree
if (m_havelocalfields)
localfieldsfromconf();
if (flg == FsTreeWalker::FtwDirReturn)
return FsTreeWalker::FtwOk;
}
@ -616,6 +612,25 @@ FsIndexer::processone(const std::string &fn, const struct stat *stp,
return processonefile(m_config, fn, stp, m_localfields);
}
// Start db update, either by queueing or by direct call
bool FsIndexer::launchAddOrUpdate(const string& udi, const string& parent_udi,
Rcl::Doc& doc)
{
#ifdef IDX_THREADS
if (m_haveSplitQ) {
DbUpdTask *tp = new DbUpdTask(udi, parent_udi, doc);
if (!m_dwqueue.put(tp)) {
LOGERR("processonefile: wqueue.put failed\n");
return false;
} else {
return true;
}
}
#endif
return m_db->addOrUpdate(udi, parent_udi, doc);
}
FsTreeWalker::Status
FsIndexer::processonefile(RclConfig *config,
const std::string &fn, const struct stat *stp,
@ -633,10 +648,9 @@ FsIndexer::processonefile(RclConfig *config,
// excludedmimetypes, etc.
config->setKeyDir(path_getfather(fn));
// Document signature. This is based on m/ctime and size and used
// for the uptodate check (the value computed here is checked
// against the stored one). Changing the computation forces a full
// reindex of course.
// File signature and up to date check. The sig is based on
// m/ctime and size and the possibly new value is checked against
// the stored one.
string sig;
makesig(stp, sig);
string udi;
@ -738,8 +752,9 @@ FsIndexer::processonefile(RclConfig *config,
// If there is an error and the base doc was already seen,
// we're done
if (fis == FileInterner::FIError && hadNullIpath)
if (fis == FileInterner::FIError && hadNullIpath) {
return FsTreeWalker::FtwOk;
}
// Internal access path for multi-document files. If empty, this is
// for the main file.
@ -794,23 +809,10 @@ FsIndexer::processonefile(RclConfig *config,
// Add document to database. If there is an ipath, add it
// as a child of the file document.
#ifdef IDX_THREADS
if (m_haveSplitQ) {
DbUpdTask *tp = new DbUpdTask(udi, doc.ipath.empty() ?
cstr_null : parent_udi, doc);
if (!m_dwqueue.put(tp)) {
LOGERR("processonefile: wqueue.put failed\n");
return FsTreeWalker::FtwError;
}
} else {
#endif
if (!m_db->addOrUpdate(udi, doc.ipath.empty() ?
if (!launchAddOrUpdate(udi, doc.ipath.empty() ?
cstr_null : parent_udi, doc)) {
return FsTreeWalker::FtwError;
}
#ifdef IDX_THREADS
}
#endif
// Tell what we are doing and check for interrupt request
if (m_updater) {
@ -835,6 +837,15 @@ FsIndexer::processonefile(RclConfig *config,
}
}
if (fis == FileInterner::FIError) {
// In case of error, avoid purging any existing
// subdoc. For example on windows, this will avoid erasing
// all the emails from a .ost because it is currently
// locked by Outlook.
LOGDEB("processonefile: internfile error, marking "
"subdocs as existing\n");
m_db->udiTreeMarkExisting(parent_udi);
} else {
// If this doc existed and it's a container, recording for
// possible subdoc purge (this will be used only if we don't do a
// db-wide purge, e.g. if we're called from indexfiles()).
@ -844,6 +855,7 @@ FsIndexer::processonefile(RclConfig *config,
m_purgeCandidates.record(parent_udi);
}
}
}
// If we had no instance with a null ipath, we create an empty
// document to stand for the file itself, to be used mainly for up
@ -873,17 +885,9 @@ FsIndexer::processonefile(RclConfig *config,
fileDoc.sig = sig;
#ifdef IDX_THREADS
if (m_haveSplitQ) {
DbUpdTask *tp = new DbUpdTask(parent_udi, cstr_null, fileDoc);
if (!m_dwqueue.put(tp))
if (!launchAddOrUpdate(parent_udi, cstr_null, fileDoc)) {
return FsTreeWalker::FtwError;
else
return FsTreeWalker::FtwOk;
}
#endif
if (!m_db->addOrUpdate(parent_udi, cstr_null, fileDoc))
return FsTreeWalker::FtwError;
}
return FsTreeWalker::FtwOk;

37
src/index/fsindexer.h
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2009 J.F.Dockes
/* Copyright (C) 2009-2019 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
@ -33,19 +33,23 @@ struct stat;
class DbUpdTask;
class InternfileTask;
namespace Rcl {
class Doc;
}
/** Index selected parts of the file system
Tree indexing: we inherits FsTreeWalkerCB so that, the processone()
method is called by the file-system tree walk code for each file and
directory. We keep all state needed while indexing, and finally call
the methods to purge the db of stale entries and create the stemming
databases.
Tree indexing: we inherits FsTreeWalkerCB so that, the processone()
method is called by the file-system tree walk code for each file and
directory. We keep all state needed while indexing, and finally call
the methods to purge the db of stale entries and create the stemming
databases.
Single file(s) indexing: there are also calls to index or purge lists of files.
No database purging or stem db updating in this case.
Single file(s) indexing: there are also calls to index or purge lists of files.
No database purging or stem db updating in this case.
*/
class FsIndexer : public FsTreeWalkerCB {
public:
public:
/** Constructor does nothing but store parameters
*
* @param cnf Configuration data
@ -76,18 +80,17 @@ class FsIndexer : public FsTreeWalkerCB {
/** Make signature for file up to date checks */
static void makesig(const struct stat *stp, string& out);
private:
private:
class PurgeCandidateRecorder {
public:
PurgeCandidateRecorder()
: dorecord(false) {}
void setRecord(bool onoff)
{
void setRecord(bool onoff) {
dorecord = onoff;
}
void record(const string& udi)
{
void record(const string& udi) {
// This test does not need to be protected: the value is set at
// init and never changed.
if (!dorecord)
@ -97,8 +100,7 @@ class FsIndexer : public FsTreeWalkerCB {
#endif
udis.push_back(udi);
}
const vector<string>& getCandidates()
{
const vector<string>& getCandidates() {
return udis;
}
private:
@ -109,6 +111,9 @@ class FsIndexer : public FsTreeWalkerCB {
std::vector<std::string> udis;
};
bool launchAddOrUpdate(const std::string& udi,
const std::string& parent_udi, Rcl::Doc& doc);
FsTreeWalker m_walker;
RclConfig *m_config;
Rcl::Db *m_db;

2
src/rcldb/rcldb.cpp
View File

@ -2590,7 +2590,7 @@ bool Db::getSubDocs(const Doc &idoc, vector<Doc>& subdocs)
// used for absent FS mountable volumes.
bool Db::udiTreeMarkExisting(const string& udi)
{
LOGDEB("Db::udiTreeWalk: " << udi << endl);
LOGDEB("Db::udiTreeMarkExisting: " << udi << endl);
string wrapd = wrap_prefix(udi_prefix);
string expr = udi + "*";