893 lines
30 KiB
C++
893 lines
30 KiB
C++
/* Copyright (C) 2009-2020 J.F.Dockes
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program; if not, write to the
|
|
* Free Software Foundation, Inc.,
|
|
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
*/
|
|
#include "autoconfig.h"
|
|
|
|
#include "fsindexer.h"
|
|
|
|
#include <stdio.h>
|
|
#include <errno.h>
|
|
#include <cstring>
|
|
|
|
#include <iostream>
|
|
#include <list>
|
|
#include <map>
|
|
#include <algorithm>
|
|
|
|
#include "cstr.h"
|
|
#include "pathut.h"
|
|
#include "rclutil.h"
|
|
#include "conftree.h"
|
|
#include "rclconfig.h"
|
|
#include "fstreewalk.h"
|
|
#include "rcldb.h"
|
|
#include "readfile.h"
|
|
#include "indexer.h"
|
|
#include "transcode.h"
|
|
#include "log.h"
|
|
#include "internfile.h"
|
|
#include "smallut.h"
|
|
#include "chrono.h"
|
|
#include "wipedir.h"
|
|
#include "fileudi.h"
|
|
#include "cancelcheck.h"
|
|
#include "rclinit.h"
|
|
#include "extrameta.h"
|
|
#include "utf8fn.h"
|
|
#include "idxdiags.h"
|
|
#include "fsfetcher.h"
|
|
#if defined(HAVE_POSIX_FADVISE)
|
|
#include <unistd.h>
|
|
#include <fcntl.h>
|
|
#endif
|
|
|
|
using namespace std;
|
|
|
|
#ifdef IDX_THREADS
|
|
class DbUpdTask {
|
|
public:
|
|
// Take some care to avoid sharing string data (if string impl is cow)
|
|
DbUpdTask(const string& u, const string& p, const Rcl::Doc& d)
|
|
: udi(u.begin(), u.end()), parent_udi(p.begin(), p.end())
|
|
{
|
|
d.copyto(&doc);
|
|
}
|
|
string udi;
|
|
string parent_udi;
|
|
Rcl::Doc doc;
|
|
};
|
|
extern void *FsIndexerDbUpdWorker(void*);
|
|
|
|
class InternfileTask {
|
|
public:
|
|
// Take some care to avoid sharing string data (if string impl is cow)
|
|
InternfileTask(const std::string &f, const struct PathStat *i_stp,
|
|
map<string,string> lfields)
|
|
: fn(f.begin(), f.end()), statbuf(*i_stp)
|
|
{
|
|
map_ss_cp_noshr(lfields, &localfields);
|
|
}
|
|
string fn;
|
|
struct PathStat statbuf;
|
|
map<string,string> localfields;
|
|
};
|
|
extern void *FsIndexerInternfileWorker(void*);
|
|
#endif // IDX_THREADS
|
|
|
|
// Thread safe variation of the "missing helpers" storage. Only the
|
|
// addMissing method needs protection, the rest are called from the
|
|
// main thread either before or after the exciting part
|
|
class FSIFIMissingStore : public FIMissingStore {
|
|
#ifdef IDX_THREADS
|
|
std::mutex m_mutex;
|
|
#endif
|
|
public:
|
|
virtual void addMissing(const string& prog, const string& mt)
|
|
{
|
|
#ifdef IDX_THREADS
|
|
std::unique_lock<std::mutex> locker(m_mutex);
|
|
#endif
|
|
FIMissingStore::addMissing(prog, mt);
|
|
}
|
|
};
|
|
|
|
FsIndexer::FsIndexer(RclConfig *cnf, Rcl::Db *db)
|
|
: m_config(cnf), m_db(db),
|
|
m_missing(new FSIFIMissingStore), m_detectxattronly(false),
|
|
m_noretryfailed(false)
|
|
#ifdef IDX_THREADS
|
|
, m_iwqueue("Internfile", cnf->getThrConf(RclConfig::ThrIntern).first),
|
|
m_dwqueue("Split", cnf->getThrConf(RclConfig::ThrSplit).first)
|
|
#endif // IDX_THREADS
|
|
{
|
|
LOGDEB1("FsIndexer::FsIndexer\n");
|
|
m_havelocalfields = m_config->hasNameAnywhere("localfields");
|
|
m_config->getConfParam("detectxattronly", &m_detectxattronly);
|
|
|
|
#ifdef IDX_THREADS
|
|
m_stableconfig = new RclConfig(*m_config);
|
|
m_haveInternQ = m_haveSplitQ = false;
|
|
int internqlen = cnf->getThrConf(RclConfig::ThrIntern).first;
|
|
int internthreads = cnf->getThrConf(RclConfig::ThrIntern).second;
|
|
if (internqlen >= 0) {
|
|
if (!m_iwqueue.start(internthreads, FsIndexerInternfileWorker, this)) {
|
|
LOGERR("FsIndexer::FsIndexer: intern worker start failed\n");
|
|
return;
|
|
}
|
|
m_haveInternQ = true;
|
|
}
|
|
int splitqlen = cnf->getThrConf(RclConfig::ThrSplit).first;
|
|
int splitthreads = cnf->getThrConf(RclConfig::ThrSplit).second;
|
|
if (splitqlen >= 0) {
|
|
if (!m_dwqueue.start(splitthreads, FsIndexerDbUpdWorker, this)) {
|
|
LOGERR("FsIndexer::FsIndexer: split worker start failed\n");
|
|
return;
|
|
}
|
|
m_haveSplitQ = true;
|
|
}
|
|
LOGDEB("FsIndexer: threads: haveIQ " << m_haveInternQ << " iql " <<
|
|
internqlen << " iqts " << internthreads << " haveSQ " <<
|
|
m_haveSplitQ << " sql " << splitqlen << " sqts " << splitthreads <<
|
|
"\n");
|
|
#endif // IDX_THREADS
|
|
}
|
|
|
|
FsIndexer::~FsIndexer()
|
|
{
|
|
LOGDEB1("FsIndexer::~FsIndexer()\n");
|
|
|
|
#ifdef IDX_THREADS
|
|
void *status;
|
|
if (m_haveInternQ) {
|
|
status = m_iwqueue.setTerminateAndWait();
|
|
LOGDEB0("FsIndexer: internfile wrkr status: "<< status << " (1->ok)\n");
|
|
}
|
|
if (m_haveSplitQ) {
|
|
status = m_dwqueue.setTerminateAndWait();
|
|
LOGDEB0("FsIndexer: dbupd worker status: " << status << " (1->ok)\n");
|
|
}
|
|
delete m_stableconfig;
|
|
#endif // IDX_THREADS
|
|
|
|
delete m_missing;
|
|
}
|
|
|
|
bool FsIndexer::init()
|
|
{
|
|
if (m_tdl.empty()) {
|
|
m_tdl = m_config->getTopdirs();
|
|
if (m_tdl.empty()) {
|
|
LOGERR("FsIndexers: no topdirs list defined\n");
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Recursively index each directory in the topdirs:
|
|
bool FsIndexer::index(int flags)
|
|
{
|
|
bool quickshallow = (flags & ConfIndexer::IxFQuickShallow) != 0;
|
|
m_noretryfailed = (flags & ConfIndexer::IxFNoRetryFailed) != 0;
|
|
m_cleancache = (flags & ConfIndexer::IxFCleanCache) != 0;
|
|
Chrono chron;
|
|
if (!init())
|
|
return false;
|
|
|
|
statusUpdater()->setDbTotDocs(m_db->docCnt());
|
|
|
|
m_walker.setSkippedPaths(m_config->getSkippedPaths());
|
|
if (quickshallow) {
|
|
m_walker.setOpts(m_walker.getOpts() | FsTreeWalker::FtwSkipDotFiles);
|
|
m_walker.setMaxDepth(2);
|
|
}
|
|
|
|
for (const auto& topdir : m_tdl) {
|
|
LOGDEB("FsIndexer::index: Indexing " << topdir << " into " <<
|
|
getDbDir() << "\n");
|
|
|
|
// If a topdirs member appears to be not here or not mounted
|
|
// (empty), avoid deleting all the related index content by
|
|
// marking the current docs as existing.
|
|
if (path_empty(topdir)) {
|
|
m_db->udiTreeMarkExisting(topdir);
|
|
continue;
|
|
}
|
|
|
|
// Set the current directory in config so that subsequent
|
|
// getConfParams() will get local values
|
|
m_config->setKeyDir(topdir);
|
|
|
|
// Adjust the "follow symlinks" option
|
|
bool follow;
|
|
int opts = m_walker.getOpts();
|
|
if (m_config->getConfParam("followLinks", &follow) && follow) {
|
|
opts |= FsTreeWalker::FtwFollow;
|
|
} else {
|
|
opts &= ~FsTreeWalker::FtwFollow;
|
|
}
|
|
m_walker.setOpts(opts);
|
|
|
|
int abslen;
|
|
if (m_config->getConfParam("idxabsmlen", &abslen))
|
|
m_db->setAbstractParams(abslen, -1, -1);
|
|
|
|
// Walk the directory tree
|
|
if (m_walker.walk(topdir, *this) != FsTreeWalker::FtwOk) {
|
|
LOGERR("FsIndexer::index: error while indexing " << topdir <<
|
|
": " << m_walker.getReason() << "\n");
|
|
return false;
|
|
}
|
|
}
|
|
|
|
#ifdef IDX_THREADS
|
|
if (m_haveInternQ)
|
|
m_iwqueue.waitIdle();
|
|
if (m_haveSplitQ)
|
|
m_dwqueue.waitIdle();
|
|
m_db->waitUpdIdle();
|
|
#endif // IDX_THREADS
|
|
|
|
if (m_missing) {
|
|
string missing;
|
|
m_missing->getMissingDescription(missing);
|
|
if (!missing.empty()) {
|
|
LOGINFO("FsIndexer::index missing helper program(s):\n" <<
|
|
missing << "\n");
|
|
}
|
|
m_config->storeMissingHelperDesc(missing);
|
|
}
|
|
LOGINFO("fsindexer index time: " << chron.millis() << " mS\n");
|
|
return true;
|
|
}
|
|
|
|
static bool matchesSkipped(
|
|
const vector<string>& tdl, FsTreeWalker& walker, const string& path)
|
|
{
|
|
// Check path against topdirs and skippedPaths. We go up the
|
|
// ancestors until we find either a topdirs or a skippedPaths
|
|
// match. If topdirs is found first-> ok to index (it's possible
|
|
// and useful to configure a topdir under a skippedPath in the
|
|
// config). This matches what happens during the normal fs tree
|
|
// walk.
|
|
string canonpath = path_canon(path);
|
|
|
|
string mpath = canonpath;
|
|
string topdir;
|
|
for (;;) { // Used to test not root here, but root may be in topdirs !
|
|
|
|
for (const auto tdlent : tdl) {
|
|
// the topdirs members are already canonized.
|
|
LOGDEB1("matchesSkipped: comparing ancestor [" << mpath <<
|
|
"] to topdir [" << tdlent << "]\n");
|
|
if (mpath == tdlent) {
|
|
topdir = tdlent;
|
|
goto goodpath;
|
|
}
|
|
}
|
|
|
|
if (walker.inSkippedPaths(mpath, false)) {
|
|
LOGDEB("FsIndexer::indexFiles: skipping [" << path << "] (skpp)\n");
|
|
return true;
|
|
}
|
|
|
|
if (path_isroot(mpath)) {
|
|
break;
|
|
}
|
|
|
|
// Compute father
|
|
string::size_type len = mpath.length();
|
|
mpath = path_getfather(mpath);
|
|
// getfather normally returns a path ending with /, canonic
|
|
// paths don't (except for '/' itself).
|
|
if (!path_isroot(mpath) && mpath[mpath.size()-1] == '/')
|
|
mpath.erase(mpath.size()-1);
|
|
// should not be necessary, but lets be prudent. If the
|
|
// path did not shorten, something is seriously amiss
|
|
// (could be an assert actually)
|
|
if (mpath.length() >= len) {
|
|
LOGERR("FsIndexer::indexFile: internal Error: path [" << mpath <<
|
|
"] did not shorten\n");
|
|
return true;
|
|
}
|
|
|
|
}
|
|
|
|
// We get there if neither topdirs nor skippedPaths tests matched
|
|
LOGDEB("FsIndexer::indexFiles: skipping [" << path << "] (ntd)\n");
|
|
return true;
|
|
|
|
goodpath:
|
|
|
|
// Then check all path components up to the topdir against skippedNames
|
|
mpath = canonpath;
|
|
while (mpath.length() >= topdir.length() && mpath.length() > 1) {
|
|
string fn = path_getsimple(mpath);
|
|
if (walker.inSkippedNames(fn)) {
|
|
LOGDEB("FsIndexer::indexFiles: skipping [" << path << "] (skpn)\n");
|
|
return true;
|
|
}
|
|
|
|
string::size_type len = mpath.length();
|
|
mpath = path_getfather(mpath);
|
|
// getfather normally returns a path ending with /, getsimple
|
|
// would then return ''
|
|
if (!mpath.empty() && mpath[mpath.size()-1] == '/')
|
|
mpath.erase(mpath.size()-1);
|
|
// should not be necessary, but lets be prudent. If the
|
|
// path did not shorten, something is seriously amiss
|
|
// (could be an assert actually)
|
|
if (mpath.length() >= len)
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Index individual files, out of a full tree run. No database purging
|
|
*/
|
|
bool FsIndexer::indexFiles(list<string>& files, int flags)
|
|
{
|
|
LOGDEB("FsIndexer::indexFiles\n");
|
|
m_noretryfailed = (flags & ConfIndexer::IxFNoRetryFailed) != 0;
|
|
m_cleancache = (flags & ConfIndexer::IxFCleanCache) != 0;
|
|
bool ret = false;
|
|
|
|
if (!init())
|
|
return false;
|
|
|
|
int abslen;
|
|
if (m_config->getConfParam("idxabsmlen", &abslen))
|
|
m_db->setAbstractParams(abslen, -1, -1);
|
|
|
|
m_purgeCandidates.setRecord(true);
|
|
|
|
// We use an FsTreeWalker just for handling the skipped path/name lists
|
|
FsTreeWalker walker;
|
|
walker.setSkippedPaths(m_config->getSkippedPaths());
|
|
|
|
for (list<string>::iterator it = files.begin(); it != files.end(); ) {
|
|
LOGDEB2("FsIndexer::indexFiles: [" << *it << "]\n");
|
|
|
|
m_config->setKeyDir(path_getfather(*it));
|
|
if (m_havelocalfields)
|
|
localfieldsfromconf();
|
|
|
|
bool follow = false;
|
|
m_config->getConfParam("followLinks", &follow);
|
|
|
|
walker.setOnlyNames(m_config->getOnlyNames());
|
|
walker.setSkippedNames(m_config->getSkippedNames());
|
|
// Check path against indexed areas and skipped names/paths
|
|
if (!(flags & ConfIndexer::IxFIgnoreSkip) &&
|
|
matchesSkipped(m_tdl, walker, *it)) {
|
|
it++;
|
|
continue;
|
|
}
|
|
|
|
struct PathStat stb;
|
|
int ststat = path_fileprops(*it, &stb, follow);
|
|
if (ststat != 0) {
|
|
LOGERR("FsIndexer::indexFiles: (l)stat " << *it << ": " <<
|
|
strerror(errno) << "\n");
|
|
it++;
|
|
continue;
|
|
}
|
|
if (!(flags & ConfIndexer::IxFIgnoreSkip) &&
|
|
(stb.pst_type == PathStat::PST_REGULAR ||
|
|
stb.pst_type == PathStat::PST_SYMLINK)) {
|
|
if (!walker.inOnlyNames(path_getsimple(*it))) {
|
|
it++;
|
|
continue;
|
|
}
|
|
}
|
|
if (processone(*it, &stb, FsTreeWalker::FtwRegular) != FsTreeWalker::FtwOk) {
|
|
LOGERR("FsIndexer::indexFiles: processone failed\n");
|
|
goto out;
|
|
}
|
|
it = files.erase(it);
|
|
}
|
|
|
|
ret = true;
|
|
out:
|
|
#ifdef IDX_THREADS
|
|
if (m_haveInternQ)
|
|
m_iwqueue.waitIdle();
|
|
if (m_haveSplitQ)
|
|
m_dwqueue.waitIdle();
|
|
m_db->waitUpdIdle();
|
|
#endif // IDX_THREADS
|
|
|
|
// Purge possible orphan documents
|
|
if (ret == true) {
|
|
LOGDEB("Indexfiles: purging orphans\n");
|
|
const vector<string>& purgecandidates = m_purgeCandidates.getCandidates();
|
|
for (vector<string>::const_iterator it = purgecandidates.begin();
|
|
it != purgecandidates.end(); it++) {
|
|
LOGDEB("Indexfiles: purging orphans for " << *it << "\n");
|
|
m_db->purgeOrphans(*it);
|
|
}
|
|
#ifdef IDX_THREADS
|
|
m_db->waitUpdIdle();
|
|
#endif // IDX_THREADS
|
|
}
|
|
|
|
LOGDEB("FsIndexer::indexFiles: done\n");
|
|
return ret;
|
|
}
|
|
|
|
|
|
/** Purge docs for given files out of the database */
|
|
bool FsIndexer::purgeFiles(list<string>& files)
|
|
{
|
|
LOGDEB("FsIndexer::purgeFiles\n");
|
|
bool ret = false;
|
|
if (!init())
|
|
return false;
|
|
|
|
for (list<string>::iterator it = files.begin(); it != files.end(); ) {
|
|
string udi;
|
|
make_udi(*it, cstr_null, udi);
|
|
// rcldb::purgefile returns true if the udi was either not
|
|
// found or deleted, false only in case of actual error
|
|
bool existed;
|
|
if (!m_db->purgeFile(udi, &existed)) {
|
|
LOGERR("FsIndexer::purgeFiles: Database error\n");
|
|
goto out;
|
|
}
|
|
// If we actually deleted something, take it off the list
|
|
if (existed) {
|
|
it = files.erase(it);
|
|
} else {
|
|
it++;
|
|
}
|
|
}
|
|
|
|
ret = true;
|
|
out:
|
|
#ifdef IDX_THREADS
|
|
if (m_haveInternQ)
|
|
m_iwqueue.waitIdle();
|
|
if (m_haveSplitQ)
|
|
m_dwqueue.waitIdle();
|
|
m_db->waitUpdIdle();
|
|
#endif // IDX_THREADS
|
|
LOGDEB("FsIndexer::purgeFiles: done\n");
|
|
return ret;
|
|
}
|
|
|
|
// Local fields can be set for fs subtrees in the configuration file
|
|
void FsIndexer::localfieldsfromconf()
|
|
{
|
|
LOGDEB1("FsIndexer::localfieldsfromconf\n");
|
|
|
|
string sfields;
|
|
m_config->getConfParam("localfields", sfields);
|
|
if (!sfields.compare(m_slocalfields))
|
|
return;
|
|
|
|
m_slocalfields = sfields;
|
|
m_localfields.clear();
|
|
if (sfields.empty())
|
|
return;
|
|
|
|
string value;
|
|
ConfSimple attrs;
|
|
m_config->valueSplitAttributes(sfields, value, attrs);
|
|
vector<string> nmlst = attrs.getNames(cstr_null);
|
|
for (vector<string>::const_iterator it = nmlst.begin();
|
|
it != nmlst.end(); it++) {
|
|
string nm = m_config->fieldCanon(*it);
|
|
attrs.get(*it, m_localfields[nm]);
|
|
LOGDEB2("FsIndexer::localfieldsfromconf: [" << nm << "]->[" <<
|
|
m_localfields[nm] << "]\n");
|
|
}
|
|
}
|
|
|
|
void FsIndexer::setlocalfields(const map<string, string>& fields, Rcl::Doc& doc)
|
|
{
|
|
for (map<string, string>::const_iterator it = fields.begin();
|
|
it != fields.end(); it++) {
|
|
// Being chosen by the user, localfields override values from
|
|
// the filter. The key is already canonic (see
|
|
// localfieldsfromconf())
|
|
doc.meta[it->first] = it->second;
|
|
}
|
|
}
|
|
|
|
#ifdef IDX_THREADS
|
|
// Called updworker as seen from here, but the first step (and only in
|
|
// most meaningful configurations) is doing the word-splitting, which
|
|
// is why the task is referred as "Split" in the grand scheme of
|
|
// things. An other stage usually deals with the actual index update.
|
|
void *FsIndexerDbUpdWorker(void * fsp)
|
|
{
|
|
recoll_threadinit();
|
|
FsIndexer *fip = (FsIndexer*)fsp;
|
|
WorkQueue<DbUpdTask*> *tqp = &fip->m_dwqueue;
|
|
|
|
DbUpdTask *tsk;
|
|
for (;;) {
|
|
size_t qsz;
|
|
if (!tqp->take(&tsk, &qsz)) {
|
|
tqp->workerExit();
|
|
return (void*)1;
|
|
}
|
|
LOGDEB0("FsIndexerDbUpdWorker: task ql " << qsz << "\n");
|
|
if (!fip->m_db->addOrUpdate(tsk->udi, tsk->parent_udi, tsk->doc)) {
|
|
LOGERR("FsIndexerDbUpdWorker: addOrUpdate failed\n");
|
|
tqp->workerExit();
|
|
return (void*)0;
|
|
}
|
|
delete tsk;
|
|
}
|
|
}
|
|
|
|
void *FsIndexerInternfileWorker(void * fsp)
|
|
{
|
|
recoll_threadinit();
|
|
FsIndexer *fip = (FsIndexer*)fsp;
|
|
WorkQueue<InternfileTask*> *tqp = &fip->m_iwqueue;
|
|
RclConfig myconf(*(fip->m_stableconfig));
|
|
|
|
InternfileTask *tsk = 0;
|
|
for (;;) {
|
|
if (!tqp->take(&tsk)) {
|
|
tqp->workerExit();
|
|
return (void*)1;
|
|
}
|
|
LOGDEB0("FsIndexerInternfileWorker: task fn " << tsk->fn << "\n");
|
|
if (fip->processonefile(
|
|
&myconf, tsk->fn, &tsk->statbuf, tsk->localfields) != FsTreeWalker::FtwOk) {
|
|
LOGERR("FsIndexerInternfileWorker: processone failed\n");
|
|
tqp->workerExit();
|
|
return (void*)0;
|
|
}
|
|
LOGDEB1("FsIndexerInternfileWorker: done fn " << tsk->fn << "\n");
|
|
delete tsk;
|
|
}
|
|
}
|
|
#endif // IDX_THREADS
|
|
|
|
/// This method gets called for every file and directory found by the
|
|
/// tree walker.
|
|
///
|
|
/// It checks with the db if the file has changed and needs to be
|
|
/// reindexed. If so, it calls internfile() which will identify the
|
|
/// file type and call an appropriate handler to convert the document into
|
|
/// internal format, which we then add to the database.
|
|
///
|
|
/// Accent and majuscule handling are performed by the db module when doing
|
|
/// the actual indexing work. The Rcl::Doc created by internfile()
|
|
/// mostly contains pretty raw utf8 data.
|
|
FsTreeWalker::Status FsIndexer::processone(
|
|
const std::string &fn, const struct PathStat *stp, FsTreeWalker::CbFlag flg)
|
|
{
|
|
if (!statusUpdater()->update(DbIxStatus::DBIXS_FILES, fn)) {
|
|
return FsTreeWalker::FtwStop;
|
|
}
|
|
|
|
// If we're changing directories, possibly adjust parameters (set
|
|
// the current directory in configuration object)
|
|
if (flg == FsTreeWalker::FtwDirEnter || flg == FsTreeWalker::FtwDirReturn) {
|
|
m_config->setKeyDir(fn);
|
|
// Set up filter/skipped patterns for this subtree.
|
|
m_walker.setOnlyNames(m_config->getOnlyNames());
|
|
m_walker.setSkippedNames(m_config->getSkippedNames());
|
|
// Adjust local fields from config for this subtree
|
|
if (m_havelocalfields)
|
|
localfieldsfromconf();
|
|
if (flg == FsTreeWalker::FtwDirReturn)
|
|
return FsTreeWalker::FtwOk;
|
|
}
|
|
if (flg == FsTreeWalker::FtwSkipped) {
|
|
IdxDiags::theDiags().record(IdxDiags::Skipped, fn);
|
|
return FsTreeWalker::FtwOk;
|
|
}
|
|
#ifdef IDX_THREADS
|
|
if (m_haveInternQ) {
|
|
InternfileTask *tp = new InternfileTask(fn, stp, m_localfields);
|
|
if (m_iwqueue.put(tp)) {
|
|
return FsTreeWalker::FtwOk;
|
|
} else {
|
|
return FsTreeWalker::FtwError;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
return processonefile(m_config, fn, stp, m_localfields);
|
|
}
|
|
|
|
// Start db update, either by queueing or by direct call
|
|
bool FsIndexer::launchAddOrUpdate(const string& udi, const string& parent_udi,
|
|
Rcl::Doc& doc)
|
|
{
|
|
#ifdef IDX_THREADS
|
|
if (m_haveSplitQ) {
|
|
DbUpdTask *tp = new DbUpdTask(udi, parent_udi, doc);
|
|
if (!m_dwqueue.put(tp)) {
|
|
LOGERR("processonefile: wqueue.put failed\n");
|
|
return false;
|
|
} else {
|
|
return true;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
return m_db->addOrUpdate(udi, parent_udi, doc);
|
|
}
|
|
|
|
FsTreeWalker::Status FsIndexer::processonefile(
|
|
RclConfig *config, const std::string &fn, const struct PathStat *stp,
|
|
const map<string, string>& localfields)
|
|
{
|
|
////////////////////
|
|
// Check db up to date ? Doing this before file type
|
|
// identification means that, if usesystemfilecommand is switched
|
|
// from on to off it may happen that some files which are now
|
|
// without mime type will not be purged from the db, resulting
|
|
// in possible 'cannot intern file' messages at query time...
|
|
|
|
// This is needed if we are in a separate thread than processone()
|
|
// (mostly always when multithreading). Needed esp. for
|
|
// excludedmimetypes, etc.
|
|
config->setKeyDir(path_getfather(fn));
|
|
|
|
// File signature and up to date check. The sig is based on
|
|
// m/ctime and size and the possibly new value is checked against
|
|
// the stored one.
|
|
string sig;
|
|
fsmakesig(stp, sig);
|
|
string udi;
|
|
make_udi(fn, cstr_null, udi);
|
|
unsigned int existingDoc;
|
|
string oldsig;
|
|
bool needupdate;
|
|
if (m_noretryfailed) {
|
|
needupdate = m_db->needUpdate(udi, sig, &existingDoc, &oldsig);
|
|
} else {
|
|
needupdate = m_db->needUpdate(udi, sig, &existingDoc, 0);
|
|
}
|
|
|
|
// If ctime (which we use for the sig) differs from mtime, then at most
|
|
// the extended attributes were changed, no need to index content.
|
|
// This unfortunately leaves open the case where the data was
|
|
// modified, then the extended attributes, in which case we will
|
|
// miss the data update. We would have to store both the mtime and
|
|
// the ctime to avoid this
|
|
bool xattronly = m_detectxattronly && !m_db->inFullReset() &&
|
|
existingDoc && needupdate && (stp->pst_mtime < stp->pst_ctime);
|
|
|
|
LOGDEB("processone: needupdate " << needupdate << " noretry " <<
|
|
m_noretryfailed << " existing " << existingDoc << " oldsig [" <<
|
|
oldsig << "]\n");
|
|
|
|
// If noretryfailed is set, check for a file which previously
|
|
// failed to index, and avoid re-processing it
|
|
if (needupdate && m_noretryfailed && existingDoc &&
|
|
!oldsig.empty() && oldsig.back() == '+') {
|
|
// Check that the sigs are the same except for the '+'. If the file
|
|
// actually changed, we always retry (maybe it was fixed)
|
|
string nold = oldsig.substr(0, oldsig.size()-1);
|
|
if (!nold.compare(sig)) {
|
|
LOGDEB("processone: not retrying previously failed file\n");
|
|
m_db->setExistingFlags(udi, existingDoc);
|
|
needupdate = false;
|
|
}
|
|
}
|
|
|
|
if (!needupdate) {
|
|
LOGDEB0("processone: up to date: " << fn << "\n");
|
|
if (!statusUpdater()->update(
|
|
DbIxStatus::DBIXS_FILES, fn, DbIxStatusUpdater::IncrFilesDone)) {
|
|
return FsTreeWalker::FtwStop;
|
|
}
|
|
return FsTreeWalker::FtwOk;
|
|
}
|
|
|
|
LOGDEB0("processone: processing: [" << displayableBytes(stp->pst_size) << "] " << fn << "\n");
|
|
|
|
// Note that we used to do the full path here, but I ended up
|
|
// believing that it made more sense to use only the file name
|
|
string utf8fn = compute_utf8fn(config, fn, true);
|
|
|
|
// parent_udi is initially the same as udi, it will be used if there
|
|
// are subdocs.
|
|
string parent_udi = udi;
|
|
|
|
Rcl::Doc doc;
|
|
char ascdate[30];
|
|
sprintf(ascdate, "%ld", long(stp->pst_mtime));
|
|
|
|
bool hadNullIpath = false;
|
|
string mimetype;
|
|
|
|
if (!xattronly) {
|
|
FileInterner interner(fn, stp, config, FileInterner::FIF_none);
|
|
if (!interner.ok()) {
|
|
// no indexing whatsoever in this case. This typically means that
|
|
// indexallfilenames is not set
|
|
return FsTreeWalker::FtwOk;
|
|
}
|
|
mimetype = interner.getMimetype();
|
|
|
|
interner.setMissingStore(m_missing);
|
|
FileInterner::Status fis = FileInterner::FIAgain;
|
|
bool hadNonNullIpath = false;
|
|
while (fis == FileInterner::FIAgain) {
|
|
doc.erase();
|
|
try {
|
|
fis = interner.internfile(doc);
|
|
} catch (CancelExcept) {
|
|
LOGERR("fsIndexer::processone: interrupted\n");
|
|
return FsTreeWalker::FtwStop;
|
|
}
|
|
|
|
// We index at least the file name even if there was an error.
|
|
// We'll change the signature to ensure that the indexing will
|
|
// be retried every time.
|
|
|
|
// If there is an error and the base doc was already seen,
|
|
// we're done
|
|
if (fis == FileInterner::FIError && hadNullIpath) {
|
|
return FsTreeWalker::FtwOk;
|
|
}
|
|
|
|
// Internal access path for multi-document files. If empty, this is
|
|
// for the main file.
|
|
if (doc.ipath.empty()) {
|
|
hadNullIpath = true;
|
|
if (hadNonNullIpath) {
|
|
// Note that only the filters can reliably compute
|
|
// this. What we do is dependant of the doc order (if
|
|
// we see the top doc first, we won't set the flag)
|
|
doc.haschildren = true;
|
|
}
|
|
} else {
|
|
hadNonNullIpath = true;
|
|
}
|
|
make_udi(fn, doc.ipath, udi);
|
|
|
|
// Set file name, mod time and url if not done by
|
|
// filter. We used to set the top-level container file
|
|
// name for all subdocs without a proper file name, but
|
|
// this did not make sense (resulted in multiple not
|
|
// useful hits on the subdocs when searching for the
|
|
// file name).
|
|
if (doc.fmtime.empty())
|
|
doc.fmtime = ascdate;
|
|
if (doc.url.empty())
|
|
doc.url = path_pathtofileurl(fn);
|
|
const string *fnp = 0;
|
|
if (doc.ipath.empty()) {
|
|
if (!doc.peekmeta(Rcl::Doc::keyfn, &fnp) || fnp->empty())
|
|
doc.meta[Rcl::Doc::keyfn] = utf8fn;
|
|
}
|
|
// Set container file name for all docs, top or subdoc
|
|
doc.meta[Rcl::Doc::keyctfn] = utf8fn;
|
|
|
|
doc.pcbytes = lltodecstr(stp->pst_size);
|
|
// Document signature for up to date checks. All subdocs inherit the
|
|
// file's.
|
|
doc.sig = sig;
|
|
|
|
// If there was an error, ensure indexing will be
|
|
// retried. This is for the once missing, later installed
|
|
// filter case. It can make indexing much slower (if there are
|
|
// myriads of such files, the ext script is executed for them
|
|
// and fails every time)
|
|
if (fis == FileInterner::FIError) {
|
|
IdxDiags::theDiags().record(IdxDiags::Error, fn, doc.ipath);
|
|
doc.sig += cstr_plus;
|
|
}
|
|
|
|
// Possibly add fields from local config
|
|
if (m_havelocalfields)
|
|
setlocalfields(localfields, doc);
|
|
|
|
// Add document to database. If there is an ipath, add it
|
|
// as a child of the file document.
|
|
if (!launchAddOrUpdate(udi, doc.ipath.empty() ? cstr_null : parent_udi, doc)) {
|
|
return FsTreeWalker::FtwError;
|
|
}
|
|
|
|
// Tell what we are doing and check for interrupt request
|
|
int incr = DbIxStatusUpdater::IncrDocsDone;
|
|
std::string sfn(fn);
|
|
if (!doc.ipath.empty()) {
|
|
sfn += "|" + doc.ipath;
|
|
} else {
|
|
if (fis == FileInterner::FIError) {
|
|
incr |= DbIxStatusUpdater::IncrFileErrors;
|
|
}
|
|
incr |= DbIxStatusUpdater::IncrFilesDone;
|
|
}
|
|
if (!statusUpdater()->update(DbIxStatus::DBIXS_FILES, sfn, incr)) {
|
|
return FsTreeWalker::FtwStop;
|
|
}
|
|
|
|
}
|
|
|
|
if (fis == FileInterner::FIError) {
|
|
// In case of error, avoid purging any existing
|
|
// subdoc. For example on windows, this will avoid erasing
|
|
// all the emails from a .ost because it is currently
|
|
// locked by Outlook.
|
|
LOGDEB("processonefile: internfile error, marking "
|
|
"subdocs as existing\n");
|
|
m_db->udiTreeMarkExisting(parent_udi);
|
|
} else {
|
|
// If this doc existed and it's a container, recording for
|
|
// possible subdoc purge (this will be used only if we don't do a
|
|
// db-wide purge, e.g. if we're called from indexfiles()).
|
|
LOGDEB2("processOnefile: existingDoc " << existingDoc <<
|
|
" hadNonNullIpath " << hadNonNullIpath << "\n");
|
|
if (existingDoc && hadNonNullIpath) {
|
|
m_purgeCandidates.record(parent_udi);
|
|
}
|
|
}
|
|
#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
|
|
// See framagit issue 26. If this appears to be a good idea
|
|
// after all (not sure), we'll need a command line switch to
|
|
// control it. For now it's compile-time only.
|
|
if (m_cleancache) {
|
|
int fd = open(fn.c_str(), O_RDONLY);
|
|
if (fd >= 0) {
|
|
if (posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED)) {
|
|
LOGSYSERR("processonefile", "posix_fadvise", fn);
|
|
}
|
|
close(fd);
|
|
}
|
|
}
|
|
#endif
|
|
}
|
|
|
|
// If we had no instance with a null ipath, we create an empty
|
|
// document to stand for the file itself, to be used mainly for up
|
|
// to date checks. Typically this happens for an mbox file.
|
|
//
|
|
// If xattronly is set, ONLY the extattr metadata is valid and will be used
|
|
// by the following step.
|
|
if (xattronly || hadNullIpath == false) {
|
|
LOGDEB("Creating empty doc for file or pure xattr update\n");
|
|
Rcl::Doc fileDoc;
|
|
if (xattronly) {
|
|
map<string, string> xfields;
|
|
reapXAttrs(config, fn, xfields);
|
|
docFieldsFromXattrs(config, xfields, fileDoc);
|
|
fileDoc.onlyxattr = true;
|
|
} else {
|
|
fileDoc.fmtime = ascdate;
|
|
fileDoc.meta[Rcl::Doc::keyfn] = fileDoc.meta[Rcl::Doc::keyctfn] = utf8fn;
|
|
fileDoc.haschildren = true;
|
|
fileDoc.mimetype = mimetype;
|
|
fileDoc.url = path_pathtofileurl(fn);
|
|
if (m_havelocalfields)
|
|
setlocalfields(localfields, fileDoc);
|
|
fileDoc.pcbytes = lltodecstr(stp->pst_size);
|
|
}
|
|
|
|
fileDoc.sig = sig;
|
|
|
|
if (!launchAddOrUpdate(parent_udi, cstr_null, fileDoc)) {
|
|
return FsTreeWalker::FtwError;
|
|
}
|
|
}
|
|
|
|
return FsTreeWalker::FtwOk;
|
|
}
|
|
|