tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

indents

Browse Source
This commit is contained in:
Jean-Francois Dockes 2020-07-15 10:47:18 +02:00
parent 3cf7fb3b65
commit 96ba5acd32
14 changed files with 505 additions and 533 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/index/Makefile
View File

@ -1,18 +0,0 @@
PROGS = subtreelist mimetype
all: $(PROGS)
SUBTREELIST_OBJS= subtreelist.o
subtreelist : $(SUBTREELIST_OBJS)
$(CXX) $(ALL_CXXFLAGS) -o subtreelist $(SUBTREELIST_OBJS) \
$(LIBRECOLL)
subtreelist.o : subtreelist.cpp
$(CXX) $(ALL_CXXFLAGS) -DTEST_SUBTREELIST -c subtreelist.cpp
MIMETYPE_OBJS= trmimetype.o
mimetype : $(MIMETYPE_OBJS)
$(CXX) $(ALL_CXXFLAGS) -o mimetype $(MIMETYPE_OBJS) $(LIBRECOLL)
trmimetype.o : mimetype.cpp
$(CXX) $(ALL_CXXFLAGS) -DTEST_MIMETYPE -c -o trmimetype.o \
mimetype.cpp
include ../utils/utmkdefs.mk

6
src/index/fetcher.cpp
View File

@ -35,17 +35,17 @@ std::unique_ptr<DocFetcher> docFetcherMake(RclConfig *config,
string backend;
idoc.getmeta(Rcl::Doc::keybcknd, &backend);
if (backend.empty() || !backend.compare("FS")) {
return std::unique_ptr<DocFetcher>(new FSDocFetcher);
return std::unique_ptr<DocFetcher>(new FSDocFetcher);
#ifndef DISABLE_WEB_INDEXER
} else if (!backend.compare("BGL")) {
return std::unique_ptr<DocFetcher>(new WQDocFetcher);
return std::unique_ptr<DocFetcher>(new WQDocFetcher);
#endif
} else {
std::unique_ptr<DocFetcher> f(exeDocFetcherMake(config, backend));
if (!f) {
LOGERR("DocFetcherFactory: unknown backend [" << backend << "]\n");
}
return f;
return f;
}
}

14
src/index/idxstatus.h
View File

@ -23,11 +23,11 @@
// Current status of an indexing operation. This is updated in
// $RECOLL_CONFDIR/idxstatus.txt
class DbIxStatus {
public:
public:
enum Phase {DBIXS_NONE,
DBIXS_FILES, DBIXS_PURGE, DBIXS_STEMDB, DBIXS_CLOSING,
DBIXS_MONITOR,
DBIXS_DONE};
DBIXS_FILES, DBIXS_PURGE, DBIXS_STEMDB, DBIXS_CLOSING,
DBIXS_MONITOR,
DBIXS_DONE};
Phase phase;
std::string fn; // Last file processed
int docsdone; // Documents actually updated
@ -43,9 +43,9 @@ class DbIxStatus {
bool hasmonitor{false};
void reset() {
phase = DBIXS_FILES;
fn.erase();
docsdone = filesdone = fileerrors = dbtotdocs = totfiles = 0;
phase = DBIXS_FILES;
fn.erase();
docsdone = filesdone = fileerrors = dbtotdocs = totfiles = 0;
}
DbIxStatus() {reset();}
};

115
src/index/indexer.cpp
View File

@ -117,9 +117,9 @@ ConfIndexer::ConfIndexer(RclConfig *cnf, DbIxStatusUpdater *updfunc)
ConfIndexer::~ConfIndexer()
{
deleteZ(m_fsindexer);
deleteZ(m_fsindexer);
#ifndef DISABLE_WEB_INDEXER
deleteZ(m_webindexer);
deleteZ(m_webindexer);
#endif
}
@ -132,14 +132,14 @@ bool ConfIndexer::runFirstIndexing()
{
// Indexing status file existing and not empty ?
if (path_filesize(m_config->getIdxStatusFile()) > 0) {
LOGDEB0("ConfIndexer::runFirstIndexing: no: status file not empty\n");
return false;
LOGDEB0("ConfIndexer::runFirstIndexing: no: status file not empty\n");
return false;
}
// And only do this if the user has kept the default topdirs (~).
vector<string> tdl = m_config->getTopdirs();
if (tdl.size() != 1 || tdl[0].compare(path_canon(path_tildexpand("~")))) {
LOGDEB0("ConfIndexer::runFirstIndexing: no: not home only\n");
return false;
LOGDEB0("ConfIndexer::runFirstIndexing: no: not home only\n");
return false;
}
return true;
}
@ -150,7 +150,7 @@ bool ConfIndexer::firstFsIndexingSequence()
deleteZ(m_fsindexer);
m_fsindexer = new FsIndexer(m_config, &m_db, m_updater);
if (!m_fsindexer) {
return false;
return false;
}
int flushmb = m_db.getFlushMb();
m_db.setFlushMb(2);
@ -164,17 +164,17 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
{
Rcl::Db::OpenMode mode = resetbefore ? Rcl::Db::DbTrunc : Rcl::Db::DbUpd;
if (!m_db.open(mode)) {
LOGERR("ConfIndexer: error opening database " << m_config->getDbDir() <<
LOGERR("ConfIndexer: error opening database " << m_config->getDbDir() <<
" : " << m_db.getReason() << "\n");
addIdxReason("indexer", m_db.getReason());
return false;
return false;
}
m_config->setKeyDir(cstr_null);
if (typestorun & IxTFs) {
if (runFirstIndexing()) {
firstFsIndexingSequence();
}
if (runFirstIndexing()) {
firstFsIndexingSequence();
}
deleteZ(m_fsindexer);
m_fsindexer = new FsIndexer(m_config, &m_db, m_updater);
if (!m_fsindexer || !m_fsindexer->index(flags)) {
@ -183,7 +183,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
} else {
addIdxReason("indexer", "Index creation failed. See log.");
}
m_db.close();
m_db.close();
return false;
}
}
@ -193,7 +193,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
deleteZ(m_webindexer);
m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater);
if (!m_webindexer || !m_webindexer->index()) {
m_db.close();
m_db.close();
addIdxReason("indexer", "Web index creation failed. See log");
return false;
}
@ -203,10 +203,10 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
// Get rid of all database entries that don't exist in the
// filesystem anymore. Only if all *configured* indexers ran.
if (m_updater && !m_updater->update(DbIxStatus::DBIXS_PURGE, "")) {
m_db.close();
m_db.close();
addIdxReason("indexer", "Index purge failed. See log");
return false;
}
return false;
}
m_db.purge();
}
@ -214,22 +214,22 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
// here. Makes no sense to check for cancel, we'll have to close
// anyway
if (m_updater)
m_updater->update(DbIxStatus::DBIXS_CLOSING, string());
m_updater->update(DbIxStatus::DBIXS_CLOSING, string());
if (!m_db.close()) {
LOGERR("ConfIndexer::index: error closing database in " <<
LOGERR("ConfIndexer::index: error closing database in " <<
m_config->getDbDir() << "\n");
addIdxReason("indexer", "Index close/flush failed. See log");
return false;
return false;
}
if (m_updater && !m_updater->update(DbIxStatus::DBIXS_CLOSING, string()))
return false;
return false;
bool ret = true;
if (!createStemmingDatabases()) {
ret = false;
ret = false;
}
if (m_updater && !m_updater->update(DbIxStatus::DBIXS_CLOSING, string()))
return false;
return false;
// Don't fail indexing because of an aspell issue: we ignore the status.
// Messages were written to the reasons output
@ -237,7 +237,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
clearMimeHandlerCache();
if (m_updater)
m_updater->update(DbIxStatus::DBIXS_DONE, string());
m_updater->update(DbIxStatus::DBIXS_DONE, string());
return ret;
}
@ -246,14 +246,14 @@ bool ConfIndexer::indexFiles(list<string>& ifiles, int flag)
list<string> myfiles;
string origcwd = m_config->getOrigCwd();
for (const auto& entry : ifiles) {
myfiles.push_back(path_canon(entry, &origcwd));
myfiles.push_back(path_canon(entry, &origcwd));
}
myfiles.sort();
if (!m_db.open(Rcl::Db::DbUpd)) {
LOGERR("ConfIndexer: indexFiles error opening database " <<
LOGERR("ConfIndexer: indexFiles error opening database " <<
m_config->getDbDir() << "\n");
return false;
return false;
}
m_config->setKeyDir(cstr_null);
bool ret = false;
@ -280,9 +280,9 @@ bool ConfIndexer::indexFiles(list<string>& ifiles, int flag)
}
// The close would be done in our destructor, but we want status here
if (!m_db.close()) {
LOGERR("ConfIndexer::index: error closing database in " <<
LOGERR("ConfIndexer::index: error closing database in " <<
m_config->getDbDir() << "\n");
return false;
return false;
}
ifiles = myfiles;
clearMimeHandlerCache();
@ -297,7 +297,7 @@ bool ConfIndexer::updateDocs(vector<Rcl::Doc> &docs, IxFlag flag)
docsToPaths(docs, paths);
list<string> files(paths.begin(), paths.end());
if (!files.empty()) {
return indexFiles(files, flag);
return indexFiles(files, flag);
}
return true;
}
@ -307,14 +307,14 @@ bool ConfIndexer::purgeFiles(list<string> &files, int flag)
list<string> myfiles;
string origcwd = m_config->getOrigCwd();
for (const auto& entry : files) {
myfiles.push_back(path_canon(entry, &origcwd));
myfiles.push_back(path_canon(entry, &origcwd));
}
myfiles.sort();
if (!m_db.open(Rcl::Db::DbUpd)) {
LOGERR("ConfIndexer: purgeFiles error opening database " <<
LOGERR("ConfIndexer: purgeFiles error opening database " <<
m_config->getDbDir() << "\n");
return false;
return false;
}
bool ret = false;
m_config->setKeyDir(cstr_null);
@ -337,9 +337,9 @@ bool ConfIndexer::purgeFiles(list<string> &files, int flag)
// The close would be done in our destructor, but we want status here
if (!m_db.close()) {
LOGERR("ConfIndexer::purgefiles: error closing database in " <<
LOGERR("ConfIndexer::purgefiles: error closing database in " <<
m_config->getDbDir() << "\n");
return false;
return false;
}
return ret;
}
@ -356,18 +356,18 @@ bool ConfIndexer::createStemmingDatabases()
addIdxReason("stemming", "could not open db");
return false;
}
vector<string> langs;
stringToStrings(slangs, langs);
vector<string> langs;
stringToStrings(slangs, langs);
// Get the list of existing stem dbs from the database (some may have
// been manually created, we just keep those from the config
vector<string> dblangs = m_db.getStemLangs();
vector<string>::const_iterator it;
for (it = dblangs.begin(); it != dblangs.end(); it++) {
if (find(langs.begin(), langs.end(), *it) == langs.end())
m_db.deleteStemDb(*it);
}
ret = ret && m_db.createStemDbs(langs);
// Get the list of existing stem dbs from the database (some may have
// been manually created, we just keep those from the config
vector<string> dblangs = m_db.getStemLangs();
vector<string>::const_iterator it;
for (it = dblangs.begin(); it != dblangs.end(); it++) {
if (find(langs.begin(), langs.end(), *it) == langs.end())
m_db.deleteStemDb(*it);
}
ret = ret && m_db.createStemDbs(langs);
if (!ret) {
addIdxReason("stemming", "stem db creation failed");
}
@ -379,7 +379,7 @@ bool ConfIndexer::createStemmingDatabases()
bool ConfIndexer::createStemDb(const string &lang)
{
if (!m_db.open(Rcl::Db::DbUpd))
return false;
return false;
vector<string> langs;
stringToStrings(lang, langs);
return m_db.createStemDbs(langs);
@ -397,32 +397,32 @@ bool ConfIndexer::createAspellDict()
// it forever.
static int noaspell = -12345;
if (noaspell == -12345) {
noaspell = false;
m_config->getConfParam("noaspell", &noaspell);
noaspell = false;
m_config->getConfParam("noaspell", &noaspell);
}
if (noaspell)
return true;
return true;
if (!m_db.open(Rcl::Db::DbRO)) {
LOGERR("ConfIndexer::createAspellDict: could not open db\n");
return false;
return false;
}
Aspell aspell(m_config);
string reason;
if (!aspell.init(reason)) {
LOGERR("ConfIndexer::createAspellDict: aspell init failed: " <<
LOGERR("ConfIndexer::createAspellDict: aspell init failed: " <<
reason << "\n");
noaspell = true;
return false;
noaspell = true;
return false;
}
LOGDEB("ConfIndexer::createAspellDict: creating dictionary\n");
if (!aspell.buildDict(m_db, reason)) {
LOGERR("ConfIndexer::createAspellDict: aspell buildDict failed: " <<
LOGERR("ConfIndexer::createAspellDict: aspell buildDict failed: " <<
reason << "\n");
addIdxReason("aspell", reason);
noaspell = true;
return false;
noaspell = true;
return false;
}
#endif
return true;
@ -432,4 +432,3 @@ vector<string> ConfIndexer::getStemmerNames()
{
return Rcl::Db::getStemmerNames();
}

18
src/index/indexer.h
View File

@ -34,7 +34,7 @@ class WebQueueIndexer;
/** Callback to say what we're doing. If the update func returns false, we
* stop as soon as possible without corrupting state */
class DbIxStatusUpdater {
public:
public:
#ifdef IDX_THREADS
std::mutex m_mutex;
#endif
@ -43,14 +43,14 @@ class DbIxStatusUpdater {
// Convenience: change phase/fn and update
virtual bool update(DbIxStatus::Phase phase, const string& fn)
{
{
#ifdef IDX_THREADS
std::unique_lock<std::mutex> lock(m_mutex);
std::unique_lock<std::mutex> lock(m_mutex);
#endif
status.phase = phase;
status.fn = fn;
return update();
}
status.phase = phase;
status.fn = fn;
return update();
}
// To be implemented by user for sending info somewhere
virtual bool update() = 0;
@ -62,7 +62,7 @@ class DbIxStatusUpdater {
* database(s).
*/
class ConfIndexer {
public:
public:
enum runStatus {IndexerOk, IndexerError};
ConfIndexer(RclConfig *cnf, DbIxStatusUpdater *updfunc = 0);
virtual ~ConfIndexer();
@ -113,7 +113,7 @@ class ConfIndexer {
/** Set in place reset mode */
void setInPlaceReset() {m_db.setInPlaceReset();}
private:
private:
RclConfig *m_config;
Rcl::Db m_db;
FsIndexer *m_fsindexer;

146
src/index/mimetype.cpp
View File

@ -55,11 +55,11 @@ static string mimetypefromdata(RclConfig *cfg, const string &fn, bool usfc)
#ifdef USE_SYSTEM_FILE_COMMAND
if (usfc && mime.empty()) {
// Last resort: use "file -i", or its configured replacement.
// Last resort: use "file -i", or its configured replacement.
// 'file' fallback if the configured command (default:
// xdg-mime) is not found
static const vector<string> tradfilecmd = {{FILE_PROG}, {"-i"}};
static const vector<string> tradfilecmd = {{FILE_PROG}, {"-i"}};
vector<string> cmd;
string scommand;
@ -81,55 +81,55 @@ static string mimetypefromdata(RclConfig *cfg, const string &fn, bool usfc)
cmd = tradfilecmd;
}
string result;
string result;
LOGDEB2("mimetype: executing: [" << stringsToString(cmd) << "]\n");
if (!ExecCmd::backtick(cmd, result)) {
LOGERR("mimetypefromdata: exec " <<
if (!ExecCmd::backtick(cmd, result)) {
LOGERR("mimetypefromdata: exec " <<
stringsToString(cmd) << " failed\n");
return string();
}
trimstring(result, " \t\n\r");
LOGDEB2("mimetype: systemfilecommand output [" << result << "]\n");
return string();
}
trimstring(result, " \t\n\r");
LOGDEB2("mimetype: systemfilecommand output [" << result << "]\n");
// The normal output from "file -i" looks like the following:
// thefilename.xxx: text/plain; charset=us-ascii
// Sometimes the semi-colon is missing like in:
// mimetype.cpp: text/x-c charset=us-ascii
// And sometimes we only get the mime type. This apparently happens
// when 'file' believes that the file name is binary
// The normal output from "file -i" looks like the following:
// thefilename.xxx: text/plain; charset=us-ascii
// Sometimes the semi-colon is missing like in:
// mimetype.cpp: text/x-c charset=us-ascii
// And sometimes we only get the mime type. This apparently happens
// when 'file' believes that the file name is binary
// xdg-mime only outputs the MIME type.
// If there is no colon and there is a slash, this is hopefuly
// the mime type
if (result.find_first_of(":") == string::npos &&
result.find_first_of("/") != string::npos) {
return result;
}
// If there is no colon and there is a slash, this is hopefuly
// the mime type
if (result.find_first_of(":") == string::npos &&
result.find_first_of("/") != string::npos) {
return result;
}
// Else the result should begin with the file name. Get rid of it:
if (result.find(fn) != 0) {
// Garbage "file" output. Maybe the result of a charset
// conversion attempt?
LOGERR("mimetype: can't interpret output from [" <<
// Else the result should begin with the file name. Get rid of it:
if (result.find(fn) != 0) {
// Garbage "file" output. Maybe the result of a charset
// conversion attempt?
LOGERR("mimetype: can't interpret output from [" <<
stringsToString(cmd) << "] : [" << result << "]\n");
return string();
}
result = result.substr(fn.size());
return string();
}
result = result.substr(fn.size());
// Now should look like ": text/plain; charset=us-ascii"
// Split it, and take second field
list<string> res;
stringToStrings(result, res);
if (res.size() <= 1)
return string();
list<string>::iterator it = res.begin();
mime = *++it;
// Remove possible semi-colon at the end
trimstring(mime, " \t;");
// Now should look like ": text/plain; charset=us-ascii"
// Split it, and take second field
list<string> res;
stringToStrings(result, res);
if (res.size() <= 1)
return string();
list<string>::iterator it = res.begin();
mime = *++it;
// Remove possible semi-colon at the end
trimstring(mime, " \t;");
// File -i will sometimes return strange stuff (ie: "very small file")
if(mime.find("/") == string::npos)
mime.clear();
// File -i will sometimes return strange stuff (ie: "very small file")
if(mime.find("/") == string::npos)
mime.clear();
}
#endif //USE_SYSTEM_FILE_COMMAND
@ -139,24 +139,24 @@ static string mimetypefromdata(RclConfig *cfg, const string &fn, bool usfc)
/// Guess mime type, first from suffix, then from file data. We also
/// have a list of suffixes that we don't touch at all.
string mimetype(const string &fn, const struct PathStat *stp,
RclConfig *cfg, bool usfc)
RclConfig *cfg, bool usfc)
{
// Use stat data if available to check for non regular files
if (stp) {
// Note: the value used for directories is different from what
// file -i would print on Linux (inode/directory). Probably
// comes from bsd. Thos may surprise a user trying to use a
// 'mime:' filter with the query language, but it's not work
// changing (would force a reindex).
if (stp->pst_type == PathStat::PST_DIR)
return "inode/directory";
if (stp->pst_type == PathStat::PST_SYMLINK)
return "inode/symlink";
if (stp->pst_type != PathStat::PST_REGULAR)
return "inode/x-fsspecial";
// Empty files are just this: avoid further errors with actual filters.
if (stp->pst_size == 0)
return "inode/x-empty";
// Note: the value used for directories is different from what
// file -i would print on Linux (inode/directory). Probably
// comes from bsd. Thos may surprise a user trying to use a
// 'mime:' filter with the query language, but it's not work
// changing (would force a reindex).
if (stp->pst_type == PathStat::PST_DIR)
return "inode/directory";
if (stp->pst_type == PathStat::PST_SYMLINK)
return "inode/symlink";
if (stp->pst_type != PathStat::PST_REGULAR)
return "inode/x-fsspecial";
// Empty files are just this: avoid further errors with actual filters.
if (stp->pst_size == 0)
return "inode/x-empty";
}
string mtype;
@ -165,40 +165,40 @@ string mimetype(const string &fn, const struct PathStat *stp,
// Extended attribute has priority on everything, as per:
// http://freedesktop.org/wiki/CommonExtendedAttributes
if (pxattr::get(fn, "mime_type", &mtype)) {
LOGDEB0("Mimetype: 'mime_type' xattr : [" << mtype << "]\n");
if (mtype.empty()) {
LOGDEB0("Mimetype: getxattr() returned empty mime type !\n");
} else {
return mtype;
}
LOGDEB0("Mimetype: 'mime_type' xattr : [" << mtype << "]\n");
if (mtype.empty()) {
LOGDEB0("Mimetype: getxattr() returned empty mime type !\n");
} else {
return mtype;
}
}
#endif
if (cfg == 0) {
LOGERR("Mimetype: null config ??\n");
return mtype;
LOGERR("Mimetype: null config ??\n");
return mtype;
}
if (cfg->inStopSuffixes(fn)) {
LOGDEB("mimetype: fn [" << fn << "] in stopsuffixes\n");
return mtype;
LOGDEB("mimetype: fn [" << fn << "] in stopsuffixes\n");
return mtype;
}
// Compute file name suffix and search the mimetype map
string::size_type dot = fn.find_first_of(".");
while (dot != string::npos) {
string suff = stringtolower(fn.substr(dot));
mtype = cfg->getMimeTypeFromSuffix(suff);
if (!mtype.empty() || dot >= fn.size() - 1)
break;
dot = fn.find_first_of(".", dot + 1);
string suff = stringtolower(fn.substr(dot));
mtype = cfg->getMimeTypeFromSuffix(suff);
if (!mtype.empty() || dot >= fn.size() - 1)
break;
dot = fn.find_first_of(".", dot + 1);
}
// If type was not determined from suffix, examine file data. Can
// only do this if we have an actual file (as opposed to a pure
// name).
if (mtype.empty() && stp)
mtype = mimetypefromdata(cfg, fn, usfc);
mtype = mimetypefromdata(cfg, fn, usfc);
return mtype;
}

12
src/index/rclmon.h
View File

@ -45,9 +45,9 @@ using std::multimap;
* Monitoring event: something changed in the filesystem
*/
class RclMonEvent {
public:
public:
enum EvType {RCLEVT_NONE= 0, RCLEVT_MODIFY=1, RCLEVT_DELETE=2,
RCLEVT_DIRCREATE=3, RCLEVT_ISDIR=0x10};
RCLEVT_DIRCREATE=3, RCLEVT_ISDIR=0x10};
string m_path;
// Type and flags
int m_etyp;
@ -61,13 +61,13 @@ class RclMonEvent {
bool m_needidx;
RclMonEvent() : m_etyp(RCLEVT_NONE),
m_itvsecs(0), m_minclock(0), m_needidx(false) {}
m_itvsecs(0), m_minclock(0), m_needidx(false) {}
EvType evtype() {return EvType(m_etyp & 0xf);}
int evflags() {return m_etyp & 0xf0;}
};
enum RclMonitorOption {RCLMON_NONE=0, RCLMON_NOFORK=1, RCLMON_NOX11=2,
RCLMON_NOCONFCHECK=4};
RCLMON_NOCONFCHECK=4};
/**
* Monitoring event queue. This is the shared object between the main thread
@ -76,7 +76,7 @@ enum RclMonitorOption {RCLMON_NONE=0, RCLMON_NOFORK=1, RCLMON_NOX11=2,
*/
class RclEQData;
class RclMonEventQueue {
public:
public:
RclMonEventQueue();
~RclMonEventQueue();
/** Wait for event or timeout. Returns with the queue locked */
@ -94,7 +94,7 @@ class RclMonEventQueue {
void setConfig(RclConfig *conf);
RclConfig *getConfig();
private:
private:
RclEQData *m_data;
};

278
src/index/rclmonprc.cpp
View File

@ -142,50 +142,50 @@ public:
std::condition_variable m_cond;
RclEQData()
: m_config(0), m_ok(true)
{
}
: m_config(0), m_ok(true)
{
}
void readDelayPats(int dfltsecs);
DelayPat searchDelayPats(const string& path)
{
for (vector<DelayPat>::iterator it = m_delaypats.begin();
it != m_delaypats.end(); it++) {
if (fnmatch(it->pattern.c_str(), path.c_str(), 0) == 0) {
return *it;
{
for (vector<DelayPat>::iterator it = m_delaypats.begin();
it != m_delaypats.end(); it++) {
if (fnmatch(it->pattern.c_str(), path.c_str(), 0) == 0) {
return *it;
}
}
return DelayPat();
}
}
return DelayPat();
}
void delayInsert(const queue_type::iterator &qit);
};
void RclEQData::readDelayPats(int dfltsecs)
{
if (m_config == 0)
return;
return;
string patstring;
if (!m_config->getConfParam("mondelaypatterns", patstring) ||
patstring.empty())
return;
patstring.empty())
return;
vector<string> dplist;
if (!stringToStrings(patstring, dplist)) {
LOGERR("rclEQData: bad pattern list: [" << (patstring) << "]\n" );
return;
LOGERR("rclEQData: bad pattern list: [" << (patstring) << "]\n" );
return;
}
for (vector<string>::iterator it = dplist.begin();
it != dplist.end(); it++) {
string::size_type pos = it->find_last_of(":");
DelayPat dp;
dp.pattern = it->substr(0, pos);
if (pos != string::npos && pos != it->size()-1) {
dp.seconds = atoi(it->substr(pos+1).c_str());
} else {
dp.seconds = dfltsecs;
}
m_delaypats.push_back(dp);
LOGDEB2("rclmon::readDelayPats: add [" << (dp.pattern) << "] " << (dp.seconds) << "\n" );
it != dplist.end(); it++) {
string::size_type pos = it->find_last_of(":");
DelayPat dp;
dp.pattern = it->substr(0, pos);
if (pos != string::npos && pos != it->size()-1) {
dp.seconds = atoi(it->substr(pos+1).c_str());
} else {
dp.seconds = dfltsecs;
}
m_delaypats.push_back(dp);
LOGDEB2("rclmon::readDelayPats: add [" << (dp.pattern) << "] " << (dp.seconds) << "\n" );
}
}
@ -197,12 +197,12 @@ void RclEQData::delayInsert(const queue_type::iterator &qit)
MONDEB("RclEQData::delayInsert: minclock " << qit->second.m_minclock <<
std::endl);
for (delays_type::iterator dit = m_delays.begin();
dit != m_delays.end(); dit++) {
queue_type::iterator qit1 = *dit;
if ((*qit1).second.m_minclock > qit->second.m_minclock) {
m_delays.insert(dit, qit);
return;
}
dit != m_delays.end(); dit++) {
queue_type::iterator qit1 = *dit;
if ((*qit1).second.m_minclock > qit->second.m_minclock) {
m_delays.insert(dit, qit);
return;
}
}
m_delays.push_back(qit);
}
@ -220,7 +220,7 @@ RclMonEventQueue::~RclMonEventQueue()
void RclMonEventQueue::setopts(int opts)
{
if (m_data)
m_data->m_opts = opts;
m_data->m_opts = opts;
}
/** Wait until there is something to process on the queue, or timeout.
@ -232,22 +232,22 @@ std::unique_lock<std::mutex> RclMonEventQueue::wait(int seconds, bool *top)
MONDEB("RclMonEventQueue::wait, seconds: " << seconds << std::endl);
if (!empty()) {
MONDEB("RclMonEventQueue:: immediate return\n");
return lock;
MONDEB("RclMonEventQueue:: immediate return\n");
return lock;
}
int err;
if (seconds > 0) {
if (top)
*top = false;
if (m_data->m_cond.wait_for(lock, std::chrono::seconds(seconds)) ==
if (top)
*top = false;
if (m_data->m_cond.wait_for(lock, std::chrono::seconds(seconds)) ==
std::cv_status::timeout) {
*top = true;
MONDEB("RclMonEventQueue:: timeout\n");
return lock;
}
} else {
m_data->m_cond.wait(lock);
m_data->m_cond.wait(lock);
}
MONDEB("RclMonEventQueue:: non-timeout return\n");
return lock;
@ -269,16 +269,16 @@ RclConfig *RclMonEventQueue::getConfig()
bool RclMonEventQueue::ok()
{
if (m_data == 0) {
LOGINFO("RclMonEventQueue: not ok: bad state\n" );
return false;
LOGINFO("RclMonEventQueue: not ok: bad state\n" );
return false;
}
if (stopindexing) {
LOGINFO("RclMonEventQueue: not ok: stop request\n" );
return false;
LOGINFO("RclMonEventQueue: not ok: stop request\n" );
return false;
}
if (!m_data->m_ok) {
LOGINFO("RclMonEventQueue: not ok: queue terminated\n" );
return false;
LOGINFO("RclMonEventQueue: not ok: queue terminated\n" );
return false;
}
return true;
}
@ -295,24 +295,24 @@ void RclMonEventQueue::setTerminate()
bool RclMonEventQueue::empty()
{
if (m_data == 0) {
MONDEB("RclMonEventQueue::empty(): true (m_data==0)\n");
return true;
MONDEB("RclMonEventQueue::empty(): true (m_data==0)\n");
return true;
}
if (!m_data->m_iqueue.empty()) {
MONDEB("RclMonEventQueue::empty(): false (m_iqueue not empty)\n");
return true;
MONDEB("RclMonEventQueue::empty(): false (m_iqueue not empty)\n");
return true;
}
if (m_data->m_dqueue.empty()) {
MONDEB("RclMonEventQueue::empty(): true (m_Xqueue both empty)\n");
return true;
MONDEB("RclMonEventQueue::empty(): true (m_Xqueue both empty)\n");
return true;
}
// Only dqueue has events. Have to check the delays (only the
// first, earliest one):
queue_type::iterator qit = *(m_data->m_delays.begin());
if (qit->second.m_minclock > time(0)) {
MONDEB("RclMonEventQueue::empty(): true (no delay ready " <<
MONDEB("RclMonEventQueue::empty(): true (no delay ready " <<
qit->second.m_minclock << ")\n");
return true;
return true;
}
MONDEB("RclMonEventQueue::empty(): returning false (delay expired)\n");
return false;
@ -329,36 +329,36 @@ RclMonEvent RclMonEventQueue::pop()
// Look at the delayed events, get rid of the expired/unactive
// ones, possibly return an expired/needidx one.
while (!m_data->m_delays.empty()) {
delays_type::iterator dit = m_data->m_delays.begin();
queue_type::iterator qit = *dit;
MONDEB("RclMonEventQueue::pop(): in delays: evt minclock " <<
qit->second.m_minclock << std::endl);
if (qit->second.m_minclock <= now) {
if (qit->second.m_needidx) {
RclMonEvent ev = qit->second;
qit->second.m_minclock = time(0) + qit->second.m_itvsecs;
qit->second.m_needidx = false;
m_data->m_delays.erase(dit);
m_data->delayInsert(qit);
return ev;
delays_type::iterator dit = m_data->m_delays.begin();
queue_type::iterator qit = *dit;
MONDEB("RclMonEventQueue::pop(): in delays: evt minclock " <<
qit->second.m_minclock << std::endl);
if (qit->second.m_minclock <= now) {
if (qit->second.m_needidx) {
RclMonEvent ev = qit->second;
qit->second.m_minclock = time(0) + qit->second.m_itvsecs;
qit->second.m_needidx = false;
m_data->m_delays.erase(dit);
m_data->delayInsert(qit);
return ev;
} else {
// Delay elapsed without new update, get rid of event.
m_data->m_dqueue.erase(qit);
m_data->m_delays.erase(dit);
}
} else {
// Delay elapsed without new update, get rid of event.
m_data->m_dqueue.erase(qit);
m_data->m_delays.erase(dit);
// This and following events are for later processing, we
// are done with the delayed event list.
break;
}
} else {
// This and following events are for later processing, we
// are done with the delayed event list.
break;
}
}
// Look for non-delayed event
if (!m_data->m_iqueue.empty()) {
queue_type::iterator qit = m_data->m_iqueue.begin();
RclMonEvent ev = qit->second;
m_data->m_iqueue.erase(qit);
return ev;
queue_type::iterator qit = m_data->m_iqueue.begin();
RclMonEvent ev = qit->second;
m_data->m_iqueue.erase(qit);
return ev;
}
return RclMonEvent();
@ -376,32 +376,32 @@ bool RclMonEventQueue::pushEvent(const RclMonEvent &ev)
DelayPat pat = m_data->searchDelayPats(ev.m_path);
if (pat.seconds != 0) {
// Using delayed reindex queue. Need to take care of minclock and also
// insert into the in-minclock-order list
queue_type::iterator qit = m_data->m_dqueue.find(ev.m_path);
if (qit == m_data->m_dqueue.end()) {
// Not there yet, insert new
qit =
m_data->m_dqueue.insert(queue_type::value_type(ev.m_path, ev)).first;
// Set the time to next index to "now" as it has not been
// indexed recently (otherwise it would still be in the
// queue), and add the iterator to the delay queue.
qit->second.m_minclock = time(0);
qit->second.m_needidx = true;
qit->second.m_itvsecs = pat.seconds;
m_data->delayInsert(qit);
// Using delayed reindex queue. Need to take care of minclock and also
// insert into the in-minclock-order list
queue_type::iterator qit = m_data->m_dqueue.find(ev.m_path);
if (qit == m_data->m_dqueue.end()) {
// Not there yet, insert new
qit =
m_data->m_dqueue.insert(queue_type::value_type(ev.m_path, ev)).first;
// Set the time to next index to "now" as it has not been
// indexed recently (otherwise it would still be in the
// queue), and add the iterator to the delay queue.
qit->second.m_minclock = time(0);
qit->second.m_needidx = true;
qit->second.m_itvsecs = pat.seconds;
m_data->delayInsert(qit);
} else {
// Already in queue. Possibly update type but save minclock
// (so no need to touch m_delays). Flag as needing indexing
time_t saved_clock = qit->second.m_minclock;
qit->second = ev;
qit->second.m_minclock = saved_clock;
qit->second.m_needidx = true;
}
} else {
// Already in queue. Possibly update type but save minclock
// (so no need to touch m_delays). Flag as needing indexing
time_t saved_clock = qit->second.m_minclock;
qit->second = ev;
qit->second.m_minclock = saved_clock;
qit->second.m_needidx = true;
}
} else {
// Immediate event: just insert it, erasing any previously
// existing entry
m_data->m_iqueue[ev.m_path] = ev;
// Immediate event: just insert it, erasing any previously
// existing entry
m_data->m_iqueue[ev.m_path] = ev;
}
m_data->m_cond.notify_all();
@ -429,19 +429,19 @@ static bool expeditedIndexingRequested(RclConfig *conf)
{
static vector<string> rqfiles;
if (rqfiles.empty()) {
rqfiles.push_back(path_cat(conf->getConfDir(), "rclmonixnow"));
const char *cp;
if ((cp = getenv("RECOLL_CONFTOP"))) {
rqfiles.push_back(path_cat(cp, "rclmonixnow"));
}
if ((cp = getenv("RECOLL_CONFMID"))) {
rqfiles.push_back(path_cat(cp, "rclmonixnow"));
}
rqfiles.push_back(path_cat(conf->getConfDir(), "rclmonixnow"));
const char *cp;
if ((cp = getenv("RECOLL_CONFTOP"))) {
rqfiles.push_back(path_cat(cp, "rclmonixnow"));
}
if ((cp = getenv("RECOLL_CONFMID"))) {
rqfiles.push_back(path_cat(cp, "rclmonixnow"));
}
}
bool found = false;
for (vector<string>::const_iterator it = rqfiles.begin();
it != rqfiles.end(); it++) {
found = found || checkfileanddelete(*it);
it != rqfiles.end(); it++) {
found = found || checkfileanddelete(*it);
}
return found;
}
@ -449,9 +449,9 @@ static bool expeditedIndexingRequested(RclConfig *conf)
bool startMonitor(RclConfig *conf, int opts)
{
if (!conf->getConfParam("monauxinterval", &auxinterval))
auxinterval = dfltauxinterval;
auxinterval = dfltauxinterval;
if (!conf->getConfParam("monixinterval", &ixinterval))
ixinterval = dfltixinterval;
ixinterval = dfltixinterval;
rclEQ.setConfig(conf);
rclEQ.setopts(opts);
@ -533,14 +533,14 @@ bool startMonitor(RclConfig *conf, int opts)
}
now = time(0);
// Process. We don't do this every time but let the lists accumulate
// Process. We don't do this every time but let the lists accumulate
// a little, this saves processing. Start at once if list is big.
if (expeditedIndexingRequested(conf) ||
(now - lastixtime > ixinterval) ||
(deleted.size() + modified.size() > 20)) {
(now - lastixtime > ixinterval) ||
(deleted.size() + modified.size() > 20)) {
lastixtime = now;
// Used to do the modified list first, but it does seem
// smarter to make room first...
// Used to do the modified list first, but it does seem
// smarter to make room first...
if (!deleted.empty()) {
deleted.sort();
deleted.unique();
@ -559,28 +559,28 @@ bool startMonitor(RclConfig *conf, int opts)
}
}
// Recreate the auxiliary dbs every hour at most.
// Recreate the auxiliary dbs every hour at most.
now = time(0);
if (didsomething && now - lastauxtime > auxinterval) {
lastauxtime = now;
didsomething = false;
if (!createAuxDbs(conf)) {
// We used to bail out on error here. Not anymore,
// because this is most of the time due to a failure
// of aspell dictionary generation, which is not
// critical.
if (didsomething && now - lastauxtime > auxinterval) {
lastauxtime = now;
didsomething = false;
if (!createAuxDbs(conf)) {
// We used to bail out on error here. Not anymore,
// because this is most of the time due to a failure
// of aspell dictionary generation, which is not
// critical.
}
}
}
// Check for a config change
if (!(opts & RCLMON_NOCONFCHECK) && o_reexec && conf->sourceChanged()) {
LOGDEB("Rclmonprc: config changed, reexecuting myself\n" );
// We never want to have a -n option after a config
// change. -n was added by the reexec after the initial
// pass even if it was not given on the command line
o_reexec->removeArg("-n");
o_reexec->reexec();
}
// Check for a config change
if (!(opts & RCLMON_NOCONFCHECK) && o_reexec && conf->sourceChanged()) {
LOGDEB("Rclmonprc: config changed, reexecuting myself\n" );
// We never want to have a -n option after a config
// change. -n was added by the reexec after the initial
// pass even if it was not given on the command line
o_reexec->removeArg("-n");
o_reexec->reexec();
}
}
LOGDEB("Rclmonprc: calling queue setTerminate\n" );
rclEQ.setTerminate();

246
src/index/recollindex.cpp
View File

@ -120,51 +120,51 @@ public:
}
virtual bool update()
{
// Update the status file. Avoid doing it too often. Always do
// it at the end (status DONE)
if (status.phase == DbIxStatus::DBIXS_DONE ||
status.phase != m_prevphase || m_chron.millis() > 300) {
if (status.totfiles < status.filesdone ||
status.phase == DbIxStatus::DBIXS_DONE) {
status.totfiles = status.filesdone;
{
// Update the status file. Avoid doing it too often. Always do
// it at the end (status DONE)
if (status.phase == DbIxStatus::DBIXS_DONE ||
status.phase != m_prevphase || m_chron.millis() > 300) {
if (status.totfiles < status.filesdone ||
status.phase == DbIxStatus::DBIXS_DONE) {
status.totfiles = status.filesdone;
}
m_prevphase = status.phase;
m_chron.restart();
m_file.holdWrites(true);
m_file.set("phase", int(status.phase));
m_file.set("docsdone", status.docsdone);
m_file.set("filesdone", status.filesdone);
m_file.set("fileerrors", status.fileerrors);
m_file.set("dbtotdocs", status.dbtotdocs);
m_file.set("totfiles", status.totfiles);
m_file.set("fn", status.fn);
m_file.set("hasmonitor", status.hasmonitor);
m_file.holdWrites(false);
}
if (path_exists(m_stopfilename)) {
LOGINF("recollindex: asking indexer to stop because " <<
m_stopfilename << " exists\n");
path_unlink(m_stopfilename);
stopindexing = true;
}
if (stopindexing) {
return false;
}
m_prevphase = status.phase;
m_chron.restart();
m_file.holdWrites(true);
m_file.set("phase", int(status.phase));
m_file.set("docsdone", status.docsdone);
m_file.set("filesdone", status.filesdone);
m_file.set("fileerrors", status.fileerrors);
m_file.set("dbtotdocs", status.dbtotdocs);
m_file.set("totfiles", status.totfiles);
m_file.set("fn", status.fn);
m_file.set("hasmonitor", status.hasmonitor);
m_file.holdWrites(false);
}
if (path_exists(m_stopfilename)) {
LOGINF("recollindex: asking indexer to stop because " <<
m_stopfilename << " exists\n");
path_unlink(m_stopfilename);
stopindexing = true;
}
if (stopindexing) {
return false;
}
#ifndef DISABLE_X11MON
// If we are in the monitor, we also need to check X11 status
// during the initial indexing pass (else the user could log
// out and the indexing would go on, not good (ie: if the user
// logs in again, the new recollindex will fail).
if ((op_flags & OPT_m) && !(op_flags & OPT_x) && !x11IsAlive()) {
LOGDEB("X11 session went away during initial indexing pass\n");
stopindexing = true;
return false;
}
// If we are in the monitor, we also need to check X11 status
// during the initial indexing pass (else the user could log
// out and the indexing would go on, not good (ie: if the user
// logs in again, the new recollindex will fail).
if ((op_flags & OPT_m) && !(op_flags & OPT_x) && !x11IsAlive()) {
LOGDEB("X11 session went away during initial indexing pass\n");
stopindexing = true;
return false;
}
#endif
return true;
}
return true;
}
private:
ConfSimple m_file;
@ -263,8 +263,8 @@ class MakeListWalkerCB : public FsTreeWalkerCB {
public:
MakeListWalkerCB(list<string>& files, const vector<string>& selpats)
: m_files(files), m_pats(selpats)
{
}
{
}
virtual FsTreeWalker::Status
processone(const string& fn, const struct PathStat *,
FsTreeWalker::CbFlag flg) {
@ -427,103 +427,103 @@ static bool checktopdirs(RclConfig *config, vector<string>& nonexist)
string thisprog;
static const char usage [] =
"\n"
"recollindex [-h] \n"
" Print help\n"
"recollindex [-z|-Z] [-k]\n"
" Index everything according to configuration file\n"
" -z : reset database before starting indexing\n"
" -Z : in place reset: consider all documents as changed. Can also\n"
" be combined with -i or -r but not -m\n"
" -k : retry files on which we previously failed\n"
"\n"
"recollindex [-h] \n"
" Print help\n"
"recollindex [-z|-Z] [-k]\n"
" Index everything according to configuration file\n"
" -z : reset database before starting indexing\n"
" -Z : in place reset: consider all documents as changed. Can also\n"
" be combined with -i or -r but not -m\n"
" -k : retry files on which we previously failed\n"
#ifdef RCL_MONITOR
"recollindex -m [-w <secs>] -x [-D] [-C]\n"
" Perform real time indexing. Don't become a daemon if -D is set.\n"
" -w sets number of seconds to wait before starting.\n"
" -C disables monitoring config for changes/reexecuting.\n"
" -n disables initial incremental indexing (!and purge!).\n"
"recollindex -m [-w <secs>] -x [-D] [-C]\n"
" Perform real time indexing. Don't become a daemon if -D is set.\n"
" -w sets number of seconds to wait before starting.\n"
" -C disables monitoring config for changes/reexecuting.\n"
" -n disables initial incremental indexing (!and purge!).\n"
#ifndef DISABLE_X11MON
" -x disables exit on end of x11 session\n"
" -x disables exit on end of x11 session\n"
#endif /* DISABLE_X11MON */
#endif /* RCL_MONITOR */
"recollindex -e [<filepath [path ...]>]\n"
" Purge data for individual files. No stem database updates.\n"
" Reads paths on stdin if none is given as argument.\n"
"recollindex -i [-f] [-Z] [<filepath [path ...]>]\n"
" Index individual files. No database purge or stem database updates\n"
" Will read paths on stdin if none is given as argument\n"
" -f : ignore skippedPaths and skippedNames while doing this\n"
"recollindex -r [-K] [-f] [-Z] [-p pattern] <top> \n"
" Recursive partial reindex. \n"
" -p : filter file names, multiple instances are allowed, e.g.: \n"
" -p *.odt -p *.pdf\n"
" -K : skip previously failed files (they are retried by default)\n"
"recollindex -l\n"
" List available stemming languages\n"
"recollindex -s <lang>\n"
" Build stem database for additional language <lang>\n"
"recollindex -E\n"
" Check configuration file for topdirs and other paths existence\n"
"recollindex -e [<filepath [path ...]>]\n"
" Purge data for individual files. No stem database updates.\n"
" Reads paths on stdin if none is given as argument.\n"
"recollindex -i [-f] [-Z] [<filepath [path ...]>]\n"
" Index individual files. No database purge or stem database updates\n"
" Will read paths on stdin if none is given as argument\n"
" -f : ignore skippedPaths and skippedNames while doing this\n"
"recollindex -r [-K] [-f] [-Z] [-p pattern] <top> \n"
" Recursive partial reindex. \n"
" -p : filter file names, multiple instances are allowed, e.g.: \n"
" -p *.odt -p *.pdf\n"
" -K : skip previously failed files (they are retried by default)\n"
"recollindex -l\n"
" List available stemming languages\n"
"recollindex -s <lang>\n"
" Build stem database for additional language <lang>\n"
"recollindex -E\n"
" Check configuration file for topdirs and other paths existence\n"
#ifdef FUTURE_IMPROVEMENT
"recollindex -W\n"
" Process the Web queue\n"
"recollindex -W\n"
" Process the Web queue\n"
#endif
#ifdef RCL_USE_ASPELL
"recollindex -S\n"
" Build aspell spelling dictionary.>\n"
"recollindex -S\n"
" Build aspell spelling dictionary.>\n"
#endif
"Common options:\n"
" -c <configdir> : specify config directory, overriding $RECOLL_CONFDIR\n"
;
"Common options:\n"
" -c <configdir> : specify config directory, overriding $RECOLL_CONFDIR\n"
;
static void Usage()
{
FILE *fp = (op_flags & OPT_h) ? stdout : stderr;
fprintf(fp, "%s: Usage: %s", path_getsimple(thisprog).c_str(), usage);
fprintf(fp, "Recoll version: %s\n", Rcl::version_string().c_str());
exit((op_flags & OPT_h)==0);
FILE *fp = (op_flags & OPT_h) ? stdout : stderr;
fprintf(fp, "%s: Usage: %s", path_getsimple(thisprog).c_str(), usage);
fprintf(fp, "Recoll version: %s\n", Rcl::version_string().c_str());
exit((op_flags & OPT_h)==0);
}
static RclConfig *config;
static void lockorexit(Pidfile *pidfile, RclConfig *config)
{
PRETEND_USE(config);
pid_t pid;
if ((pid = pidfile->open()) != 0) {
if (pid > 0) {
cerr << "Can't become exclusive indexer: " << pidfile->getreason()
<< ". Return (other pid?): " << pid << endl;
PRETEND_USE(config);
pid_t pid;
if ((pid = pidfile->open()) != 0) {
if (pid > 0) {
cerr << "Can't become exclusive indexer: " << pidfile->getreason()
<< ". Return (other pid?): " << pid << endl;
#ifndef _WIN32
// Have a look at the status file. If the other process is
// a monitor we can tell it to start an incremental pass
// by touching the configuration file
DbIxStatus status;
readIdxStatus(config, status);
if (status.hasmonitor) {
string cmd("touch ");
string path = path_cat(config->getConfDir(), "recoll.conf");
cmd += path;
int status;
if ((status = system(cmd.c_str()))) {
cerr << cmd << " failed with status " << status << endl;
} else {
cerr << "Monitoring indexer process was notified of "
"indexing request\n";
}
}
// Have a look at the status file. If the other process is
// a monitor we can tell it to start an incremental pass
// by touching the configuration file
DbIxStatus status;
readIdxStatus(config, status);
if (status.hasmonitor) {
string cmd("touch ");
string path = path_cat(config->getConfDir(), "recoll.conf");
cmd += path;
int status;
if ((status = system(cmd.c_str()))) {
cerr << cmd << " failed with status " << status << endl;
} else {
cerr << "Monitoring indexer process was notified of "
"indexing request\n";
}
}
#endif
} else {
cerr << "Can't become exclusive indexer: " << pidfile->getreason()
<< endl;
}
exit(1);
}
if (pidfile->write_pid() != 0) {
cerr << "Can't become exclusive indexer: " << pidfile->getreason() <<
endl;
exit(1);
}
} else {
cerr << "Can't become exclusive indexer: " << pidfile->getreason()
<< endl;
}
exit(1);
}
if (pidfile->write_pid() != 0) {
cerr << "Can't become exclusive indexer: " << pidfile->getreason() <<
endl;
exit(1);
}
}
static string reasonsfile;
@ -559,7 +559,7 @@ static void flushIdxReasons()
static vector<string> argstovector(int argc, wchar_t **argv)
#else
#define WARGTOSTRING(w) (w)
static vector<string> argstovector(int argc, char **argv)
static vector<string> argstovector(int argc, char **argv)
#endif
{
thisprog = path_absolute(WARGTOSTRING(argv[0]));
@ -599,7 +599,7 @@ static std::string orig_cwd;
#if USE_WMAIN
int wmain(int argc, wchar_t *argv[])
#else
int main(int argc, char *argv[])
int main(int argc, char *argv[])
#endif
{
// The reexec struct is used by the daemon to shed memory after

93
src/index/subtreelist.cpp
View File

@ -15,7 +15,6 @@
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef TEST_SUBTREELIST
#include "autoconfig.h"
#include <memory>
@ -28,13 +27,14 @@
#include "log.h"
bool subtreelist(RclConfig *config, const string& top,
vector<string>& paths)
vector<string>& paths)
{
LOGDEB("subtreelist: top: [" << (top) << "]\n" );
Rcl::Db rcldb(config);
if (!rcldb.open(Rcl::Db::DbRO)) {
LOGERR("subtreelist: can't open database in [" << (config->getDbDir()) << "]: " << (rcldb.getReason()) << "\n" );
return false;
LOGERR("subtreelist: can't open database in [" << config->getDbDir() <<
"]: " << rcldb.getReason() << "\n");
return false;
}
Rcl::SearchData *sd = new Rcl::SearchData(Rcl::SCLT_OR, cstr_null);
@ -47,85 +47,12 @@ bool subtreelist(RclConfig *config, const string& top,
int cnt = query.getResCnt();
for (int i = 0; i < cnt; i++) {
Rcl::Doc doc;
if (!query.getDoc(i, doc))
break;
string path = fileurltolocalpath(doc.url);
if (!path.empty())
paths.push_back(path);
Rcl::Doc doc;
if (!query.getDoc(i, doc))
break;
string path = fileurltolocalpath(doc.url);
if (!path.empty())
paths.push_back(path);
}
return true;
}
#else // TEST
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <iostream>
#include <vector>
#include <string>
using namespace std;
#include "subtreelist.h"
#include "rclconfig.h"
#include "rclinit.h"
static char *thisprog;
static char usage [] =
" <path> : list document paths in this tree\n"
;
static void
Usage(void)
{
cerr << thisprog << ": usage:" << endl << usage;
exit(1);
}
static int op_flags;
#define OPT_o 0x2
int main(int argc, char **argv)
{
string top;
thisprog = argv[0];
argc--; argv++;
while (argc > 0 && **argv == '-') {
(*argv)++;
if (!(**argv))
/* Cas du "adb - core" */
Usage();
while (**argv)
switch (*(*argv)++) {
default: Usage(); break;
}
argc--; argv++;
}
if (argc < 1)
Usage();
top = *argv++;argc--;
string reason;
RclConfig *config = recollinit(0, 0, 0, reason, 0);
if (!config || !config->ok()) {
fprintf(stderr, "Recoll init failed: %s\n", reason.c_str());
exit(1);
}
vector<string> paths;
if (!subtreelist(config, top, paths)) {
cerr << "subtreelist failed" << endl;
exit(1);
}
for (vector<string>::const_iterator it = paths.begin();
it != paths.end(); it++) {
cout << *it << endl;
}
exit(0);
}
#endif

2
src/index/subtreelist.h
View File

@ -28,6 +28,6 @@ class RclConfig;
// the real time indexer to purge entries when a top directory is
// renamed. This is really convoluted, I'd like a better way.
extern bool subtreelist(RclConfig *config, const string& top,
std::vector<std::string>& paths);
std::vector<std::string>& paths);
#endif /* _SUBTREELIST_H_INCLUDED_ */

4
src/index/webqueue.h
View File

@ -35,13 +35,13 @@ class CirCache;
class RclConfig;
class WebStore;
namespace Rcl {
class Db;
class Db;
}
class WebQueueIndexer : public FsTreeWalkerCB {
public:
WebQueueIndexer(RclConfig *cnf, Rcl::Db *db,
DbIxStatusUpdater *updfunc = 0);
DbIxStatusUpdater *updfunc = 0);
~WebQueueIndexer();
/** This is called by the top indexer in recollindex.

22
src/index/webqueuefetcher.cpp
View File

@ -35,23 +35,23 @@ bool WQDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
{
string udi;
if (!idoc.getmeta(Rcl::Doc::keyudi, &udi) || udi.empty()) {
LOGERR("WQDocFetcher:: no udi in idoc\n" );
return false;
LOGERR("WQDocFetcher:: no udi in idoc\n" );
return false;
}
Rcl::Doc dotdoc;
{
std::unique_lock<std::mutex> locker(o_beagler_mutex);
// Retrieve from our webcache (beagle data). The beagler
// object is created at the first call of this routine and
// deleted when the program exits.
static WebStore o_beagler(cnf);
if (!o_beagler.getFromCache(udi, dotdoc, out.data)) {
LOGINFO("WQDocFetcher::fetch: failed for [" << udi << "]\n");
return false;
}
// Retrieve from our webcache (beagle data). The beagler
// object is created at the first call of this routine and
// deleted when the program exits.
static WebStore o_beagler(cnf);
if (!o_beagler.getFromCache(udi, dotdoc, out.data)) {
LOGINFO("WQDocFetcher::fetch: failed for [" << udi << "]\n");
return false;
}
}
if (dotdoc.mimetype.compare(idoc.mimetype)) {
LOGINFO("WQDocFetcher:: udi [" << udi << "], mimetp mismatch: in: [" <<
LOGINFO("WQDocFetcher:: udi [" << udi << "], mimetp mismatch: in: [" <<
idoc.mimetype << "], bgl [" << dotdoc.mimetype << "]\n");
}
out.kind = RawDoc::RDK_DATA;

64
src/testmains/trsubtreelist.cpp Normal file
View File

@ -0,0 +1,64 @@
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <iostream>
#include <vector>
#include <string>
using namespace std;
#include "subtreelist.h"
#include "rclconfig.h"
#include "rclinit.h"
static char *thisprog;
static char usage [] = " <path> : list document paths in this tree\n";
static void Usage(void)
{
std::cerr << thisprog << ": usage:" << endl << usage;
exit(1);
}
static int op_flags;
#define OPT_o 0x2
int main(int argc, char **argv)
{
string top;
thisprog = argv[0];
argc--; argv++;
while (argc > 0 && **argv == '-') {
(*argv)++;
if (!(**argv))
/* Cas du "adb - core" */
Usage();
while (**argv)
switch (*(*argv)++) {
default: Usage(); break;
}
argc--; argv++;
}
if (argc < 1)
Usage();
top = *argv++;argc--;
string reason;
RclConfig *config = recollinit(0, 0, 0, reason, 0);
if (!config || !config->ok()) {
fprintf(stderr, "Recoll init failed: %s\n", reason.c_str());
exit(1);
}
vector<string> paths;
if (!subtreelist(config, top, paths)) {
cerr << "subtreelist failed" << endl;
exit(1);
}
for (vector<string>::const_iterator it = paths.begin();
it != paths.end(); it++) {
cout << *it << endl;
}
exit(0);
}