tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

cleared out errant tabs

Browse Source
This commit is contained in:
Jean-Francois Dockes
2020-05-30 15:54:49 +02:00
parent 796db76fc6
commit 560041cab9
167 changed files with 18868 additions and 18868 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/index/fetcher.cpp
+3 -3
View File
@@ -35,17 +35,17 @@ std::unique_ptr<DocFetcher> docFetcherMake(RclConfig *config,
string backend;
idoc.getmeta(Rcl::Doc::keybcknd, &backend);
if (backend.empty() || !backend.compare("FS")) {
return std::unique_ptr<DocFetcher>(new FSDocFetcher);
return std::unique_ptr<DocFetcher>(new FSDocFetcher);
#ifndef DISABLE_WEB_INDEXER
} else if (!backend.compare("BGL")) {
return std::unique_ptr<DocFetcher>(new WQDocFetcher);
return std::unique_ptr<DocFetcher>(new WQDocFetcher);
#endif
} else {
std::unique_ptr<DocFetcher> f(exeDocFetcherMake(config, backend));
if (!f) {
LOGERR("DocFetcherFactory: unknown backend [" << backend << "]\n");
}
return f;
return f;
}
}
src/index/idxstatus.h
+6 -6
View File
@@ -25,9 +25,9 @@
class DbIxStatus {
public:
enum Phase {DBIXS_NONE,
DBIXS_FILES, DBIXS_PURGE, DBIXS_STEMDB, DBIXS_CLOSING,
DBIXS_MONITOR,
DBIXS_DONE};
DBIXS_FILES, DBIXS_PURGE, DBIXS_STEMDB, DBIXS_CLOSING,
DBIXS_MONITOR,
DBIXS_DONE};
Phase phase;
std::string fn; // Last file processed
int docsdone; // Documents actually updated
@@ -43,9 +43,9 @@ class DbIxStatus {
bool hasmonitor{false};
void reset() {
phase = DBIXS_FILES;
fn.erase();
docsdone = filesdone = fileerrors = dbtotdocs = totfiles = 0;
phase = DBIXS_FILES;
fn.erase();
docsdone = filesdone = fileerrors = dbtotdocs = totfiles = 0;
}
DbIxStatus() {reset();}
};
src/index/indexer.cpp
+55 -55
View File
@@ -132,14 +132,14 @@ bool ConfIndexer::runFirstIndexing()
{
// Indexing status file existing and not empty ?
if (path_filesize(m_config->getIdxStatusFile()) > 0) {
LOGDEB0("ConfIndexer::runFirstIndexing: no: status file not empty\n");
return false;
LOGDEB0("ConfIndexer::runFirstIndexing: no: status file not empty\n");
return false;
}
// And only do this if the user has kept the default topdirs (~).
vector<string> tdl = m_config->getTopdirs();
if (tdl.size() != 1 || tdl[0].compare(path_canon(path_tildexpand("~")))) {
LOGDEB0("ConfIndexer::runFirstIndexing: no: not home only\n");
return false;
LOGDEB0("ConfIndexer::runFirstIndexing: no: not home only\n");
return false;
}
return true;
}
@@ -150,7 +150,7 @@ bool ConfIndexer::firstFsIndexingSequence()
deleteZ(m_fsindexer);
m_fsindexer = new FsIndexer(m_config, &m_db, m_updater);
if (!m_fsindexer) {
return false;
return false;
}
int flushmb = m_db.getFlushMb();
m_db.setFlushMb(2);
@@ -164,17 +164,17 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
{
Rcl::Db::OpenMode mode = resetbefore ? Rcl::Db::DbTrunc : Rcl::Db::DbUpd;
if (!m_db.open(mode)) {
LOGERR("ConfIndexer: error opening database " << m_config->getDbDir() <<
LOGERR("ConfIndexer: error opening database " << m_config->getDbDir() <<
" : " << m_db.getReason() << "\n");
addIdxReason("indexer", m_db.getReason());
return false;
return false;
}
m_config->setKeyDir(cstr_null);
if (typestorun & IxTFs) {
if (runFirstIndexing()) {
firstFsIndexingSequence();
}
if (runFirstIndexing()) {
firstFsIndexingSequence();
}
deleteZ(m_fsindexer);
m_fsindexer = new FsIndexer(m_config, &m_db, m_updater);
if (!m_fsindexer || !m_fsindexer->index(flags)) {
@@ -183,7 +183,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
} else {
addIdxReason("indexer", "Index creation failed. See log.");
}
m_db.close();
m_db.close();
return false;
}
}
@@ -193,7 +193,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
deleteZ(m_webindexer);
m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater);
if (!m_webindexer || !m_webindexer->index()) {
m_db.close();
m_db.close();
addIdxReason("indexer", "Web index creation failed. See log");
return false;
}
@@ -203,10 +203,10 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
// Get rid of all database entries that don't exist in the
// filesystem anymore. Only if all *configured* indexers ran.
if (m_updater && !m_updater->update(DbIxStatus::DBIXS_PURGE, "")) {
m_db.close();
m_db.close();
addIdxReason("indexer", "Index purge failed. See log");
return false;
}
return false;
}
m_db.purge();
}
@@ -214,22 +214,22 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
// here. Makes no sense to check for cancel, we'll have to close
// anyway
if (m_updater)
m_updater->update(DbIxStatus::DBIXS_CLOSING, string());
m_updater->update(DbIxStatus::DBIXS_CLOSING, string());
if (!m_db.close()) {
LOGERR("ConfIndexer::index: error closing database in " <<
LOGERR("ConfIndexer::index: error closing database in " <<
m_config->getDbDir() << "\n");
addIdxReason("indexer", "Index close/flush failed. See log");
return false;
return false;
}
if (m_updater && !m_updater->update(DbIxStatus::DBIXS_CLOSING, string()))
return false;
return false;
bool ret = true;
if (!createStemmingDatabases()) {
ret = false;
ret = false;
}
if (m_updater && !m_updater->update(DbIxStatus::DBIXS_CLOSING, string()))
return false;
return false;
// Don't fail indexing because of an aspell issue: we ignore the status.
// Messages were written to the reasons output
@@ -237,7 +237,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
clearMimeHandlerCache();
if (m_updater)
m_updater->update(DbIxStatus::DBIXS_DONE, string());
m_updater->update(DbIxStatus::DBIXS_DONE, string());
return ret;
}
@@ -246,14 +246,14 @@ bool ConfIndexer::indexFiles(list<string>& ifiles, int flag)
list<string> myfiles;
string origcwd = m_config->getOrigCwd();
for (const auto& entry : ifiles) {
myfiles.push_back(path_canon(entry, &origcwd));
myfiles.push_back(path_canon(entry, &origcwd));
}
myfiles.sort();
if (!m_db.open(Rcl::Db::DbUpd)) {
LOGERR("ConfIndexer: indexFiles error opening database " <<
LOGERR("ConfIndexer: indexFiles error opening database " <<
m_config->getDbDir() << "\n");
return false;
return false;
}
m_config->setKeyDir(cstr_null);
bool ret = false;
@@ -280,9 +280,9 @@ bool ConfIndexer::indexFiles(list<string>& ifiles, int flag)
}
// The close would be done in our destructor, but we want status here
if (!m_db.close()) {
LOGERR("ConfIndexer::index: error closing database in " <<
LOGERR("ConfIndexer::index: error closing database in " <<
m_config->getDbDir() << "\n");
return false;
return false;
}
ifiles = myfiles;
clearMimeHandlerCache();
@@ -297,7 +297,7 @@ bool ConfIndexer::updateDocs(vector<Rcl::Doc> &docs, IxFlag flag)
docsToPaths(docs, paths);
list<string> files(paths.begin(), paths.end());
if (!files.empty()) {
return indexFiles(files, flag);
return indexFiles(files, flag);
}
return true;
}
@@ -307,14 +307,14 @@ bool ConfIndexer::purgeFiles(list<string> &files, int flag)
list<string> myfiles;
string origcwd = m_config->getOrigCwd();
for (const auto& entry : files) {
myfiles.push_back(path_canon(entry, &origcwd));
myfiles.push_back(path_canon(entry, &origcwd));
}
myfiles.sort();
if (!m_db.open(Rcl::Db::DbUpd)) {
LOGERR("ConfIndexer: purgeFiles error opening database " <<
LOGERR("ConfIndexer: purgeFiles error opening database " <<
m_config->getDbDir() << "\n");
return false;
return false;
}
bool ret = false;
m_config->setKeyDir(cstr_null);
@@ -337,9 +337,9 @@ bool ConfIndexer::purgeFiles(list<string> &files, int flag)
// The close would be done in our destructor, but we want status here
if (!m_db.close()) {
LOGERR("ConfIndexer::purgefiles: error closing database in " <<
LOGERR("ConfIndexer::purgefiles: error closing database in " <<
m_config->getDbDir() << "\n");
return false;
return false;
}
return ret;
}
@@ -356,18 +356,18 @@ bool ConfIndexer::createStemmingDatabases()
addIdxReason("stemming", "could not open db");
return false;
}
vector<string> langs;
stringToStrings(slangs, langs);
vector<string> langs;
stringToStrings(slangs, langs);
// Get the list of existing stem dbs from the database (some may have
// been manually created, we just keep those from the config
vector<string> dblangs = m_db.getStemLangs();
vector<string>::const_iterator it;
for (it = dblangs.begin(); it != dblangs.end(); it++) {
if (find(langs.begin(), langs.end(), *it) == langs.end())
m_db.deleteStemDb(*it);
}
ret = ret && m_db.createStemDbs(langs);
// Get the list of existing stem dbs from the database (some may have
// been manually created, we just keep those from the config
vector<string> dblangs = m_db.getStemLangs();
vector<string>::const_iterator it;
for (it = dblangs.begin(); it != dblangs.end(); it++) {
if (find(langs.begin(), langs.end(), *it) == langs.end())
m_db.deleteStemDb(*it);
}
ret = ret && m_db.createStemDbs(langs);
if (!ret) {
addIdxReason("stemming", "stem db creation failed");
}
@@ -379,7 +379,7 @@ bool ConfIndexer::createStemmingDatabases()
bool ConfIndexer::createStemDb(const string &lang)
{
if (!m_db.open(Rcl::Db::DbUpd))
return false;
return false;
vector<string> langs;
stringToStrings(lang, langs);
return m_db.createStemDbs(langs);
@@ -397,32 +397,32 @@ bool ConfIndexer::createAspellDict()
// it forever.
static int noaspell = -12345;
if (noaspell == -12345) {
noaspell = false;
m_config->getConfParam("noaspell", &noaspell);
noaspell = false;
m_config->getConfParam("noaspell", &noaspell);
}
if (noaspell)
return true;
return true;
if (!m_db.open(Rcl::Db::DbRO)) {
LOGERR("ConfIndexer::createAspellDict: could not open db\n");
return false;
return false;
}
Aspell aspell(m_config);
string reason;
if (!aspell.init(reason)) {
LOGERR("ConfIndexer::createAspellDict: aspell init failed: " <<
LOGERR("ConfIndexer::createAspellDict: aspell init failed: " <<
reason << "\n");
noaspell = true;
return false;
noaspell = true;
return false;
}
LOGDEB("ConfIndexer::createAspellDict: creating dictionary\n");
if (!aspell.buildDict(m_db, reason)) {
LOGERR("ConfIndexer::createAspellDict: aspell buildDict failed: " <<
LOGERR("ConfIndexer::createAspellDict: aspell buildDict failed: " <<
reason << "\n");
addIdxReason("aspell", reason);
noaspell = true;
return false;
noaspell = true;
return false;
}
#endif
return true;
src/index/indexer.h
+1 -1
View File
@@ -45,7 +45,7 @@ class DbIxStatusUpdater {
virtual bool update(DbIxStatus::Phase phase, const string& fn)
{
#ifdef IDX_THREADS
std::unique_lock<std::mutex> lock(m_mutex);
std::unique_lock<std::mutex> lock(m_mutex);
#endif
status.phase = phase;
status.fn = fn;
src/index/mimetype.cpp
+74 -74
View File
@@ -55,11 +55,11 @@ static string mimetypefromdata(RclConfig *cfg, const string &fn, bool usfc)
#ifdef USE_SYSTEM_FILE_COMMAND
if (usfc && mime.empty()) {
// Last resort: use "file -i", or its configured replacement.
// Last resort: use "file -i", or its configured replacement.
// 'file' fallback if the configured command (default:
// xdg-mime) is not found
static const vector<string> tradfilecmd = {{FILE_PROG}, {"-i"}};
static const vector<string> tradfilecmd = {{FILE_PROG}, {"-i"}};
vector<string> cmd;
string scommand;
@@ -81,55 +81,55 @@ static string mimetypefromdata(RclConfig *cfg, const string &fn, bool usfc)
cmd = tradfilecmd;
}
string result;
string result;
LOGDEB2("mimetype: executing: [" << stringsToString(cmd) << "]\n");
if (!ExecCmd::backtick(cmd, result)) {
LOGERR("mimetypefromdata: exec " <<
if (!ExecCmd::backtick(cmd, result)) {
LOGERR("mimetypefromdata: exec " <<
stringsToString(cmd) << " failed\n");
return string();
}
trimstring(result, " \t\n\r");
LOGDEB2("mimetype: systemfilecommand output [" << result << "]\n");
// The normal output from "file -i" looks like the following:
// thefilename.xxx: text/plain; charset=us-ascii
// Sometimes the semi-colon is missing like in:
// mimetype.cpp: text/x-c charset=us-ascii
// And sometimes we only get the mime type. This apparently happens
// when 'file' believes that the file name is binary
return string();
}
trimstring(result, " \t\n\r");
LOGDEB2("mimetype: systemfilecommand output [" << result << "]\n");
// The normal output from "file -i" looks like the following:
// thefilename.xxx: text/plain; charset=us-ascii
// Sometimes the semi-colon is missing like in:
// mimetype.cpp: text/x-c charset=us-ascii
// And sometimes we only get the mime type. This apparently happens
// when 'file' believes that the file name is binary
// xdg-mime only outputs the MIME type.
// If there is no colon and there is a slash, this is hopefuly
// the mime type
if (result.find_first_of(":") == string::npos &&
result.find_first_of("/") != string::npos) {
return result;
}
// If there is no colon and there is a slash, this is hopefuly
// the mime type
if (result.find_first_of(":") == string::npos &&
result.find_first_of("/") != string::npos) {
return result;
}
// Else the result should begin with the file name. Get rid of it:
if (result.find(fn) != 0) {
// Garbage "file" output. Maybe the result of a charset
// conversion attempt?
LOGERR("mimetype: can't interpret output from [" <<
// Else the result should begin with the file name. Get rid of it:
if (result.find(fn) != 0) {
// Garbage "file" output. Maybe the result of a charset
// conversion attempt?
LOGERR("mimetype: can't interpret output from [" <<
stringsToString(cmd) << "] : [" << result << "]\n");
return string();
}
result = result.substr(fn.size());
return string();
}
result = result.substr(fn.size());
// Now should look like ": text/plain; charset=us-ascii"
// Split it, and take second field
list<string> res;
stringToStrings(result, res);
if (res.size() <= 1)
return string();
list<string>::iterator it = res.begin();
mime = *++it;
// Remove possible semi-colon at the end
trimstring(mime, " \t;");
// Now should look like ": text/plain; charset=us-ascii"
// Split it, and take second field
list<string> res;
stringToStrings(result, res);
if (res.size() <= 1)
return string();
list<string>::iterator it = res.begin();
mime = *++it;
// Remove possible semi-colon at the end
trimstring(mime, " \t;");
// File -i will sometimes return strange stuff (ie: "very small file")
if(mime.find("/") == string::npos)
mime.clear();
// File -i will sometimes return strange stuff (ie: "very small file")
if(mime.find("/") == string::npos)
mime.clear();
}
#endif //USE_SYSTEM_FILE_COMMAND
@@ -139,24 +139,24 @@ static string mimetypefromdata(RclConfig *cfg, const string &fn, bool usfc)
/// Guess mime type, first from suffix, then from file data. We also
/// have a list of suffixes that we don't touch at all.
string mimetype(const string &fn, const struct PathStat *stp,
RclConfig *cfg, bool usfc)
RclConfig *cfg, bool usfc)
{
// Use stat data if available to check for non regular files
if (stp) {
// Note: the value used for directories is different from what
// file -i would print on Linux (inode/directory). Probably
// comes from bsd. Thos may surprise a user trying to use a
// 'mime:' filter with the query language, but it's not work
// changing (would force a reindex).
if (stp->pst_type == PathStat::PST_DIR)
return "inode/directory";
if (stp->pst_type == PathStat::PST_SYMLINK)
return "inode/symlink";
if (stp->pst_type != PathStat::PST_REGULAR)
return "inode/x-fsspecial";
// Empty files are just this: avoid further errors with actual filters.
if (stp->pst_size == 0)
return "inode/x-empty";
// Note: the value used for directories is different from what
// file -i would print on Linux (inode/directory). Probably
// comes from bsd. Thos may surprise a user trying to use a
// 'mime:' filter with the query language, but it's not work
// changing (would force a reindex).
if (stp->pst_type == PathStat::PST_DIR)
return "inode/directory";
if (stp->pst_type == PathStat::PST_SYMLINK)
return "inode/symlink";
if (stp->pst_type != PathStat::PST_REGULAR)
return "inode/x-fsspecial";
// Empty files are just this: avoid further errors with actual filters.
if (stp->pst_size == 0)
return "inode/x-empty";
}
string mtype;
@@ -165,40 +165,40 @@ string mimetype(const string &fn, const struct PathStat *stp,
// Extended attribute has priority on everything, as per:
// http://freedesktop.org/wiki/CommonExtendedAttributes
if (pxattr::get(fn, "mime_type", &mtype)) {
LOGDEB0("Mimetype: 'mime_type' xattr : [" << mtype << "]\n");
if (mtype.empty()) {
LOGDEB0("Mimetype: getxattr() returned empty mime type !\n");
} else {
return mtype;
}
LOGDEB0("Mimetype: 'mime_type' xattr : [" << mtype << "]\n");
if (mtype.empty()) {
LOGDEB0("Mimetype: getxattr() returned empty mime type !\n");
} else {
return mtype;
}
}
#endif
if (cfg == 0) {
LOGERR("Mimetype: null config ??\n");
return mtype;
LOGERR("Mimetype: null config ??\n");
return mtype;
}
if (cfg->inStopSuffixes(fn)) {
LOGDEB("mimetype: fn [" << fn << "] in stopsuffixes\n");
return mtype;
LOGDEB("mimetype: fn [" << fn << "] in stopsuffixes\n");
return mtype;
}
// Compute file name suffix and search the mimetype map
string::size_type dot = fn.find_first_of(".");
while (dot != string::npos) {
string suff = stringtolower(fn.substr(dot));
mtype = cfg->getMimeTypeFromSuffix(suff);
if (!mtype.empty() || dot >= fn.size() - 1)
break;
dot = fn.find_first_of(".", dot + 1);
string suff = stringtolower(fn.substr(dot));
mtype = cfg->getMimeTypeFromSuffix(suff);
if (!mtype.empty() || dot >= fn.size() - 1)
break;
dot = fn.find_first_of(".", dot + 1);
}
// If type was not determined from suffix, examine file data. Can
// only do this if we have an actual file (as opposed to a pure
// name).
if (mtype.empty() && stp)
mtype = mimetypefromdata(cfg, fn, usfc);
mtype = mimetypefromdata(cfg, fn, usfc);
return mtype;
}
src/index/rclmon.h
+3 -3
View File
@@ -47,7 +47,7 @@ using std::multimap;
class RclMonEvent {
public:
enum EvType {RCLEVT_NONE= 0, RCLEVT_MODIFY=1, RCLEVT_DELETE=2,
RCLEVT_DIRCREATE=3, RCLEVT_ISDIR=0x10};
RCLEVT_DIRCREATE=3, RCLEVT_ISDIR=0x10};
string m_path;
// Type and flags
int m_etyp;
@@ -61,13 +61,13 @@ class RclMonEvent {
bool m_needidx;
RclMonEvent() : m_etyp(RCLEVT_NONE),
m_itvsecs(0), m_minclock(0), m_needidx(false) {}
m_itvsecs(0), m_minclock(0), m_needidx(false) {}
EvType evtype() {return EvType(m_etyp & 0xf);}
int evflags() {return m_etyp & 0xf0;}
};
enum RclMonitorOption {RCLMON_NONE=0, RCLMON_NOFORK=1, RCLMON_NOX11=2,
RCLMON_NOCONFCHECK=4};
RCLMON_NOCONFCHECK=4};
/**
* Monitoring event queue. This is the shared object between the main thread
src/index/rclmonprc.cpp
+140 -140
View File
@@ -142,19 +142,19 @@ public:
std::condition_variable m_cond;
RclEQData()
: m_config(0), m_ok(true)
: m_config(0), m_ok(true)
{
}
void readDelayPats(int dfltsecs);
DelayPat searchDelayPats(const string& path)
{
for (vector<DelayPat>::iterator it = m_delaypats.begin();
it != m_delaypats.end(); it++) {
if (fnmatch(it->pattern.c_str(), path.c_str(), 0) == 0) {
return *it;
}
}
return DelayPat();
for (vector<DelayPat>::iterator it = m_delaypats.begin();
it != m_delaypats.end(); it++) {
if (fnmatch(it->pattern.c_str(), path.c_str(), 0) == 0) {
return *it;
}
}
return DelayPat();
}
void delayInsert(const queue_type::iterator &qit);
};
@@ -162,30 +162,30 @@ public:
void RclEQData::readDelayPats(int dfltsecs)
{
if (m_config == 0)
return;
return;
string patstring;
if (!m_config->getConfParam("mondelaypatterns", patstring) ||
patstring.empty())
return;
patstring.empty())
return;
vector<string> dplist;
if (!stringToStrings(patstring, dplist)) {
LOGERR("rclEQData: bad pattern list: [" << (patstring) << "]\n" );
return;
LOGERR("rclEQData: bad pattern list: [" << (patstring) << "]\n" );
return;
}
for (vector<string>::iterator it = dplist.begin();
it != dplist.end(); it++) {
string::size_type pos = it->find_last_of(":");
DelayPat dp;
dp.pattern = it->substr(0, pos);
if (pos != string::npos && pos != it->size()-1) {
dp.seconds = atoi(it->substr(pos+1).c_str());
} else {
dp.seconds = dfltsecs;
}
m_delaypats.push_back(dp);
LOGDEB2("rclmon::readDelayPats: add [" << (dp.pattern) << "] " << (dp.seconds) << "\n" );
it != dplist.end(); it++) {
string::size_type pos = it->find_last_of(":");
DelayPat dp;
dp.pattern = it->substr(0, pos);
if (pos != string::npos && pos != it->size()-1) {
dp.seconds = atoi(it->substr(pos+1).c_str());
} else {
dp.seconds = dfltsecs;
}
m_delaypats.push_back(dp);
LOGDEB2("rclmon::readDelayPats: add [" << (dp.pattern) << "] " << (dp.seconds) << "\n" );
}
}
@@ -197,12 +197,12 @@ void RclEQData::delayInsert(const queue_type::iterator &qit)
MONDEB("RclEQData::delayInsert: minclock " << qit->second.m_minclock <<
std::endl);
for (delays_type::iterator dit = m_delays.begin();
dit != m_delays.end(); dit++) {
queue_type::iterator qit1 = *dit;
if ((*qit1).second.m_minclock > qit->second.m_minclock) {
m_delays.insert(dit, qit);
return;
}
dit != m_delays.end(); dit++) {
queue_type::iterator qit1 = *dit;
if ((*qit1).second.m_minclock > qit->second.m_minclock) {
m_delays.insert(dit, qit);
return;
}
}
m_delays.push_back(qit);
}
@@ -220,7 +220,7 @@ RclMonEventQueue::~RclMonEventQueue()
void RclMonEventQueue::setopts(int opts)
{
if (m_data)
m_data->m_opts = opts;
m_data->m_opts = opts;
}
/** Wait until there is something to process on the queue, or timeout.
@@ -232,22 +232,22 @@ std::unique_lock<std::mutex> RclMonEventQueue::wait(int seconds, bool *top)
MONDEB("RclMonEventQueue::wait, seconds: " << seconds << std::endl);
if (!empty()) {
MONDEB("RclMonEventQueue:: immediate return\n");
return lock;
MONDEB("RclMonEventQueue:: immediate return\n");
return lock;
}
int err;
if (seconds > 0) {
if (top)
*top = false;
if (m_data->m_cond.wait_for(lock, std::chrono::seconds(seconds)) ==
if (top)
*top = false;
if (m_data->m_cond.wait_for(lock, std::chrono::seconds(seconds)) ==
std::cv_status::timeout) {
*top = true;
MONDEB("RclMonEventQueue:: timeout\n");
return lock;
}
} else {
m_data->m_cond.wait(lock);
m_data->m_cond.wait(lock);
}
MONDEB("RclMonEventQueue:: non-timeout return\n");
return lock;
@@ -269,16 +269,16 @@ RclConfig *RclMonEventQueue::getConfig()
bool RclMonEventQueue::ok()
{
if (m_data == 0) {
LOGINFO("RclMonEventQueue: not ok: bad state\n" );
return false;
LOGINFO("RclMonEventQueue: not ok: bad state\n" );
return false;
}
if (stopindexing) {
LOGINFO("RclMonEventQueue: not ok: stop request\n" );
return false;
LOGINFO("RclMonEventQueue: not ok: stop request\n" );
return false;
}
if (!m_data->m_ok) {
LOGINFO("RclMonEventQueue: not ok: queue terminated\n" );
return false;
LOGINFO("RclMonEventQueue: not ok: queue terminated\n" );
return false;
}
return true;
}
@@ -295,24 +295,24 @@ void RclMonEventQueue::setTerminate()
bool RclMonEventQueue::empty()
{
if (m_data == 0) {
MONDEB("RclMonEventQueue::empty(): true (m_data==0)\n");
return true;
MONDEB("RclMonEventQueue::empty(): true (m_data==0)\n");
return true;
}
if (!m_data->m_iqueue.empty()) {
MONDEB("RclMonEventQueue::empty(): false (m_iqueue not empty)\n");
return true;
MONDEB("RclMonEventQueue::empty(): false (m_iqueue not empty)\n");
return true;
}
if (m_data->m_dqueue.empty()) {
MONDEB("RclMonEventQueue::empty(): true (m_Xqueue both empty)\n");
return true;
MONDEB("RclMonEventQueue::empty(): true (m_Xqueue both empty)\n");
return true;
}
// Only dqueue has events. Have to check the delays (only the
// first, earliest one):
queue_type::iterator qit = *(m_data->m_delays.begin());
if (qit->second.m_minclock > time(0)) {
MONDEB("RclMonEventQueue::empty(): true (no delay ready " <<
MONDEB("RclMonEventQueue::empty(): true (no delay ready " <<
qit->second.m_minclock << ")\n");
return true;
return true;
}
MONDEB("RclMonEventQueue::empty(): returning false (delay expired)\n");
return false;
@@ -329,36 +329,36 @@ RclMonEvent RclMonEventQueue::pop()
// Look at the delayed events, get rid of the expired/unactive
// ones, possibly return an expired/needidx one.
while (!m_data->m_delays.empty()) {
delays_type::iterator dit = m_data->m_delays.begin();
queue_type::iterator qit = *dit;
MONDEB("RclMonEventQueue::pop(): in delays: evt minclock " <<
qit->second.m_minclock << std::endl);
if (qit->second.m_minclock <= now) {
if (qit->second.m_needidx) {
RclMonEvent ev = qit->second;
qit->second.m_minclock = time(0) + qit->second.m_itvsecs;
qit->second.m_needidx = false;
m_data->m_delays.erase(dit);
m_data->delayInsert(qit);
return ev;
} else {
// Delay elapsed without new update, get rid of event.
m_data->m_dqueue.erase(qit);
m_data->m_delays.erase(dit);
}
} else {
// This and following events are for later processing, we
// are done with the delayed event list.
break;
}
delays_type::iterator dit = m_data->m_delays.begin();
queue_type::iterator qit = *dit;
MONDEB("RclMonEventQueue::pop(): in delays: evt minclock " <<
qit->second.m_minclock << std::endl);
if (qit->second.m_minclock <= now) {
if (qit->second.m_needidx) {
RclMonEvent ev = qit->second;
qit->second.m_minclock = time(0) + qit->second.m_itvsecs;
qit->second.m_needidx = false;
m_data->m_delays.erase(dit);
m_data->delayInsert(qit);
return ev;
} else {
// Delay elapsed without new update, get rid of event.
m_data->m_dqueue.erase(qit);
m_data->m_delays.erase(dit);
}
} else {
// This and following events are for later processing, we
// are done with the delayed event list.
break;
}
}
// Look for non-delayed event
if (!m_data->m_iqueue.empty()) {
queue_type::iterator qit = m_data->m_iqueue.begin();
RclMonEvent ev = qit->second;
m_data->m_iqueue.erase(qit);
return ev;
queue_type::iterator qit = m_data->m_iqueue.begin();
RclMonEvent ev = qit->second;
m_data->m_iqueue.erase(qit);
return ev;
}
return RclMonEvent();
@@ -376,32 +376,32 @@ bool RclMonEventQueue::pushEvent(const RclMonEvent &ev)
DelayPat pat = m_data->searchDelayPats(ev.m_path);
if (pat.seconds != 0) {
// Using delayed reindex queue. Need to take care of minclock and also
// insert into the in-minclock-order list
queue_type::iterator qit = m_data->m_dqueue.find(ev.m_path);
if (qit == m_data->m_dqueue.end()) {
// Not there yet, insert new
qit =
m_data->m_dqueue.insert(queue_type::value_type(ev.m_path, ev)).first;
// Set the time to next index to "now" as it has not been
// indexed recently (otherwise it would still be in the
// queue), and add the iterator to the delay queue.
qit->second.m_minclock = time(0);
qit->second.m_needidx = true;
qit->second.m_itvsecs = pat.seconds;
m_data->delayInsert(qit);
} else {
// Already in queue. Possibly update type but save minclock
// (so no need to touch m_delays). Flag as needing indexing
time_t saved_clock = qit->second.m_minclock;
qit->second = ev;
qit->second.m_minclock = saved_clock;
qit->second.m_needidx = true;
}
// Using delayed reindex queue. Need to take care of minclock and also
// insert into the in-minclock-order list
queue_type::iterator qit = m_data->m_dqueue.find(ev.m_path);
if (qit == m_data->m_dqueue.end()) {
// Not there yet, insert new
qit =
m_data->m_dqueue.insert(queue_type::value_type(ev.m_path, ev)).first;
// Set the time to next index to "now" as it has not been
// indexed recently (otherwise it would still be in the
// queue), and add the iterator to the delay queue.
qit->second.m_minclock = time(0);
qit->second.m_needidx = true;
qit->second.m_itvsecs = pat.seconds;
m_data->delayInsert(qit);
} else {
// Immediate event: just insert it, erasing any previously
// existing entry
m_data->m_iqueue[ev.m_path] = ev;
// Already in queue. Possibly update type but save minclock
// (so no need to touch m_delays). Flag as needing indexing
time_t saved_clock = qit->second.m_minclock;
qit->second = ev;
qit->second.m_minclock = saved_clock;
qit->second.m_needidx = true;
}
} else {
// Immediate event: just insert it, erasing any previously
// existing entry
m_data->m_iqueue[ev.m_path] = ev;
}
m_data->m_cond.notify_all();
@@ -429,19 +429,19 @@ static bool expeditedIndexingRequested(RclConfig *conf)
{
static vector<string> rqfiles;
if (rqfiles.empty()) {
rqfiles.push_back(path_cat(conf->getConfDir(), "rclmonixnow"));
const char *cp;
if ((cp = getenv("RECOLL_CONFTOP"))) {
rqfiles.push_back(path_cat(cp, "rclmonixnow"));
}
if ((cp = getenv("RECOLL_CONFMID"))) {
rqfiles.push_back(path_cat(cp, "rclmonixnow"));
}
rqfiles.push_back(path_cat(conf->getConfDir(), "rclmonixnow"));
const char *cp;
if ((cp = getenv("RECOLL_CONFTOP"))) {
rqfiles.push_back(path_cat(cp, "rclmonixnow"));
}
if ((cp = getenv("RECOLL_CONFMID"))) {
rqfiles.push_back(path_cat(cp, "rclmonixnow"));
}
}
bool found = false;
for (vector<string>::const_iterator it = rqfiles.begin();
it != rqfiles.end(); it++) {
found = found || checkfileanddelete(*it);
it != rqfiles.end(); it++) {
found = found || checkfileanddelete(*it);
}
return found;
}
@@ -449,9 +449,9 @@ static bool expeditedIndexingRequested(RclConfig *conf)
bool startMonitor(RclConfig *conf, int opts)
{
if (!conf->getConfParam("monauxinterval", &auxinterval))
auxinterval = dfltauxinterval;
auxinterval = dfltauxinterval;
if (!conf->getConfParam("monixinterval", &ixinterval))
ixinterval = dfltixinterval;
ixinterval = dfltixinterval;
rclEQ.setConfig(conf);
rclEQ.setopts(opts);
@@ -494,7 +494,7 @@ bool startMonitor(RclConfig *conf, int opts)
if (!rclEQ.ok() || x11dead) {
break;
}
// Process event queue
for (;;) {
// Retrieve event
@@ -533,14 +533,14 @@ bool startMonitor(RclConfig *conf, int opts)
}
now = time(0);
// Process. We don't do this every time but let the lists accumulate
// Process. We don't do this every time but let the lists accumulate
// a little, this saves processing. Start at once if list is big.
if (expeditedIndexingRequested(conf) ||
(now - lastixtime > ixinterval) ||
(deleted.size() + modified.size() > 20)) {
(now - lastixtime > ixinterval) ||
(deleted.size() + modified.size() > 20)) {
lastixtime = now;
// Used to do the modified list first, but it does seem
// smarter to make room first...
// Used to do the modified list first, but it does seem
// smarter to make room first...
if (!deleted.empty()) {
deleted.sort();
deleted.unique();
@@ -559,28 +559,28 @@ bool startMonitor(RclConfig *conf, int opts)
}
}
// Recreate the auxiliary dbs every hour at most.
// Recreate the auxiliary dbs every hour at most.
now = time(0);
if (didsomething && now - lastauxtime > auxinterval) {
lastauxtime = now;
didsomething = false;
if (!createAuxDbs(conf)) {
// We used to bail out on error here. Not anymore,
// because this is most of the time due to a failure
// of aspell dictionary generation, which is not
// critical.
}
}
if (didsomething && now - lastauxtime > auxinterval) {
lastauxtime = now;
didsomething = false;
if (!createAuxDbs(conf)) {
// We used to bail out on error here. Not anymore,
// because this is most of the time due to a failure
// of aspell dictionary generation, which is not
// critical.
}
}
// Check for a config change
if (!(opts & RCLMON_NOCONFCHECK) && o_reexec && conf->sourceChanged()) {
LOGDEB("Rclmonprc: config changed, reexecuting myself\n" );
// We never want to have a -n option after a config
// change. -n was added by the reexec after the initial
// pass even if it was not given on the command line
o_reexec->removeArg("-n");
o_reexec->reexec();
}
// Check for a config change
if (!(opts & RCLMON_NOCONFCHECK) && o_reexec && conf->sourceChanged()) {
LOGDEB("Rclmonprc: config changed, reexecuting myself\n" );
// We never want to have a -n option after a config
// change. -n was added by the reexec after the initial
// pass even if it was not given on the command line
o_reexec->removeArg("-n");
o_reexec->reexec();
}
}
LOGDEB("Rclmonprc: calling queue setTerminate\n" );
rclEQ.setTerminate();
src/index/recollindex.cpp
+32 -32
View File
@@ -550,22 +550,22 @@ static void flushIdxReasons()
static vector<string> argstovector(int argc, char **argv)
{
vector<string> args;
for (int i = 0; i < argc; i++) {
args.push_back(argv[i]);
}
return args;
vector<string> args;
for (int i = 0; i < argc; i++) {
args.push_back(argv[i]);
}
return args;
}
static vector<string> fileToArgs(const string& fn)
{
string reason, data;
if (!file_to_string(fn, data, &reason)) {
cerr << "Failed reading args file " << fn << " errno " << errno << "\n";
exit(1);
}
vector<string> args;
stringToStrings(data, args);
return args;
string reason, data;
if (!file_to_string(fn, data, &reason)) {
cerr << "Failed reading args file " << fn << " errno " << errno << "\n";
exit(1);
}
vector<string> args;
stringToStrings(data, args);
return args;
}
int main(int argc, char **argv)
@@ -585,23 +585,23 @@ int main(int argc, char **argv)
thisprog = path_absolute(argv[0]);
argc--; argv++;
vector<string> args = argstovector(argc, argv);
vector<string> args = argstovector(argc, argv);
// Passing args through a temp file: this is used on Windows to
// avoid issues with charsets in args (avoid using wmain)
if (args.size() == 1 && args[0][0] != '-') {
args = fileToArgs(args[0]);
}
// Passing args through a temp file: this is used on Windows to
// avoid issues with charsets in args (avoid using wmain)
if (args.size() == 1 && args[0][0] != '-') {
args = fileToArgs(args[0]);
}
unsigned int aremain = args.size();
unsigned int argidx = 0;
for (; argidx < args.size(); argidx++) {
const string& arg{args[argidx]};
aremain = args.size() - argidx;
if (arg[0] != '-') {
break;
}
for (unsigned int cidx = 1; cidx < arg.size(); cidx++) {
unsigned int aremain = args.size();
unsigned int argidx = 0;
for (; argidx < args.size(); argidx++) {
const string& arg{args[argidx]};
aremain = args.size() - argidx;
if (arg[0] != '-') {
break;
}
for (unsigned int cidx = 1; cidx < arg.size(); cidx++) {
switch (arg[cidx]) {
case 'b': op_flags |= OPT_b; break;
case 'c': op_flags |= OPT_c; if (aremain < 2) Usage();
@@ -639,10 +639,10 @@ int main(int argc, char **argv)
case 'z': op_flags |= OPT_z; break;
default: Usage(); break;
}
}
b1:
;
}
}
b1:
;
}
aremain = args.size() - argidx;
if (op_flags & OPT_h)
src/index/subtreelist.cpp
+17 -17
View File
@@ -28,13 +28,13 @@
#include "log.h"
bool subtreelist(RclConfig *config, const string& top,
vector<string>& paths)
vector<string>& paths)
{
LOGDEB("subtreelist: top: [" << (top) << "]\n" );
Rcl::Db rcldb(config);
if (!rcldb.open(Rcl::Db::DbRO)) {
LOGERR("subtreelist: can't open database in [" << (config->getDbDir()) << "]: " << (rcldb.getReason()) << "\n" );
return false;
LOGERR("subtreelist: can't open database in [" << (config->getDbDir()) << "]: " << (rcldb.getReason()) << "\n" );
return false;
}
Rcl::SearchData *sd = new Rcl::SearchData(Rcl::SCLT_OR, cstr_null);
@@ -47,12 +47,12 @@ bool subtreelist(RclConfig *config, const string& top,
int cnt = query.getResCnt();
for (int i = 0; i < cnt; i++) {
Rcl::Doc doc;
if (!query.getDoc(i, doc))
break;
string path = fileurltolocalpath(doc.url);
if (!path.empty())
paths.push_back(path);
Rcl::Doc doc;
if (!query.getDoc(i, doc))
break;
string path = fileurltolocalpath(doc.url);
if (!path.empty())
paths.push_back(path);
}
return true;
}
@@ -104,26 +104,26 @@ int main(int argc, char **argv)
switch (*(*argv)++) {
default: Usage(); break;
}
argc--; argv++;
argc--; argv++;
}
if (argc < 1)
Usage();
Usage();
top = *argv++;argc--;
string reason;
RclConfig *config = recollinit(0, 0, 0, reason, 0);
if (!config || !config->ok()) {
fprintf(stderr, "Recoll init failed: %s\n", reason.c_str());
exit(1);
fprintf(stderr, "Recoll init failed: %s\n", reason.c_str());
exit(1);
}
vector<string> paths;
if (!subtreelist(config, top, paths)) {
cerr << "subtreelist failed" << endl;
exit(1);
cerr << "subtreelist failed" << endl;
exit(1);
}
for (vector<string>::const_iterator it = paths.begin();
it != paths.end(); it++) {
cout << *it << endl;
it != paths.end(); it++) {
cout << *it << endl;
}
exit(0);
}
src/index/subtreelist.h
+1 -1
View File
@@ -28,6 +28,6 @@ class RclConfig;
// the real time indexer to purge entries when a top directory is
// renamed. This is really convoluted, I'd like a better way.
extern bool subtreelist(RclConfig *config, const string& top,
std::vector<std::string>& paths);
std::vector<std::string>& paths);
#endif /* _SUBTREELIST_H_INCLUDED_ */
src/index/webqueuefetcher.cpp
+11 -11
View File
@@ -35,23 +35,23 @@ bool WQDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
{
string udi;
if (!idoc.getmeta(Rcl::Doc::keyudi, &udi) || udi.empty()) {
LOGERR("WQDocFetcher:: no udi in idoc\n" );
return false;
LOGERR("WQDocFetcher:: no udi in idoc\n" );
return false;
}
Rcl::Doc dotdoc;
{
std::unique_lock<std::mutex> locker(o_beagler_mutex);
// Retrieve from our webcache (beagle data). The beagler
// object is created at the first call of this routine and
// deleted when the program exits.
static WebStore o_beagler(cnf);
if (!o_beagler.getFromCache(udi, dotdoc, out.data)) {
LOGINFO("WQDocFetcher::fetch: failed for [" << udi << "]\n");
return false;
}
// Retrieve from our webcache (beagle data). The beagler
// object is created at the first call of this routine and
// deleted when the program exits.
static WebStore o_beagler(cnf);
if (!o_beagler.getFromCache(udi, dotdoc, out.data)) {
LOGINFO("WQDocFetcher::fetch: failed for [" << udi << "]\n");
return false;
}
}
if (dotdoc.mimetype.compare(idoc.mimetype)) {
LOGINFO("WQDocFetcher:: udi [" << udi << "], mimetp mismatch: in: [" <<
LOGINFO("WQDocFetcher:: udi [" << udi << "], mimetp mismatch: in: [" <<
idoc.mimetype << "], bgl [" << dotdoc.mimetype << "]\n");
}
out.kind = RawDoc::RDK_DATA;