tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

recollindex: add option to output more detailed explanation of partial or complete failure (e.g.: aspell problem)

Browse Source
This commit is contained in:
Jean-Francois Dockes 2017-04-24 09:16:41 +02:00
parent 7f2c29bb6c
commit ecc4db774e
3 changed files with 153 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
src/index/indexer.cpp
View File

@ -23,6 +23,7 @@
#include "cstr.h" #include "cstr.h"
#include "log.h" #include "log.h"
#include "recollindex.h"
#include "indexer.h" #include "indexer.h"
#include "fsindexer.h" #include "fsindexer.h"
#ifndef DISABLE_WEB_INDEXER #ifndef DISABLE_WEB_INDEXER
@ -92,7 +93,9 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
{ {
Rcl::Db::OpenMode mode = resetbefore ? Rcl::Db::DbTrunc : Rcl::Db::DbUpd; Rcl::Db::OpenMode mode = resetbefore ? Rcl::Db::DbTrunc : Rcl::Db::DbUpd;
if (!m_db.open(mode)) { if (!m_db.open(mode)) {
LOGERR("ConfIndexer: error opening database " << (m_config->getDbDir()) << " : " << (m_db.getReason()) << "\n" ); LOGERR("ConfIndexer: error opening database " << m_config->getDbDir() <<
" : " << m_db.getReason() << "\n");
addIdxReason("indexer", m_db.getReason());
return false; return false;
} }
@ -104,6 +107,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
deleteZ(m_fsindexer); deleteZ(m_fsindexer);
m_fsindexer = new FsIndexer(m_config, &m_db, m_updater); m_fsindexer = new FsIndexer(m_config, &m_db, m_updater);
if (!m_fsindexer || !m_fsindexer->index(flags)) { if (!m_fsindexer || !m_fsindexer->index(flags)) {
addIdxReason("indexer", "Index creation failed. See log");
m_db.close(); m_db.close();
return false; return false;
} }
@ -114,6 +118,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
m_beagler = new BeagleQueueIndexer(m_config, &m_db, m_updater); m_beagler = new BeagleQueueIndexer(m_config, &m_db, m_updater);
if (!m_beagler || !m_beagler->index()) { if (!m_beagler || !m_beagler->index()) {
m_db.close(); m_db.close();
addIdxReason("indexer", "Web index creation failed. See log");
return false; return false;
} }
} }
@ -123,6 +128,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
// filesystem anymore. Only if all *configured* indexers ran. // filesystem anymore. Only if all *configured* indexers ran.
if (m_updater && !m_updater->update(DbIxStatus::DBIXS_PURGE, string())) { if (m_updater && !m_updater->update(DbIxStatus::DBIXS_PURGE, string())) {
m_db.close(); m_db.close();
addIdxReason("indexer", "Index purge failed. See log");
return false; return false;
} }
m_db.purge(); m_db.purge();
@ -134,7 +140,9 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
if (m_updater) if (m_updater)
m_updater->update(DbIxStatus::DBIXS_CLOSING, string()); m_updater->update(DbIxStatus::DBIXS_CLOSING, string());
if (!m_db.close()) { if (!m_db.close()) {
LOGERR("ConfIndexer::index: error closing database in " << (m_config->getDbDir()) << "\n" ); LOGERR("ConfIndexer::index: error closing database in " <<
m_config->getDbDir() << "\n");
addIdxReason("indexer", "Index close/flush failed. See log");
return false; return false;
} }
@ -146,7 +154,11 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
} }
if (m_updater && !m_updater->update(DbIxStatus::DBIXS_CLOSING, string())) if (m_updater && !m_updater->update(DbIxStatus::DBIXS_CLOSING, string()))
return false; return false;
ret = ret && createAspellDict();
// Don't fail indexing because of an aspell issue: we ignore the status.
// Messages were written to the reasons output
(void)createAspellDict();
clearMimeHandlerCache(); clearMimeHandlerCache();
if (m_updater) if (m_updater)
m_updater->update(DbIxStatus::DBIXS_DONE, string()); m_updater->update(DbIxStatus::DBIXS_DONE, string());
@ -164,7 +176,8 @@ bool ConfIndexer::indexFiles(list<string>& ifiles, int flag)
myfiles.sort(); myfiles.sort();
if (!m_db.open(Rcl::Db::DbUpd)) { if (!m_db.open(Rcl::Db::DbUpd)) {
LOGERR("ConfIndexer: indexFiles error opening database " << (m_config->getDbDir()) << "\n" ); LOGERR("ConfIndexer: indexFiles error opening database " <<
m_config->getDbDir() << "\n");
return false; return false;
} }
m_config->setKeyDir(cstr_null); m_config->setKeyDir(cstr_null);
@ -173,7 +186,8 @@ bool ConfIndexer::indexFiles(list<string>& ifiles, int flag)
m_fsindexer = new FsIndexer(m_config, &m_db, m_updater); m_fsindexer = new FsIndexer(m_config, &m_db, m_updater);
if (m_fsindexer) if (m_fsindexer)
ret = m_fsindexer->indexFiles(myfiles, flag); ret = m_fsindexer->indexFiles(myfiles, flag);
LOGDEB2("ConfIndexer::indexFiles: fsindexer returned " << (ret) << ", " << (myfiles.size()) << " files remainining\n" ); LOGDEB2("ConfIndexer::indexFiles: fsindexer returned " << ret << ", " <<
myfiles.size() << " files remainining\n");
#ifndef DISABLE_WEB_INDEXER #ifndef DISABLE_WEB_INDEXER
if (m_dobeagle && !myfiles.empty() && !(flag & IxFNoWeb)) { if (m_dobeagle && !myfiles.empty() && !(flag & IxFNoWeb)) {
@ -188,7 +202,8 @@ bool ConfIndexer::indexFiles(list<string>& ifiles, int flag)
#endif #endif
// The close would be done in our destructor, but we want status here // The close would be done in our destructor, but we want status here
if (!m_db.close()) { if (!m_db.close()) {
LOGERR("ConfIndexer::index: error closing database in " << (m_config->getDbDir()) << "\n" ); LOGERR("ConfIndexer::index: error closing database in " <<
m_config->getDbDir() << "\n");
return false; return false;
} }
ifiles = myfiles; ifiles = myfiles;
@ -212,7 +227,8 @@ bool ConfIndexer::docsToPaths(vector<Rcl::Doc> &docs, vector<string> &paths)
// Filesystem document. The url has to be like file:// // Filesystem document. The url has to be like file://
if (idoc.url.find(cstr_fileu) != 0) { if (idoc.url.find(cstr_fileu) != 0) {
LOGERR("idx::docsToPaths: FS backend and non fs url: [" << (idoc.url) << "]\n" ); LOGERR("idx::docsToPaths: FS backend and non fs url: [" <<
idoc.url << "]\n");
continue; continue;
} }
paths.push_back(idoc.url.substr(7, string::npos)); paths.push_back(idoc.url.substr(7, string::npos));
@ -244,7 +260,8 @@ bool ConfIndexer::purgeFiles(std::list<string> &files, int flag)
myfiles.sort(); myfiles.sort();
if (!m_db.open(Rcl::Db::DbUpd)) { if (!m_db.open(Rcl::Db::DbUpd)) {
LOGERR("ConfIndexer: purgeFiles error opening database " << (m_config->getDbDir()) << "\n" ); LOGERR("ConfIndexer: purgeFiles error opening database " <<
m_config->getDbDir() << "\n");
return false; return false;
} }
bool ret = false; bool ret = false;
@ -268,7 +285,8 @@ bool ConfIndexer::purgeFiles(std::list<string> &files, int flag)
// The close would be done in our destructor, but we want status here // The close would be done in our destructor, but we want status here
if (!m_db.close()) { if (!m_db.close()) {
LOGERR("ConfIndexer::purgefiles: error closing database in " << (m_config->getDbDir()) << "\n" ); LOGERR("ConfIndexer::purgefiles: error closing database in " <<
m_config->getDbDir() << "\n");
return false; return false;
} }
return ret; return ret;
@ -283,6 +301,7 @@ bool ConfIndexer::createStemmingDatabases()
if (m_config->getConfParam("indexstemminglanguages", slangs)) { if (m_config->getConfParam("indexstemminglanguages", slangs)) {
if (!m_db.open(Rcl::Db::DbUpd)) { if (!m_db.open(Rcl::Db::DbUpd)) {
LOGERR("ConfIndexer::createStemmingDb: could not open db\n"); LOGERR("ConfIndexer::createStemmingDb: could not open db\n");
addIdxReason("stemming", "could not open db");
return false; return false;
} }
vector<string> langs; vector<string> langs;
@ -297,6 +316,9 @@ bool ConfIndexer::createStemmingDatabases()
m_db.deleteStemDb(*it); m_db.deleteStemDb(*it);
} }
ret = ret && m_db.createStemDbs(langs); ret = ret && m_db.createStemDbs(langs);
if (!ret) {
addIdxReason("stemming", "stem db creation failed");
}
} }
m_db.close(); m_db.close();
return ret; return ret;
@ -337,13 +359,16 @@ bool ConfIndexer::createAspellDict()
Aspell aspell(m_config); Aspell aspell(m_config);
string reason; string reason;
if (!aspell.init(reason)) { if (!aspell.init(reason)) {
LOGERR("ConfIndexer::createAspellDict: aspell init failed: " << (reason) << "\n" ); LOGERR("ConfIndexer::createAspellDict: aspell init failed: " <<
reason << "\n");
noaspell = true; noaspell = true;
return false; return false;
} }
LOGDEB("ConfIndexer::createAspellDict: creating dictionary\n"); LOGDEB("ConfIndexer::createAspellDict: creating dictionary\n");
if (!aspell.buildDict(m_db, reason)) { if (!aspell.buildDict(m_db, reason)) {
LOGERR("ConfIndexer::createAspellDict: aspell buildDict failed: " << (reason) << "\n" ); LOGERR("ConfIndexer::createAspellDict: aspell buildDict failed: " <<
reason << "\n");
addIdxReason("aspell", reason);
noaspell = true; noaspell = true;
return false; return false;
} }

122
src/index/recollindex.cpp
View File

@ -61,27 +61,28 @@ using namespace std;
// Command line options // Command line options
static int op_flags; static int op_flags;
#define OPT_MOINS 0x1 #define OPT_MOINS 0x1
#define OPT_z 0x2 #define OPT_C 0x2
#define OPT_h 0x4 #define OPT_D 0x4
#define OPT_i 0x8 #define OPT_E 0x8
#define OPT_s 0x10 #define OPT_K 0x10
#define OPT_c 0x20 #define OPT_R 0x20
#define OPT_S 0x40 #define OPT_S 0x40
#define OPT_m 0x80 #define OPT_Z 0x80
#define OPT_D 0x100 #define OPT_b 0x100
#define OPT_e 0x200 #define OPT_c 0x200
#define OPT_w 0x400 #define OPT_e 0x400
#define OPT_x 0x800 #define OPT_f 0x800
#define OPT_l 0x1000 #define OPT_h 0x1000
#define OPT_b 0x2000 #define OPT_i 0x2000
#define OPT_f 0x4000 #define OPT_k 0x4000
#define OPT_C 0x8000 #define OPT_l 0x8000
#define OPT_Z 0x10000 #define OPT_m 0x10000
#define OPT_n 0x20000 #define OPT_n 0x20000
#define OPT_r 0x40000 #define OPT_r 0x40000
#define OPT_k 0x80000 #define OPT_s 0x80000
#define OPT_E 0x100000 #define OPT_w 0x100000
#define OPT_K 0x200000 #define OPT_x 0x200000
#define OPT_z 0x400000
ReExec *o_reexec; ReExec *o_reexec;
// Globals for atexit cleanup // Globals for atexit cleanup
@ -404,6 +405,35 @@ void lockorexit(Pidfile *pidfile)
} }
} }
static ConfSimple reasons;
static string reasonsfile;
void addIdxReason(string who, string reason)
{
reason = neutchars(reason, "\r\n");
if (!reasons.set(who, reason)) {
cerr << "addIdxReason: confsimple set failed\n";
}
}
static void flushIdxReasons()
{
if (reasonsfile.empty())
return;
if (reasonsfile == "stdout") {
reasons.write(cout);
} else if (reasonsfile == "stderr") {
reasons.write(cerr);
} else {
ofstream out;
try {
out.open(reasonsfile, ofstream::out|ofstream::trunc);
reasons.write(out);
} catch (...) {
cerr << "Could not write reasons file " << reasonsfile << endl;
reasons.write(cerr);
}
}
}
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
string a_config; string a_config;
@ -449,6 +479,8 @@ int main(int argc, char **argv)
selpatterns.push_back(*(++argv)); selpatterns.push_back(*(++argv));
argc--; goto b1; argc--; goto b1;
case 'r': op_flags |= OPT_r; break; case 'r': op_flags |= OPT_r; break;
case 'R': op_flags |= OPT_R; if (argc < 2) Usage();
reasonsfile = *(++argv); argc--; goto b1;
case 's': op_flags |= OPT_s; break; case 's': op_flags |= OPT_s; break;
#ifdef RCL_USE_ASPELL #ifdef RCL_USE_ASPELL
case 'S': op_flags |= OPT_S; break; case 'S': op_flags |= OPT_S; break;
@ -481,11 +513,14 @@ int main(int argc, char **argv)
if ((op_flags & OPT_E) && (op_flags & ~(OPT_E|OPT_c))) { if ((op_flags & OPT_E) && (op_flags & ~(OPT_E|OPT_c))) {
Usage(); Usage();
} }
string reason; string reason;
RclInitFlags flags = (op_flags & OPT_m) && !(op_flags&OPT_D) ? RclInitFlags flags = (op_flags & OPT_m) && !(op_flags&OPT_D) ?
RCLINIT_DAEMON : RCLINIT_IDX; RCLINIT_DAEMON : RCLINIT_IDX;
config = recollinit(flags, cleanup, sigcleanup, reason, &a_config); config = recollinit(flags, cleanup, sigcleanup, reason, &a_config);
if (config == 0 || !config->ok()) { if (config == 0 || !config->ok()) {
addIdxReason("init", reason);
flushIdxReasons();
cerr << "Configuration problem: " << reason << endl; cerr << "Configuration problem: " << reason << endl;
exit(1); exit(1);
} }
@ -494,8 +529,11 @@ int main(int argc, char **argv)
#endif #endif
vector<string> nonexist; vector<string> nonexist;
if (!checktopdirs(config, nonexist)) if (!checktopdirs(config, nonexist)) {
addIdxReason("init", "topdirs not set");
flushIdxReasons();
exit(1); exit(1);
}
if (nonexist.size()) { if (nonexist.size()) {
ostream& out = (op_flags & OPT_E) ? cout : cerr; ostream& out = (op_flags & OPT_E) ? cout : cerr;
@ -515,14 +553,17 @@ int main(int argc, char **argv)
string rundir; string rundir;
config->getConfParam("idxrundir", rundir); config->getConfParam("idxrundir", rundir);
if (!rundir.compare("tmp")) { if (!rundir.compare("tmp")) {
LOGINFO("recollindex: changing current directory to [" << (tmplocation()) << "]\n" ); LOGINFO("recollindex: changing current directory to [" <<
tmplocation() << "]\n");
if (chdir(tmplocation().c_str()) < 0) { if (chdir(tmplocation().c_str()) < 0) {
LOGERR("chdir(" << (tmplocation()) << ") failed, errno " << (errno) << "\n" ); LOGERR("chdir(" << tmplocation() << ") failed, errno " << errno <<
"\n");
} }
} else if (!rundir.empty()) { } else if (!rundir.empty()) {
LOGINFO("recollindex: changing current directory to [" << (rundir) << "]\n" ); LOGINFO("recollindex: changing current directory to [" << rundir <<
"]\n");
if (chdir(rundir.c_str()) < 0) { if (chdir(rundir.c_str()) < 0) {
LOGERR("chdir(" << (rundir) << ") failed, errno " << (errno) << "\n" ); LOGERR("chdir(" << rundir << ") failed, errno " << errno << "\n");
} }
} }
@ -548,7 +589,7 @@ int main(int argc, char **argv)
LOGINFO("recollindex: starting up\n"); LOGINFO("recollindex: starting up\n");
#ifndef _WIN32 #ifndef _WIN32
if (setpriority(PRIO_PROCESS, 0, 20) != 0) { if (setpriority(PRIO_PROCESS, 0, 20) != 0) {
LOGINFO("recollindex: can't setpriority(), errno " << (errno) << "\n" ); LOGINFO("recollindex: can't setpriority(), errno " << errno << "\n");
} }
// Try to ionice. This does not work on all platforms // Try to ionice. This does not work on all platforms
rclIxIonice(config); rclIxIonice(config);
@ -559,8 +600,11 @@ int main(int argc, char **argv)
Usage(); Usage();
string top = *argv++; argc--; string top = *argv++; argc--;
bool status = recursive_index(config, top, selpatterns); bool status = recursive_index(config, top, selpatterns);
if (confindexer && !confindexer->getReason().empty()) if (confindexer && !confindexer->getReason().empty()) {
addIdxReason("indexer", confindexer->getReason());
cerr << confindexer->getReason() << endl; cerr << confindexer->getReason() << endl;
}
flushIdxReasons();
exit(status ? 0 : 1); exit(status ? 0 : 1);
} else if (op_flags & (OPT_i|OPT_e)) { } else if (op_flags & (OPT_i|OPT_e)) {
lockorexit(&pidfile); lockorexit(&pidfile);
@ -591,8 +635,11 @@ int main(int argc, char **argv)
if (status && (op_flags & OPT_i)) { if (status && (op_flags & OPT_i)) {
status = indexfiles(config, filenames); status = indexfiles(config, filenames);
} }
if (confindexer && !confindexer->getReason().empty()) if (confindexer && !confindexer->getReason().empty()) {
addIdxReason("indexer", confindexer->getReason());
cerr << confindexer->getReason() << endl; cerr << confindexer->getReason() << endl;
}
flushIdxReasons();
exit(status ? 0 : 1); exit(status ? 0 : 1);
} else if (op_flags & OPT_l) { } else if (op_flags & OPT_l) {
if (argc != 0) if (argc != 0)
@ -623,8 +670,10 @@ int main(int argc, char **argv)
LOGDEB("recollindex: daemonizing\n"); LOGDEB("recollindex: daemonizing\n");
#ifndef _WIN32 #ifndef _WIN32
if (daemon(0,0) != 0) { if (daemon(0,0) != 0) {
fprintf(stderr, "daemon() failed, errno %d\n", errno); addIdxReason("monitor", "daemon() failed");
LOGERR("daemon() failed, errno " << (errno) << "\n" ); cerr << "daemon() failed, errno " << errno << endl;
LOGERR("daemon() failed, errno " << errno << "\n");
flushIdxReasons();
exit(1); exit(1);
} }
#endif #endif
@ -635,14 +684,14 @@ int main(int argc, char **argv)
// Not too sure if I have to redo the nice thing after daemon(), // Not too sure if I have to redo the nice thing after daemon(),
// can't hurt anyway (easier than testing on all platforms...) // can't hurt anyway (easier than testing on all platforms...)
if (setpriority(PRIO_PROCESS, 0, 20) != 0) { if (setpriority(PRIO_PROCESS, 0, 20) != 0) {
LOGINFO("recollindex: can't setpriority(), errno " << (errno) << "\n" ); LOGINFO("recollindex: can't setpriority(), errno " << errno<< "\n");
} }
// Try to ionice. This does not work on all platforms // Try to ionice. This does not work on all platforms
rclIxIonice(config); rclIxIonice(config);
#endif #endif
if (sleepsecs > 0) { if (sleepsecs > 0) {
LOGDEB("recollindex: sleeping " << (sleepsecs) << "\n" ); LOGDEB("recollindex: sleeping " << sleepsecs << "\n");
for (int i = 0; i < sleepsecs; i++) { for (int i = 0; i < sleepsecs; i++) {
sleep(1); sleep(1);
// Check that x11 did not go away while we were sleeping. // Check that x11 did not go away while we were sleeping.
@ -657,7 +706,9 @@ int main(int argc, char **argv)
LOGDEB("Recollindex: initial indexing pass before monitoring\n"); LOGDEB("Recollindex: initial indexing pass before monitoring\n");
if (!confindexer->index(rezero, ConfIndexer::IxTAll, indexerFlags) if (!confindexer->index(rezero, ConfIndexer::IxTAll, indexerFlags)
|| stopindexing) { || stopindexing) {
LOGERR("recollindex, initial indexing pass failed, not going into monitor mode\n" ); LOGERR("recollindex, initial indexing pass failed, "
"not going into monitor mode\n");
flushIdxReasons();
exit(1); exit(1);
} else { } else {
// Record success of indexing pass with failed files retries. // Record success of indexing pass with failed files retries.
@ -668,7 +719,8 @@ int main(int argc, char **argv)
deleteZ(confindexer); deleteZ(confindexer);
#ifndef _WIN32 #ifndef _WIN32
o_reexec->insertArgs(vector<string>(1, "-n")); o_reexec->insertArgs(vector<string>(1, "-n"));
LOGINFO("recollindex: reexecuting with -n after initial full pass\n" ); LOGINFO("recollindex: reexecuting with -n after initial full "
"pass\n");
// Note that -n will be inside the reexec when we come // Note that -n will be inside the reexec when we come
// back, but the monitor will explicitely strip it before // back, but the monitor will explicitely strip it before
// starting a config change exec to ensure that we do a // starting a config change exec to ensure that we do a
@ -707,14 +759,16 @@ int main(int argc, char **argv)
} }
if (!status) if (!status)
cerr << "Indexing failed" << endl; cerr << "Indexing failed" << endl;
if (!confindexer->getReason().empty()) if (!confindexer->getReason().empty()) {
addIdxReason("indexer", confindexer->getReason());
cerr << confindexer->getReason() << endl; cerr << confindexer->getReason() << endl;
}
if (updater) { if (updater) {
updater->status.phase = DbIxStatus::DBIXS_DONE; updater->status.phase = DbIxStatus::DBIXS_DONE;
updater->status.fn.clear(); updater->status.fn.clear();
updater->update(); updater->update();
} }
flushIdxReasons();
return !status; return !status;
} }
} }

10
src/index/recollindex.h
View File

@ -16,15 +16,21 @@
*/ */
#ifndef _recollindex_h_included_ #ifndef _recollindex_h_included_
#define _recollindex_h_included_ #define _recollindex_h_included_
#include <list>
#include <string>
/** Helper methods in recollindex.cpp for initial checks/setup to index /** Helper methods in recollindex.cpp for initial checks/setup to index
* a list of files (either from the monitor or the command line) */ * a list of files (either from the monitor or the command line) */
extern bool indexfiles(RclConfig *config, list<string> &filenames); class RclConfig;
extern bool purgefiles(RclConfig *config, list<string> &filenames); extern bool indexfiles(RclConfig *config, std::list<std::string> &filenames);
extern bool purgefiles(RclConfig *config, std::list<std::string> &filenames);
extern bool createAuxDbs(RclConfig *config); extern bool createAuxDbs(RclConfig *config);
extern int stopindexing; extern int stopindexing;
// Try to explain what went wrong...
extern void addIdxReason(std::string who, std::string reason);
class ReExec; class ReExec;
extern ReExec *o_reexec; extern ReExec *o_reexec;