tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add recollindex option to write file not indexed reasons to diagnostics file

Browse Source
This commit is contained in:
Jean-Francois Dockes
2021-04-01 10:32:04 +02:00
parent 485a0fc650
commit 4756b1252b
18 changed files with 288 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/index/checkindexed.cpp
+2
View File
@@ -19,6 +19,8 @@
*/
#include "autoconfig.h"
#include "checkindexed.h"
#include <stdio.h>
#include <iostream>
src/index/fsindexer.cpp
+17 -18
View File
@@ -47,6 +47,7 @@
#include "rclinit.h"
#include "extrameta.h"
#include "utf8fn.h"
#include "idxdiags.h"
#if defined(HAVE_POSIX_FADVISE)
#include <unistd.h>
#include <fcntl.h>
@@ -397,8 +398,7 @@ bool FsIndexer::indexFiles(list<string>& files, int flags)
continue;
}
}
if (processone(*it, &stb, FsTreeWalker::FtwRegular) !=
FsTreeWalker::FtwOk) {
if (processone(*it, &stb, FsTreeWalker::FtwRegular) != FsTreeWalker::FtwOk) {
LOGERR("FsIndexer::indexFiles: processone failed\n");
goto out;
}
@@ -560,9 +560,8 @@ void *FsIndexerInternfileWorker(void * fsp)
return (void*)1;
}
LOGDEB0("FsIndexerInternfileWorker: task fn " << tsk->fn << "\n");
if (fip->processonefile(&myconf, tsk->fn, &tsk->statbuf,
tsk->localfields) !=
FsTreeWalker::FtwOk) {
if (fip->processonefile(
&myconf, tsk->fn, &tsk->statbuf, tsk->localfields) != FsTreeWalker::FtwOk) {
LOGERR("FsIndexerInternfileWorker: processone failed\n");
tqp->workerExit();
return (void*)0;
@@ -584,9 +583,8 @@ void *FsIndexerInternfileWorker(void * fsp)
/// Accent and majuscule handling are performed by the db module when doing
/// the actual indexing work. The Rcl::Doc created by internfile()
/// mostly contains pretty raw utf8 data.
FsTreeWalker::Status
FsIndexer::processone(const std::string &fn, const struct PathStat *stp,
FsTreeWalker::CbFlag flg)
FsTreeWalker::Status FsIndexer::processone(
const std::string &fn, const struct PathStat *stp, FsTreeWalker::CbFlag flg)
{
if (m_updater) {
#ifdef IDX_THREADS
@@ -610,7 +608,10 @@ FsIndexer::processone(const std::string &fn, const struct PathStat *stp,
if (flg == FsTreeWalker::FtwDirReturn)
return FsTreeWalker::FtwOk;
}
if (flg == FsTreeWalker::FtwSkipped) {
IdxDiags::theDiags().record(IdxDiags::Skipped, fn);
return FsTreeWalker::FtwOk;
}
#ifdef IDX_THREADS
if (m_haveInternQ) {
InternfileTask *tp = new InternfileTask(fn, stp, m_localfields);
@@ -644,10 +645,9 @@ bool FsIndexer::launchAddOrUpdate(const string& udi, const string& parent_udi,
return m_db->addOrUpdate(udi, parent_udi, doc);
}
FsTreeWalker::Status
FsIndexer::processonefile(RclConfig *config,
const std::string &fn, const struct PathStat *stp,
const map<string, string>& localfields)
FsTreeWalker::Status FsIndexer::processonefile(
RclConfig *config, const std::string &fn, const struct PathStat *stp,
const map<string, string>& localfields)
{
////////////////////
// Check db up to date ? Doing this before file type
@@ -693,7 +693,7 @@ FsIndexer::processonefile(RclConfig *config,
// If noretryfailed is set, check for a file which previously
// failed to index, and avoid re-processing it
if (needupdate && m_noretryfailed && existingDoc &&
!oldsig.empty() && *oldsig.rbegin() == '+') {
!oldsig.empty() && oldsig.back() == '+') {
// Check that the sigs are the same except for the '+'. If the file
// actually changed, we always retry (maybe it was fixed)
string nold = oldsig.substr(0, oldsig.size()-1);
@@ -720,8 +720,7 @@ FsIndexer::processonefile(RclConfig *config,
return FsTreeWalker::FtwOk;
}
LOGDEB0("processone: processing: [" <<
displayableBytes(stp->pst_size) << "] " << fn << "\n");
LOGDEB0("processone: processing: [" << displayableBytes(stp->pst_size) << "] " << fn << "\n");
// Note that we used to do the full path here, but I ended up
// believing that it made more sense to use only the file name
@@ -813,6 +812,7 @@ FsIndexer::processonefile(RclConfig *config,
// myriads of such files, the ext script is executed for them
// and fails every time)
if (fis == FileInterner::FIError) {
IdxDiags::theDiags().record(IdxDiags::Error, fn, doc.ipath);
doc.sig += cstr_plus;
}
@@ -822,8 +822,7 @@ FsIndexer::processonefile(RclConfig *config,
// Add document to database. If there is an ipath, add it
// as a child of the file document.
if (!launchAddOrUpdate(udi, doc.ipath.empty() ?
cstr_null : parent_udi, doc)) {
if (!launchAddOrUpdate(udi, doc.ipath.empty() ? cstr_null : parent_udi, doc)) {
return FsTreeWalker::FtwError;
}
src/index/idxdiags.cpp
+98
View File
@@ -0,0 +1,98 @@
/* Copyright (C) 2021 J.F.Dockes
*
* License: GPL 2.1
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "autoconfig.h"
#include <stdio.h>
#include <mutex>
#include "idxdiags.h"
static std::mutex diagmutex;
class IdxDiags::Internal {
public:
~Internal() {
if (fp) {
fclose(fp);
}
}
FILE *fp{nullptr};
};
IdxDiags::IdxDiags()
{
m = new Internal;
}
IdxDiags::~IdxDiags()
{
delete m;
}
bool IdxDiags::flush()
{
std::unique_lock<std::mutex> lock(diagmutex);
if (m && m->fp) {
return fflush(m->fp) ? false : true;
}
return true;
}
static IdxDiags *theInstance;
IdxDiags& IdxDiags::theDiags()
{
if (nullptr == theInstance) {
theInstance = new IdxDiags;
}
return *theInstance;
}
bool IdxDiags::init(const std::string& outpath)
{
m->fp = fopen(outpath.c_str(), "w");
if (nullptr == m->fp) {
return false;
}
return true;
}
bool IdxDiags::record(DiagKind diag, const std::string& path, const std::string& detail)
{
if (nullptr == m || nullptr == m->fp || (path.empty() && detail.empty())) {
return true;
}
const char *skind = "Unknown";
switch (diag) {
case Ok: skind = "Ok";break;
case Skipped: skind = "Skipped";break;
case NoContentSuffix: skind = "NoContentSuffix";break;
case MissingHelper: skind = "MissingHelper";break;
case Error: skind = "Error";break;
case NoHandler: skind = "NoHandler";break;
case ExcludedMime: skind = "ExcludedMime";break;
case NotIncludedMime: skind = "NotIncludedMime";break;
}
std::unique_lock<std::mutex> lock(diagmutex);
fprintf(m->fp, "%s %s | %s\n", skind, path.c_str(), detail.c_str());
return true;
}
src/index/idxdiags.h
+50
View File
@@ -0,0 +1,50 @@
/* Copyright (C) 2021 J.F.Dockes
*
* License: GPL 2.1
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef _IDXDIAGS_H_INCLUDED_
#define _IDXDIAGS_H_INCLUDED_
#include <string>
class IdxDiags {
public:
enum DiagKind {Ok, Skipped, NoContentSuffix, MissingHelper, Error, NoHandler,
ExcludedMime, NotIncludedMime};
// Retrieve a reference to the single instance.
static IdxDiags& theDiags();
// Initialize, setting the output file path. outpath will be truncated.
// No locking: this must be called from the main thread, before going multithread.
// If init is never called, further calls to record() or flush() will be noops.
bool init(const std::string& outpath);
// Record a reason for a document not to be indexed.
bool record(DiagKind diag, const std::string& path, const std::string& detail = std::string());
bool flush();
class Internal;
private:
Internal *m;
IdxDiags();
~IdxDiags();
};
#endif /* _IDXDIAGS_H_INCLUDED_ */
src/index/rclmonrcv.cpp
+6 -11
View File
@@ -66,20 +66,16 @@ static RclMonitor *makeMonitor();
*/
class WalkCB : public FsTreeWalkerCB {
public:
WalkCB(RclConfig *conf, RclMonitor *mon, RclMonEventQueue *queue,
FsTreeWalker& walker)
: m_config(conf), m_mon(mon), m_queue(queue), m_walker(walker)
{}
WalkCB(RclConfig *conf, RclMonitor *mon, RclMonEventQueue *queue, FsTreeWalker& walker)
: m_config(conf), m_mon(mon), m_queue(queue), m_walker(walker) {}
virtual ~WalkCB() {}
virtual FsTreeWalker::Status
processone(const string &fn, const struct PathStat *st,
FsTreeWalker::CbFlag flg) {
virtual FsTreeWalker::Status processone(
const string &fn, const struct PathStat *st, FsTreeWalker::CbFlag flg) {
MONDEB("rclMonRcvRun: processone " << fn << " m_mon " << m_mon <<
" m_mon->ok " << (m_mon ? m_mon->ok() : false) << std::endl);
if (flg == FsTreeWalker::FtwDirEnter ||
flg == FsTreeWalker::FtwDirReturn) {
if (flg == FsTreeWalker::FtwDirEnter || flg == FsTreeWalker::FtwDirReturn) {
m_config->setKeyDir(fn);
// Set up skipped patterns for this subtree.
m_walker.setSkippedNames(m_config->getSkippedNames());
@@ -106,8 +102,7 @@ public:
m_mon->saved_errno != ENOENT)
return FsTreeWalker::FtwError;
}
} else if (!m_mon->generatesExist() &&
flg == FsTreeWalker::FtwRegular) {
} else if (!m_mon->generatesExist() && flg == FsTreeWalker::FtwRegular) {
// Have to synthetize events for regular files existence
// at startup because the monitor does not do it
// Note 2011-09-29: no sure this is actually needed. We just ran
src/index/recollindex.cpp
+19 -11
View File
@@ -62,6 +62,7 @@ using namespace std;
#include "checkretryfailed.h"
#include "idxstatus.h"
#include "circache.h"
#include "idxdiags.h"
// Command line options
static int op_flags;
@@ -93,11 +94,13 @@ static int op_flags;
#define OPTVAL_WEBCACHE_COMPACT 1000
#define OPTVAL_WEBCACHE_BURST 1001
#define OPTVAL_DIAGS_NOTINDEXED 1002
#define OPTVAL_DIAGS_DIAGFILE 1003
static struct option long_options[] = {
{"webcache-compact", 0, 0, OPTVAL_WEBCACHE_COMPACT},
{"webcache-burst", required_argument, 0, OPTVAL_WEBCACHE_BURST},
{"notindexed", 0, 0, OPTVAL_DIAGS_NOTINDEXED},
{"diagfile", required_argument, 0, OPTVAL_DIAGS_DIAGFILE},
{0, 0, 0, 0}
};
@@ -110,6 +113,7 @@ static ConfIndexer *confindexer;
static void cleanup()
{
deleteZ(confindexer);
IdxDiags::theDiags().flush();
recoll_exitready();
}
@@ -274,20 +278,15 @@ static void setMyPriority(const RclConfig *config)
class MakeListWalkerCB : public FsTreeWalkerCB {
public:
MakeListWalkerCB(list<string>& files, const vector<string>& selpats)
: m_files(files), m_pats(selpats)
{
}
virtual FsTreeWalker::Status
processone(const string& fn, const struct PathStat *,
FsTreeWalker::CbFlag flg) {
: m_files(files), m_pats(selpats) {}
virtual FsTreeWalker::Status processone(
const string& fn, const struct PathStat *, FsTreeWalker::CbFlag flg) {
if (flg== FsTreeWalker::FtwDirEnter || flg == FsTreeWalker::FtwRegular){
if (m_pats.empty()) {
cerr << "Selecting " << fn << endl;
m_files.push_back(fn);
} else {
for (vector<string>::const_iterator it = m_pats.begin();
it != m_pats.end(); it++) {
if (fnmatch(it->c_str(), fn.c_str(), 0) == 0) {
for (const auto& pat : m_pats) {
if (fnmatch(pat.c_str(), fn.c_str(), 0) == 0) {
m_files.push_back(fn);
break;
}
@@ -451,6 +450,8 @@ static const char usage [] =
" -Z : in place reset: consider all documents as changed. Can also\n"
" be combined with -i or -r but not -m\n"
" -k : retry files on which we previously failed\n"
" --diagfile <outputpath> : list skipped or otherwise not indexed documents to <outputpath>\n"
" <outputpath> will be truncated\n"
#ifdef RCL_MONITOR
"recollindex -m [-w <secs>] -x [-D] [-C]\n"
" Perform real time indexing. Don't become a daemon if -D is set.\n"
@@ -636,6 +637,7 @@ int main(int argc, char *argv[])
bool diags_notindexed{false};
std::string burstdir;
std::string diagfile;
while ((ret = getopt_long(argc, (char *const*)&args[0], "c:CDdEefhikKlmnPp:rR:sS:w:xZz",
long_options, NULL)) != -1) {
switch (ret) {
@@ -676,7 +678,7 @@ int main(int argc, char *argv[])
case OPTVAL_WEBCACHE_COMPACT: webcache_compact = true; break;
case OPTVAL_WEBCACHE_BURST: burstdir = optarg; webcache_burst = true;break;
case OPTVAL_DIAGS_NOTINDEXED: diags_notindexed = true;break;
case OPTVAL_DIAGS_DIAGFILE: diagfile = optarg;break;
default: Usage(); break;
}
}
@@ -790,6 +792,12 @@ int main(int argc, char *argv[])
}
}
if (!diagfile.empty()) {
if (!IdxDiags::theDiags().init(diagfile)) {
std::cerr << "Could not initialize diags file " << diagfile << "\n";
LOGERR("recollindex: Could not initialize diags file " << diagfile << "\n");
}
}
bool rezero((op_flags & OPT_z) != 0);
bool inPlaceReset((op_flags & OPT_Z) != 0);
src/index/webqueue.cpp
+1 -2
View File
@@ -411,8 +411,7 @@ WebQueueIndexer::processone(
if (flg != FsTreeWalker::FtwRegular)
return FsTreeWalker::FtwOk;
string dotpath = path_cat(path_getfather(path),
string(DOTFILEPREFIX) + path_getsimple(path));
string dotpath = path_cat(path_getfather(path), string(DOTFILEPREFIX) + path_getsimple(path));
LOGDEB("WebQueueIndexer: prc1: [" << path << "]\n");
WebQueueDotFile dotfile(m_config, dotpath);