tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

improve indexing status reporting

Browse Source
This commit is contained in:
dockes
2006-04-12 10:41:39 +00:00
parent c9cf0c9460
commit 9086c6e531
8 changed files with 123 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/index/indexer.cpp
+41 -13
View File
@@ -1,5 +1,5 @@
#ifndef lint
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.30 2006-04-04 13:49:54 dockes Exp $ (C) 2004 J.F.Dockes";
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.31 2006-04-12 10:41:39 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
@@ -68,6 +68,11 @@ bool DbIndexer::indexDb(bool resetbefore, list<string> *topdirs)
if (!init(resetbefore))
return false;
if (m_updater) {
m_updater->status.reset();
m_updater->status.dbtotdocs = m_db.docCnt();
}
for (list<string>::const_iterator it = topdirs->begin();
it != topdirs->end(); it++) {
LOGDEB(("DbIndexer::index: Indexing %s into %s\n", it->c_str(),
@@ -94,6 +99,11 @@ bool DbIndexer::indexDb(bool resetbefore, list<string> *topdirs)
return false;
}
}
if (m_updater) {
m_updater->status.fn.clear();
m_updater->status.phase = DbIxStatus::DBIXS_PURGE;
m_updater->update();
}
// Get rid of all database entries that don't exist in the
// filesystem anymore.
@@ -115,11 +125,21 @@ bool DbIndexer::indexDb(bool resetbefore, list<string> *topdirs)
m_db.deleteStemDb(*it);
}
for (it = langs.begin(); it != langs.end(); it++) {
if (m_updater) {
m_updater->status.phase = DbIxStatus::DBIXS_STEMDB;
m_updater->status.fn = *it;
m_updater->update();
}
m_db.createStemDb(*it);
}
}
// The close would be done in our destructor, but we want status here
if (m_updater) {
m_updater->status.phase = DbIxStatus::DBIXS_CLOSING;
m_updater->status.fn.clear();
m_updater->update();
}
if (!m_db.close()) {
LOGERR(("DbIndexer::index: error closing database in %s\n",
m_dbdir.c_str()));
@@ -200,10 +220,8 @@ FsTreeWalker::Status
DbIndexer::processone(const std::string &fn, const struct stat *stp,
FsTreeWalker::CbFlag flg)
{
if (m_updfunc) {
if (!m_updfunc->update(fn)) {
if (m_updater && !m_updater->update()) {
return FsTreeWalker::FtwStop;
}
}
// If we're changing directories, possibly adjust parameters (set
// the current directory in configuration object)
@@ -217,15 +235,20 @@ DbIndexer::processone(const std::string &fn, const struct stat *stp,
// identification means that, if usesystemfilecommand is switched
// from on to off it may happen that some files which are now
// without mime type will not be purged from the db, resulting
// into possible 'cannot intern file' messages at query time...
// in possible 'cannot intern file' messages at query time...
if (!m_db.needUpdate(fn, stp)) {
LOGDEB(("indexfile: up to date: %s\n", fn.c_str()));
if (m_updater) {
m_updater->status.fn = fn;
if (!m_updater->update()) {
return FsTreeWalker::FtwStop;
}
}
return FsTreeWalker::FtwOk;
}
FileInterner interner(fn, m_config, m_tmpdir);
FileInterner::Status fis = FileInterner::FIAgain;
int i = 0;
while (fis == FileInterner::FIAgain) {
Rcl::Doc doc;
string ipath;
@@ -233,12 +256,6 @@ DbIndexer::processone(const std::string &fn, const struct stat *stp,
if (fis == FileInterner::FIError)
break;
if (m_updfunc) {
if ((++i % 100) == 0 && !m_updfunc->update(fn+"|"+ipath)) {
return FsTreeWalker::FtwStop;
}
}
// Set the date if this was not done in the document handler
if (doc.fmtime.empty()) {
char ascdate[20];
@@ -257,6 +274,17 @@ DbIndexer::processone(const std::string &fn, const struct stat *stp,
// Do database-specific work to update document data
if (!m_db.add(fn, doc, stp))
return FsTreeWalker::FtwError;
if (m_updater) {
if ((++(m_updater->status.docsdone) % 10) == 0) {
m_updater->status.fn = fn;
if (!ipath.empty())
m_updater->status.fn += "|" + ipath;
if (!m_updater->update()) {
return FsTreeWalker::FtwStop;
}
}
}
}
return FsTreeWalker::FtwOk;
@@ -340,7 +368,7 @@ bool ConfIndexer::index(bool resetbefore)
// cout << *dit << " ";
//}
//cout << endl;
m_dbindexer = new DbIndexer(m_config, dbit->first, m_updfunc);
m_dbindexer = new DbIndexer(m_config, dbit->first, m_updater);
if (!m_dbindexer->indexDb(resetbefore, &dbit->second)) {
deleteZ(m_dbindexer);
m_reason = "Failed indexing in " + dbit->first;
src/index/indexer.h
+31 -14
View File
@@ -16,11 +16,16 @@
*/
#ifndef _INDEXER_H_INCLUDED_
#define _INDEXER_H_INCLUDED_
/* @(#$Id: indexer.h,v 1.14 2006-04-04 13:49:54 dockes Exp $ (C) 2004 J.F.Dockes */
/* @(#$Id: indexer.h,v 1.15 2006-04-12 10:41:39 dockes Exp $ (C) 2004 J.F.Dockes */
#include <string>
#include <list>
#ifndef NO_NAMESPACES
using std::string;
using std::list;
#endif
#include "rclconfig.h"
#include "fstreewalk.h"
#include "rcldb.h"
@@ -28,12 +33,24 @@
/* Forward decl for lower level indexing object */
class DbIndexer;
/* Callback to say what we're doing. If the update func returns false, we
* stop */
class DbIxStatus {
public:
enum Phase {DBIXS_FILES, DBIXS_PURGE, DBIXS_STEMDB, DBIXS_CLOSING};
Phase phase;
string fn; // Last file processed
int docsdone; // Documents processed
int dbtotdocs; // Doc count in index at start
void reset() {phase = DBIXS_FILES;fn.clear();docsdone=dbtotdocs=0;}
DbIxStatus() {reset();}
};
/** Callback to say what we're doing. If the update func returns false, we
* stop as soon as possible without corrupting state */
class DbIxStatusUpdater {
public:
DbIxStatus status;
virtual ~DbIxStatusUpdater(){}
virtual bool update(const std::string &) = 0;
virtual bool update() = 0;
};
/**
@@ -49,7 +66,7 @@ class ConfIndexer {
public:
enum runStatus {IndexerOk, IndexerError};
ConfIndexer(RclConfig *cnf, DbIxStatusUpdater *updfunc = 0)
: m_config(cnf), m_dbindexer(0), m_updfunc(updfunc)
: m_config(cnf), m_dbindexer(0), m_updater(updfunc)
{}
virtual ~ConfIndexer();
/** Worker function: doe the actual indexing */
@@ -58,7 +75,7 @@ class ConfIndexer {
private:
RclConfig *m_config;
DbIndexer *m_dbindexer; // Object to process directories for a given db
DbIxStatusUpdater *m_updfunc;
DbIxStatusUpdater *m_updater;
string m_reason;
};
@@ -76,10 +93,10 @@ class DbIndexer : public FsTreeWalkerCB {
public:
/** Constructor does nothing but store parameters */
DbIndexer(RclConfig *cnf, // Configuration data
const std::string &dbd, // Place where the db lives
const string &dbd, // Place where the db lives
DbIxStatusUpdater *updfunc = 0 // status updater callback
)
: m_config(cnf), m_dbdir(dbd), m_updfunc(updfunc) {
: m_config(cnf), m_dbdir(dbd), m_updater(updfunc) {
}
virtual ~DbIndexer();
@@ -94,26 +111,26 @@ class DbIndexer : public FsTreeWalkerCB {
When walking is done, we create the stem databases and close
the main db.
*/
bool indexDb(bool resetbefore, std::list<std::string> *topdirs);
bool indexDb(bool resetbefore, std::list<string> *topdirs);
/** Index a list of files. No db cleaning or stemdb updating */
bool indexFiles(const std::list<std::string> &files);
bool indexFiles(const std::list<string> &files);
/** Create stem database for given language */
bool createStemDb(const string &lang);
/** Tree walker callback method */
FsTreeWalker::Status
processone(const std::string &, const struct stat *,
processone(const string &, const struct stat *,
FsTreeWalker::CbFlag);
private:
FsTreeWalker m_walker;
RclConfig *m_config;
std::string m_dbdir;
string m_dbdir;
Rcl::Db m_db;
std::string m_tmpdir;
DbIxStatusUpdater *m_updfunc;
string m_tmpdir;
DbIxStatusUpdater *m_updater;
bool init(bool rst = false);
};