From d0400bdce72f5c3dd5b976752c3c14575488581b Mon Sep 17 00:00:00 2001 From: Jean-Francois Dockes Date: Mon, 11 Apr 2016 17:23:02 +0200 Subject: [PATCH] slightly improve indexing stats by keep a count of file errors and remembering the total files --- src/index/fsindexer.cpp | 9 +++++-- src/index/indexer.h | 10 +++++--- src/index/recollindex.cpp | 54 +++++++++++++++++++-------------------- src/qtgui/rclm_idx.cpp | 24 +++++++++-------- 4 files changed, 53 insertions(+), 44 deletions(-) diff --git a/src/index/fsindexer.cpp b/src/index/fsindexer.cpp index 6b6ea995..7b7dbb77 100644 --- a/src/index/fsindexer.cpp +++ b/src/index/fsindexer.cpp @@ -183,7 +183,6 @@ bool FsIndexer::index(int flags) #ifdef IDX_THREADS PTMutexLocker locker(m_updater->m_mutex); #endif - m_updater->status.reset(); m_updater->status.dbtotdocs = m_db->docCnt(); } @@ -823,8 +822,14 @@ FsIndexer::processonefile(RclConfig *config, if (m_updater->status.dbtotdocs < m_updater->status.docsdone) m_updater->status.dbtotdocs = m_updater->status.docsdone; m_updater->status.fn = fn; - if (!doc.ipath.empty()) + if (!doc.ipath.empty()) { m_updater->status.fn += "|" + doc.ipath; + } else { + if (fis == FileInterner::FIError) { + ++(m_updater->status.fileerrors); + } + ++(m_updater->status.filesdone); + } if (!m_updater->update()) { return FsTreeWalker::FtwStop; } diff --git a/src/index/indexer.h b/src/index/indexer.h index 28dc26a9..9e117a6d 100644 --- a/src/index/indexer.h +++ b/src/index/indexer.h @@ -47,12 +47,16 @@ class DbIxStatus { string fn; // Last file processed int docsdone; // Documents actually updated int filesdone; // Files tested (updated or not) + int fileerrors; // Failed files (e.g.: missing input handler). int dbtotdocs; // Doc count in index at start - void reset() - { + // Total files in index.This is actually difficult to compute from + // the index so it's preserved from last indexing + int totfiles; + + void reset() { phase = DBIXS_FILES; fn.erase(); - docsdone = filesdone = dbtotdocs = 0; + docsdone = filesdone = fileerrors = dbtotdocs = totfiles = 0; } DbIxStatus() {reset();} }; diff --git a/src/index/recollindex.cpp b/src/index/recollindex.cpp index 6f9a452f..7e2193e4 100644 --- a/src/index/recollindex.cpp +++ b/src/index/recollindex.cpp @@ -104,39 +104,38 @@ int stopindexing; class MyUpdater : public DbIxStatusUpdater { public: MyUpdater(const RclConfig *config) - : m_fd(-1), m_stfilename(config->getIdxStatusFile()), + : m_file(config->getIdxStatusFile().c_str()), m_prevphase(DbIxStatus::DBIXS_NONE) { + // The total number of files included in the index is actually + // difficult to compute from the index itself. For display + // purposes, we save it in the status file from indexing to + // indexing (mostly...) + string stf; + if (m_file.get("totfiles", stf)) { + status.totfiles = atoi(stf.c_str()); + } } virtual bool update() { - if (m_fd < 0) { - m_fd = open(m_stfilename.c_str(), - O_WRONLY|O_CREAT|O_TRUNC, 0600); - if (m_fd < 0) { - LOGERR(("Can't open/create status file: [%s]\n", - m_stfilename.c_str())); - return stopindexing ? false : true; - } - } - // Update the status file. Avoid doing it too often - if (status.phase != m_prevphase || m_chron.millis() > 300) { + // Update the status file. Avoid doing it too often. Always do + // it at the end (status DONE) + if (status.phase == DbIxStatus::DBIXS_DONE || + status.phase != m_prevphase || m_chron.millis() > 300) { + if (status.totfiles < status.filesdone) { + status.totfiles = status.filesdone; + } m_prevphase = status.phase; m_chron.restart(); - lseek(m_fd, 0, 0); - int fd1 = dup(m_fd); - FILE *fp = fdopen(fd1, "w"); - fprintf(fp, "phase = %d\n", int(status.phase)); - fprintf(fp, "docsdone = %d\n", status.docsdone); - fprintf(fp, "filesdone = %d\n", status.filesdone); - fprintf(fp, "dbtotdocs = %d\n", status.dbtotdocs); - fprintf(fp, "fn = %s\n", status.fn.c_str()); - if (ftruncate(m_fd, off_t(ftell(fp))) < 0) { - // ? kill compiler warning about ignoring ftruncate return - LOGDEB(("Status update: ftruncate failed\n")); - } - // Flush data and closes fd1. m_fd still valid - fclose(fp); + m_file.holdWrites(true); + m_file.set("phase", int(status.phase)); + m_file.set("docsdone", status.docsdone); + m_file.set("filesdone", status.filesdone); + m_file.set("fileerrors", status.fileerrors); + m_file.set("dbtotdocs", status.dbtotdocs); + m_file.set("totfiles", status.totfiles); + m_file.set("fn", status.fn); + m_file.holdWrites(false); } if (stopindexing) { @@ -158,8 +157,7 @@ class MyUpdater : public DbIxStatusUpdater { } private: - int m_fd; - string m_stfilename; + ConfSimple m_file; Chrono m_chron; DbIxStatus::Phase m_prevphase; }; diff --git a/src/qtgui/rclm_idx.cpp b/src/qtgui/rclm_idx.cpp index f798300b..cc236c28 100644 --- a/src/qtgui/rclm_idx.cpp +++ b/src/qtgui/rclm_idx.cpp @@ -39,12 +39,11 @@ void RclMain::idxStatus() cs.get("phase", val); status.phase = DbIxStatus::Phase(atoi(val.c_str())); cs.get("fn", status.fn); - cs.get("docsdone", val); - status.docsdone = atoi(val.c_str()); - cs.get("filesdone", val); - status.filesdone = atoi(val.c_str()); - cs.get("dbtotdocs", val); - status.dbtotdocs = atoi(val.c_str()); + cs.get("docsdone", &status.docsdone); + cs.get("filesdone", &status.filesdone); + cs.get("fileerrors", &status.fileerrors); + cs.get("dbtotdocs", &status.dbtotdocs); + cs.get("totfiles", &status.totfiles); QString phs; switch (status.phase) { @@ -60,11 +59,14 @@ void RclMain::idxStatus() msg += phs + " "; if (status.phase == DbIxStatus::DBIXS_FILES) { char cnts[100]; - if (status.dbtotdocs > 0) - sprintf(cnts,"(%d/%d/%d) ", status.docsdone, status.filesdone, - status.dbtotdocs); - else - sprintf(cnts, "(%d/%d) ", status.docsdone, status.filesdone); + if (status.dbtotdocs > 0) { + sprintf(cnts, "(%d docs/%d files/%d errors/%d tot files) ", + status.docsdone, status.filesdone, status.fileerrors, + status.totfiles); + } else { + sprintf(cnts, "(%d docs/%d files/%d errors) ", + status.docsdone, status.filesdone, status.fileerrors); + } msg += QString::fromUtf8(cnts) + " "; } string mf;int ecnt = 0;