tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Get rid of the GUI indexing thread. All indexing now done by recollindex, with start/stop from the GUI

Browse Source
This commit is contained in:
Jean-Francois Dockes
2012-02-07 17:07:07 +01:00
parent f59e2e033a
commit fccc0ac09c
14 changed files with 237 additions and 350 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/qtgui/confgui/confguiindex.cpp
-5
View File
@@ -33,7 +33,6 @@ using std::list;
#include "confgui.h"
#include "recoll.h"
#include "idxthread.h"
#include "confguiindex.h"
#include "smallut.h"
#include "debuglog.h"
@@ -85,10 +84,6 @@ void ConfIndexW::acceptChanges()
m_rclconf->updateMainConfig();
snapshotConfig();
if (startIndexingAfterConfig) {
startIndexingAfterConfig = 0;
start_indexing(true);
}
hide();
}
src/qtgui/idxthread.cpp
-190
View File
@@ -1,190 +0,0 @@
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <qthread.h>
#include <qmutex.h>
#include <qwaitcondition.h>
#include "indexer.h"
#include "debuglog.h"
#include "idxthread.h"
#include "smallut.h"
#include "rclinit.h"
#include "pathut.h"
#include "recoll.h"
static int stopindexing;
static int startindexing;
static bool rezero;
static vector<Rcl::Doc> idxdocs; // Docs to update. Update all if empty
static IdxThreadStatus indexingstatus = IDXTS_OK;
static string indexingReason;
static int stopidxthread;
static QMutex curfile_mutex;
static QMutex action_mutex;
static QWaitCondition action_wait;
class IdxThread : public QThread , public DbIxStatusUpdater {
virtual void run();
public:
// This gets called at intervals by the file system walker to check for
// a requested interrupt and update the current status.
virtual bool update() {
QMutexLocker locker(&curfile_mutex);
m_statusSnap = status;
LOGDEB1(("IdxThread::update: indexing %s\n", m_statusSnap.fn.c_str()));
if (stopindexing) {
stopindexing = 0;
m_interrupted = true;
return false;
}
return true;
}
// Maintain a copy/snapshot of idx status
DbIxStatus m_statusSnap;
bool m_interrupted;
};
void IdxThread::run()
{
recoll_threadinit();
action_mutex.lock();
for (;;) {
if (startindexing) {
startindexing = 0;
action_mutex.unlock();
m_interrupted = false;
indexingstatus = IDXTS_NULL;
// We make a private snapshot of the config: setKeydir changes
// it during indexing and it may be updated by the main thread.
RclConfig *myconf;
{
PTMutexLocker locker(thestableconfiglock);
myconf = new RclConfig(*thestableconfig);
}
int loglevel;
myconf->setKeyDir("");
myconf->getConfParam("loglevel", &loglevel);
DebugLog::getdbl()->setloglevel(loglevel);
Pidfile pidfile(myconf->getPidfile());
if (pidfile.open() != 0) {
// Have to sleep a little here else the main thread
// won't see the status change because it only tests
// after seeing IDXTS_NULL at least once. Better
// interlock needed...
sleep(2);
indexingstatus = IDXTS_ERROR;
indexingReason = "Indexing failed: other process active? " +
pidfile.getreason();
} else {
pidfile.write_pid();
ConfIndexer *indexer = new ConfIndexer(myconf, this);
bool status = idxdocs.empty() ?
indexer->index(rezero, ConfIndexer::IxTAll) :
indexer->updateDocs(idxdocs);
if (status) {
indexingstatus = IDXTS_OK;
indexingReason = "";
} else {
indexingstatus = IDXTS_ERROR;
indexingReason = "Indexing failed: " + indexer->getReason();
}
pidfile.close();
delete indexer;
}
delete myconf;
myconf = 0;
rezero = false;
action_mutex.lock();
}
if (stopidxthread) {
action_mutex.unlock();
return;
}
action_wait.wait(&action_mutex);
}
}
static IdxThread idxthread;
// Functions called by the main thread
void start_idxthread()
{
idxthread.start();
}
void stop_idxthread()
{
action_mutex.lock();
startindexing = 0;
stopindexing = 1;
stopidxthread = 1;
action_mutex.unlock();
action_wait.wakeAll();
idxthread.wait();
}
void stop_indexing()
{
action_mutex.lock();
startindexing = 0;
stopindexing = 1;
action_mutex.unlock();
action_wait.wakeAll();
}
void start_indexing(bool raz, vector<Rcl::Doc> docs)
{
action_mutex.lock();
startindexing = 1;
rezero = raz;
idxdocs = docs;
action_mutex.unlock();
action_wait.wakeAll();
}
DbIxStatus idxthread_idxStatus()
{
QMutexLocker locker(&curfile_mutex);
return idxthread.m_statusSnap;
}
bool idxthread_idxInterrupted()
{
return idxthread.m_interrupted;
}
string idxthread_getReason()
{
return indexingReason;
}
IdxThreadStatus idxthread_getStatus()
{
return indexingstatus;
}
src/qtgui/idxthread.h
-45
View File
@@ -1,45 +0,0 @@
/* Copyright (C) 2004 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef _IDXTHREAD_H_INCLUDED_
#define _IDXTHREAD_H_INCLUDED_
#include <string>
#include "indexer.h"
#include "rcldoc.h"
// These two deal with starting / stopping the thread itself, not
// indexing sessions.
// cnf will be cloned each time we start an indexing pass. The pointer must
// stay valid for the whole program duration.
extern void start_idxthread();
extern void stop_idxthread();
// Use these to to request action from thread
extern void start_indexing(bool rezero = false,
const vector<Rcl::Doc> docs = vector<Rcl::Doc>());
extern void stop_indexing();
// Final status of indexing. indexingstatus is NULL iff indexing is
// currently in progress.
enum IdxThreadStatus {IDXTS_NULL = 0, IDXTS_OK = 1, IDXTS_ERROR = 2};
extern IdxThreadStatus idxthread_getStatus();
extern string idxthread_getReason();
// Current status of running indexing (phase, file name etc.)
extern DbIxStatus idxthread_idxStatus();
// Did last op fail because of stop request ?
extern bool idxthread_idxInterrupted();
#endif /* _IDXTHREAD_H_INCLUDED_ */
src/qtgui/main.cpp
-5
View File
@@ -97,7 +97,6 @@ Aspell *aspell;
#endif
int recollNeedsExit;
int startIndexingAfterConfig;
RclMain *mainWindow;
void startManual(const string& helpindex)
@@ -352,10 +351,6 @@ int main(int argc, char **argv)
app.connect(&app, SIGNAL(lastWindowClosed()), &app, SLOT(quit()));
app.connect(&app, SIGNAL(aboutToQuit()), mainWindow, SLOT(close()));
// Start the indexing thread. It will immediately go to sleep waiting for
// something to do.
start_idxthread();
mainWindow->sSearch->searchTypCMB->setCurrentIndex(prefs.ssearchTyp);
mainWindow->sSearch->searchTypeChanged(prefs.ssearchTyp);
if (op_flags & OPT_q) {
src/qtgui/rclmain_w.cpp
+142 -76
View File
@@ -48,6 +48,7 @@ using std::pair;
#include <qapplication.h>
#include <qcursor.h>
#include <qevent.h>
#include <QFileSystemWatcher>
#include "recoll.h"
#include "debuglog.h"
@@ -73,6 +74,7 @@ using std::pair;
#include "idxsched.h"
#include "crontool.h"
#include "rtitool.h"
#include "indexer.h"
using namespace confgui;
@@ -97,7 +99,8 @@ void RclMain::init()
(const char *)tr("filtered").toUtf8());
periodictimer = new QTimer(this);
m_watcher.addPath(QString::fromLocal8Bit(
theconfig->getIdxStatusFile().c_str()));
// At least some versions of qt4 don't display the status bar if
// it's not created here.
(void)statusBar();
@@ -203,6 +206,8 @@ void RclMain::init()
connect(sc, SIGNAL (activated()),
this, SLOT (focusToSearch()));
connect(&m_watcher, SIGNAL(fileChanged(QString)),
this, SLOT(idxStatus()));
connect(sSearch, SIGNAL(startSearch(RefCntr<Rcl::SearchData>)),
this, SLOT(startSearch(RefCntr<Rcl::SearchData>)));
connect(sSearch, SIGNAL(clearSearch()),
@@ -473,82 +478,117 @@ void RclMain::fileExit()
// We'd prefer to do this in the exit handler, but it's apparently to late
// and some of the qt stuff is already dead at this point?
LOGDEB2(("RclMain::fileExit() : stopping idx thread\n"));
stop_idxthread();
// Do we want to stop an ongoing index operation here ?
// I guess not. We did use to cancel the indexing thread.
// Let the exit handler clean up the rest (internal recoll stuff).
exit(0);
}
// This is called by a periodic timer to check the status of the
// indexing thread and a possible need to exit
void RclMain::idxStatus()
{
ConfSimple cs(theconfig->getIdxStatusFile().c_str(), 1);
QString msg = tr("Indexing in progress: ");
DbIxStatus status;
string val;
cs.get("phase", val);
status.phase = DbIxStatus::Phase(atoi(val.c_str()));
cs.get("fn", status.fn);
cs.get("docsdone", val);
status.docsdone = atoi(val.c_str());
cs.get("filesdone", val);
status.filesdone = atoi(val.c_str());
cs.get("dbtotdocs", val);
status.dbtotdocs = atoi(val.c_str());
QString phs;
switch (status.phase) {
case DbIxStatus::DBIXS_NONE:phs=tr("None");break;
case DbIxStatus::DBIXS_FILES: phs=tr("Updating");break;
case DbIxStatus::DBIXS_PURGE: phs=tr("Purge");break;
case DbIxStatus::DBIXS_STEMDB: phs=tr("Stemdb");break;
case DbIxStatus::DBIXS_CLOSING:phs=tr("Closing");break;
case DbIxStatus::DBIXS_DONE:phs=tr("Done");break;
case DbIxStatus::DBIXS_MONITOR:phs=tr("Monitor");break;
default: phs=tr("Unknown");break;
}
msg += phs + " ";
if (status.phase == DbIxStatus::DBIXS_FILES) {
char cnts[100];
if (status.dbtotdocs > 0)
sprintf(cnts,"(%d/%d/%d) ", status.docsdone, status.filesdone,
status.dbtotdocs);
else
sprintf(cnts, "(%d/%d) ", status.docsdone, status.filesdone);
msg += QString::fromAscii(cnts) + " ";
}
string mf;int ecnt = 0;
string fcharset = theconfig->getDefCharset(true);
if (!transcode(status.fn, mf, fcharset, "UTF-8", &ecnt) || ecnt) {
mf = url_encode(status.fn, 0);
}
msg += QString::fromUtf8(mf.c_str());
statusBar()->showMessage(msg, 4000);
}
// This is called by a periodic timer to check the status of
// indexing, a possible need to exit, and cleanup exited viewers
void RclMain::periodic100()
{
// Check if indexing thread done
if (idxthread_getStatus() != IDXTS_NULL) {
// Indexing is stopped
fileToggleIndexingAction->setText(tr("Update &Index"));
fileToggleIndexingAction->setEnabled(TRUE);
if (m_idxStatusAck == false) {
m_idxStatusAck = true;
if (idxthread_getStatus() != IDXTS_OK) {
if (idxthread_idxInterrupted()) {
QMessageBox::warning(0, "Recoll",
tr("Indexing interrupted"));
} else {
QMessageBox::warning(0, "Recoll",
QString::fromAscii(idxthread_getReason().c_str()));
}
LOGDEB2(("Periodic100\n"));
if (m_idxproc) {
// An indexing process was launched. If its' done, see status.
int status;
bool exited = m_idxproc->maybereap(&status);
if (exited) {
deleteZ(m_idxproc);
if (status) {
QMessageBox::warning(0, "Recoll",
tr("Indexing failed"));
}
// Make sure we reopen the db to get the results. If there
// is current search data, we should reset it else things
// are inconsistent (ie: applying sort will fail. But we
// don't like erasing results while the user may be
// looking at them either). Fixing this would be
// relatively complicated (keep an open/close gen number
// and check this / restart query in DocSeqDb() ?)
string reason;
maybeOpenDb(reason, 1);
periodictimer->setInterval(1000);
} else {
// update/show status even if the status file did not
// change (else the status line goes blank during
// lengthy operations).
idxStatus();
}
} else {
// Indexing is running
m_idxStatusAck = false;
}
// Update the "start/stop indexing" menu entry, can't be done from
// the "start/stop indexing" slot itself
if (m_idxproc) {
fileToggleIndexingAction->setText(tr("Stop &Indexing"));
fileToggleIndexingAction->setEnabled(TRUE);
periodictimer->setInterval(100);
// The toggle thing is for the status to flash
if (m_periodicToggle < 9) {
QString msg = tr("Indexing in progress: ");
DbIxStatus status = idxthread_idxStatus();
QString phs;
switch (status.phase) {
case DbIxStatus::DBIXS_FILES: phs=tr("Files");break;
case DbIxStatus::DBIXS_PURGE: phs=tr("Purge");break;
case DbIxStatus::DBIXS_STEMDB: phs=tr("Stemdb");break;
case DbIxStatus::DBIXS_CLOSING:phs=tr("Closing");break;
default: phs=tr("Unknown");break;
}
msg += phs + " ";
if (status.phase == DbIxStatus::DBIXS_FILES) {
char cnts[100];
if (status.dbtotdocs>0)
sprintf(cnts,"(%d/%d) ",status.docsdone, status.dbtotdocs);
else
sprintf(cnts, "(%d) ", status.docsdone);
msg += QString::fromAscii(cnts) + " ";
}
string mf;int ecnt = 0;
string fcharset = theconfig->getDefCharset(true);
if (!transcode(status.fn, mf, fcharset, "UTF-8", &ecnt) || ecnt) {
mf = url_encode(status.fn, 0);
}
msg += QString::fromUtf8(mf.c_str());
statusBar()->showMessage(msg, 4000);
} else if (m_periodicToggle == 9) {
statusBar()->showMessage("");
}
if (++m_periodicToggle >= 10)
m_periodicToggle = 0;
} else {
fileToggleIndexingAction->setText(tr("Update &Index"));
// No indexer of our own runnin, but the real time one may be up, check
// for some other indexer.
Pidfile pidfile(theconfig->getPidfile());
if (pidfile.open() == 0) {
fileToggleIndexingAction->setEnabled(TRUE);
} else {
fileToggleIndexingAction->setEnabled(FALSE);
}
}
// Possibly cleanup the dead viewers
for (vector<ExecCmd*>::iterator it = m_viewers.begin();
it != m_viewers.end(); it++) {
int status;
if ((*it)->maybereap(&status)) {
deleteZ(*it);
}
}
vector<ExecCmd*> v;
for (vector<ExecCmd*>::iterator it = m_viewers.begin();
it != m_viewers.end(); it++) {
if (*it)
v.push_back(*it);
}
m_viewers = v;
if (recollNeedsExit)
fileExit();
}
@@ -558,14 +598,16 @@ void RclMain::periodic100()
// re-enabled by the indexing status check
void RclMain::toggleIndexing()
{
if (idxthread_getStatus() == IDXTS_NULL) {
// Indexing was in progress, stop it
stop_indexing();
periodictimer->setInterval(1000);
fileToggleIndexingAction->setText(tr("Update &Index"));
if (m_idxproc) {
// Indexing was in progress, request stop. Let the periodic
// routine check for the results.
kill(m_idxproc->getChildPid(), SIGTERM);
} else {
start_indexing(false);
periodictimer->setInterval(100);
list<string> args;
args.push_back("-c");
args.push_back(theconfig->getConfDir());
m_idxproc = new ExecCmd;
m_idxproc->startExec("recollindex", args, false, false);
fileToggleIndexingAction->setText(tr("Stop &Indexing"));
}
fileToggleIndexingAction->setEnabled(FALSE);
@@ -574,15 +616,14 @@ void RclMain::toggleIndexing()
// Start a db query and set the reslist docsource
void RclMain::startSearch(RefCntr<Rcl::SearchData> sdata)
{
LOGDEB(("RclMain::startSearch. Indexing %s\n",
idxthread_getStatus() == IDXTS_NULL?"on":"off"));
LOGDEB(("RclMain::startSearch. Indexing %s\n", m_idxproc?"on":"off"));
emit searchReset();
m_source = RefCntr<DocSequence>();
// The db may have been closed at the end of indexing
string reason;
// If indexing is being performed, we reopen the db at each query.
if (!maybeOpenDb(reason, idxthread_getStatus() == IDXTS_NULL)) {
if (!maybeOpenDb(reason, m_idxproc != 0)) {
QMessageBox::critical(0, "Recoll", QString(reason.c_str()));
return;
}
@@ -970,7 +1011,7 @@ void RclMain::startPreview(int docnum, Rcl::Doc doc, int mod)
QMessageBox::warning(0, tr("Warning"),
tr("Index not up to date for this file. "
"Refusing to risk showing the wrong "
"data. Click ok to update the "
"entry. Click Ok to update the "
"index for this file, then re-run the "
"query when indexing is done. "
"Else, Cancel."),
@@ -980,7 +1021,8 @@ void RclMain::startPreview(int docnum, Rcl::Doc doc, int mod)
if (rep == QMessageBox::Ok) {
LOGDEB(("Requesting index update for %s\n",
doc.url.c_str()));
start_indexing(false, vector<Rcl::Doc>(1, doc));
vector<Rcl::Doc> docs(1, doc);
updateIdxForDocs(docs);
}
return;
}
@@ -1023,6 +1065,30 @@ void RclMain::startPreview(int docnum, Rcl::Doc doc, int mod)
curPreview->makeDocCurrent(doc, docnum);
}
void RclMain::updateIdxForDocs(vector<Rcl::Doc>& docs)
{
if (m_idxproc) {
QMessageBox::warning(0, tr("Warning"),
tr("Can't update index: indexer running"),
QMessageBox::Ok,
QMessageBox::NoButton);
return;
}
vector<string> paths;
if (ConfIndexer::docsToPaths(docs, paths)) {
list<string> args;
args.push_back("-c");
args.push_back(theconfig->getConfDir());
args.push_back("-i");
args.insert(args.end(), paths.begin(), paths.end());
m_idxproc = new ExecCmd;
m_idxproc->startExec("recollindex", args, false, false);
fileToggleIndexingAction->setText(tr("Stop &Indexing"));
}
fileToggleIndexingAction->setEnabled(FALSE);
}
/**
* Open a preview window for a given document, no linking to result list
*
src/qtgui/rclmain_w.h
+6 -2
View File
@@ -19,6 +19,7 @@
#include <qvariant.h>
#include <qmainwindow.h>
#include <QFileSystemWatcher>
#include "sortseq.h"
#include "preview_w.h"
@@ -66,7 +67,7 @@ public:
displayingTable(0),
m_idNoStem(0),
m_idAllStem(0),
m_idxStatusAck(false),
m_idxproc(0),
m_sortspecnochange(false),
m_periodicToggle(0)
{
@@ -80,6 +81,7 @@ public:
public slots:
virtual bool close();
virtual void fileExit();
virtual void idxStatus();
virtual void periodic100();
virtual void toggleIndexing();
virtual void startSearch(RefCntr<Rcl::SearchData> sdata);
@@ -155,11 +157,12 @@ private:
bool displayingTable;
QAction *m_idNoStem;
QAction *m_idAllStem;
QFileSystemWatcher m_watcher;
vector<ExecCmd*> m_viewers;
ExecCmd *m_idxproc; // Indexing process
map<QString, QAction*> m_stemLangToId;
vector<string> m_catgbutvec;
bool m_idxStatusAck; // Did we act on last status?
DocSeqFiltSpec m_filtspec;
bool m_sortspecnochange;
DocSeqSortSpec m_sortspec;
@@ -175,6 +178,7 @@ private:
virtual void showIndexSched(bool modal);
virtual void showCronTool(bool modal);
virtual void showRTITool(bool modal);
virtual void updateIdxForDocs(vector<Rcl::Doc>&);
};
#endif // RCLMAIN_W_H
src/qtgui/recoll.h
-2
View File
@@ -20,7 +20,6 @@
#include "rclconfig.h"
#include "rcldb.h"
#include "idxthread.h"
#include "ptmutex.h"
// Misc declarations in need of sharing between the UI files
@@ -41,7 +40,6 @@ extern void forgetTempFile(string &fn);
extern Rcl::Db *rcldb;
extern int recollNeedsExit;
extern int startIndexingAfterConfig; // 1st startup
extern void startManual(const string& helpindex);
#ifdef RCL_USE_ASPELL
src/qtgui/recoll.pro.in
-1
View File
@@ -30,7 +30,6 @@ SOURCES += \
confgui/confguiindex.cpp \
crontool.cpp \
guiutils.cpp \
idxthread.cpp \
main.cpp \
preview_w.cpp \
rclhelp.cpp \