tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

simplified db open by getting rid of the illusion that we could have several writeable dbs per config

Browse Source
This commit is contained in:
dockes 2008-12-17 08:01:40 +00:00
parent befebb7371
commit b9cc5fb1e0
8 changed files with 70 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/index/indexer.cpp
View File

@ -1,5 +1,5 @@
#ifndef lint #ifndef lint
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.70 2008-10-08 16:15:22 dockes Exp $ (C) 2004 J.F.Dockes"; static char rcsid[] = "@(#$Id: indexer.cpp,v 1.71 2008-12-17 08:01:40 dockes Exp $ (C) 2004 J.F.Dockes";
#endif #endif
/* /*
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
@ -93,7 +93,7 @@ bool DbIndexer::indexDb(bool resetbefore, list<string> *topdirs)
for (list<string>::const_iterator it = topdirs->begin(); for (list<string>::const_iterator it = topdirs->begin();
it != topdirs->end(); it++) { it != topdirs->end(); it++) {
LOGDEB(("DbIndexer::index: Indexing %s into %s\n", it->c_str(), LOGDEB(("DbIndexer::index: Indexing %s into %s\n", it->c_str(),
m_dbdir.c_str())); getDbDir().c_str()));
// Set the current directory in config so that subsequent // Set the current directory in config so that subsequent
// getConfParams() will get local values // getConfParams() will get local values
@ -143,7 +143,7 @@ bool DbIndexer::indexDb(bool resetbefore, list<string> *topdirs)
} }
if (!m_db.close()) { if (!m_db.close()) {
LOGERR(("DbIndexer::index: error closing database in %s\n", LOGERR(("DbIndexer::index: error closing database in %s\n",
m_dbdir.c_str())); getDbDir().c_str()));
return false; return false;
} }
string missing; string missing;
@ -197,8 +197,8 @@ bool DbIndexer::init(bool resetbefore, bool rdonly)
} }
Rcl::Db::OpenMode mode = rdonly ? Rcl::Db::DbRO : Rcl::Db::OpenMode mode = rdonly ? Rcl::Db::DbRO :
resetbefore ? Rcl::Db::DbTrunc : Rcl::Db::DbUpd; resetbefore ? Rcl::Db::DbTrunc : Rcl::Db::DbUpd;
if (!m_db.open(m_dbdir, m_config->getStopfile(), mode)) { if (!m_db.open(mode)) {
LOGERR(("DbIndexer: error opening database in %s\n", m_dbdir.c_str())); LOGERR(("DbIndexer: error opening database %s\n", getDbDir().c_str()));
return false; return false;
} }
@ -250,7 +250,7 @@ bool DbIndexer::createAspellDict()
// The close would be done in our destructor, but we want status here // The close would be done in our destructor, but we want status here
if (!m_db.close()) { if (!m_db.close()) {
LOGERR(("DbIndexer::indexfiles: error closing database in %s\n", LOGERR(("DbIndexer::indexfiles: error closing database in %s\n",
m_dbdir.c_str())); getDbDir().c_str()));
noaspell = true; noaspell = true;
return false; return false;
} }
@ -322,7 +322,7 @@ bool DbIndexer::indexFiles(const list<string> &filenames)
// The close would be done in our destructor, but we want status here // The close would be done in our destructor, but we want status here
if (!m_db.close()) { if (!m_db.close()) {
LOGERR(("DbIndexer::indexfiles: error closing database in %s\n", LOGERR(("DbIndexer::indexfiles: error closing database in %s\n",
m_dbdir.c_str())); getDbDir().c_str()));
return false; return false;
} }
return true; return true;
@ -348,7 +348,7 @@ bool DbIndexer::purgeFiles(const list<string> &filenames)
// The close would be done in our destructor, but we want status here // The close would be done in our destructor, but we want status here
if (!m_db.close()) { if (!m_db.close()) {
LOGERR(("DbIndexer::purgefiles: error closing database in %s\n", LOGERR(("DbIndexer::purgefiles: error closing database in %s\n",
m_dbdir.c_str())); getDbDir().c_str()));
return false; return false;
} }
return true; return true;

23
src/index/indexer.h
View File

@ -16,7 +16,7 @@
*/ */
#ifndef _INDEXER_H_INCLUDED_ #ifndef _INDEXER_H_INCLUDED_
#define _INDEXER_H_INCLUDED_ #define _INDEXER_H_INCLUDED_
/* @(#$Id: indexer.h,v 1.26 2008-10-08 16:15:22 dockes Exp $ (C) 2004 J.F.Dockes */ /* @(#$Id: indexer.h,v 1.27 2008-12-17 08:01:40 dockes Exp $ (C) 2004 J.F.Dockes */
#include <string> #include <string>
#include <list> #include <list>
@ -56,11 +56,14 @@ class DbIxStatusUpdater {
/** /**
The top level indexing object. Processes the configuration, then invokes The top level indexing object. Processes the configuration, then invokes
file system walking to populate/update the database(s). file system walking to populate/update the database(s).
Multiple top-level directories can be listed in the Fiction:
configuration. Each can be indexed to a different Multiple top-level directories can be listed in the
database. Directories are first grouped by database, then an configuration. Each can be indexed to a different
internal class (DbIndexer) is used to process each group. database. Directories are first grouped by database, then an
internal class (DbIndexer) is used to process each group.
Fact: we've had one db per config forever. The multidb/config code has been
kept around for no good reason, this fiction only affects indexer.cpp
*/ */
class ConfIndexer { class ConfIndexer {
public: public:
@ -93,10 +96,11 @@ class DbIndexer : public FsTreeWalkerCB {
public: public:
/** Constructor does nothing but store parameters */ /** Constructor does nothing but store parameters */
DbIndexer(RclConfig *cnf, // Configuration data DbIndexer(RclConfig *cnf, // Configuration data
const string &dbd, // Place where the db lives // Db dir not used anymore, rcl::db gets it from the cfg
const string &,
DbIxStatusUpdater *updfunc = 0 // status updater callback DbIxStatusUpdater *updfunc = 0 // status updater callback
) )
: m_config(cnf), m_dbdir(dbd), m_updater(updfunc) : m_config(cnf), m_db(cnf), m_updater(updfunc)
{} {}
virtual ~DbIndexer(); virtual ~DbIndexer();
@ -134,7 +138,7 @@ class DbIndexer : public FsTreeWalkerCB {
FsTreeWalker::CbFlag); FsTreeWalker::CbFlag);
/** Return my db dir */ /** Return my db dir */
string getDbDir() {return m_dbdir;} string getDbDir() {return m_config->getDbDir();}
/** List possible stemmer names */ /** List possible stemmer names */
static list<string> getStemmerNames(); static list<string> getStemmerNames();
@ -142,7 +146,6 @@ class DbIndexer : public FsTreeWalkerCB {
private: private:
FsTreeWalker m_walker; FsTreeWalker m_walker;
RclConfig *m_config; RclConfig *m_config;
string m_dbdir;
Rcl::Db m_db; Rcl::Db m_db;
string m_tmpdir; string m_tmpdir;
DbIxStatusUpdater *m_updater; DbIxStatusUpdater *m_updater;

8
src/kde/kioslave/recoll/kio_recoll.cpp
View File

@ -1,5 +1,5 @@
#ifndef lint #ifndef lint
static char rcsid[] = "@(#$Id: kio_recoll.cpp,v 1.25 2008-12-16 17:28:10 dockes Exp $ (C) 2005 J.F.Dockes"; static char rcsid[] = "@(#$Id: kio_recoll.cpp,v 1.26 2008-12-17 08:01:40 dockes Exp $ (C) 2005 J.F.Dockes";
#endif #endif
/* /*
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
@ -77,7 +77,7 @@ RecollProtocol::RecollProtocol(const QByteArray &pool, const QByteArray &app)
return; return;
} }
m_rcldb = new Rcl::Db; m_rcldb = new Rcl::Db(o_rclconfig);
if (!m_rcldb) { if (!m_rcldb) {
m_reason = "Could not build database object. (out of memory ?)"; m_reason = "Could not build database object. (out of memory ?)";
return; return;
@ -112,9 +112,7 @@ bool RecollProtocol::maybeOpenDb(string &reason)
reason = "Internal error: initialization error"; reason = "Internal error: initialization error";
return false; return false;
} }
if (!m_rcldb->isopen() && !m_rcldb->open(o_rclconfig->getDbDir(), if (!m_rcldb->isopen() && !m_rcldb->open(Rcl::Db::DbRO)) {
o_rclconfig->getStopfile(),
Rcl::Db::DbRO)) {
reason = "Could not open database in " + o_rclconfig->getDbDir(); reason = "Could not open database in " + o_rclconfig->getDbDir();
return false; return false;
} }

9
src/python/recoll/pyrecoll.cpp
View File

@ -1,5 +1,5 @@
#ifndef lint #ifndef lint
static char rcsid[] = "@(#$Id: pyrecoll.cpp,v 1.20 2008-10-10 08:18:27 dockes Exp $ (C) 2007 J.F.Dockes"; static char rcsid[] = "@(#$Id: pyrecoll.cpp,v 1.21 2008-12-17 08:01:40 dockes Exp $ (C) 2007 J.F.Dockes";
#endif #endif
@ -881,11 +881,8 @@ Db_init(recoll_DbObject *self, PyObject *args, PyObject *kwargs)
if (self->db) if (self->db)
the_dbs.erase(self->db); the_dbs.erase(self->db);
delete self->db; delete self->db;
self->db = new Rcl::Db; self->db = new Rcl::Db(rclconfig);
string dbdir = rclconfig->getDbDir(); if (!self->db->open(writable ? Rcl::Db::DbUpd : Rcl::Db::DbRO)) {
if (!self->db->open(dbdir, rclconfig->getStopfile(), writable ?
Rcl::Db::DbUpd : Rcl::Db::DbRO)) {
LOGERR(("Db_init: db open error\n")); LOGERR(("Db_init: db open error\n"));
PyErr_SetString(PyExc_EnvironmentError, "Can't open index"); PyErr_SetString(PyExc_EnvironmentError, "Can't open index");
return -1; return -1;

12
src/qtgui/main.cpp
View File

@ -1,5 +1,5 @@
#ifndef lint #ifndef lint
static char rcsid[] = "@(#$Id: main.cpp,v 1.72 2008-12-12 11:00:27 dockes Exp $ (C) 2005 J.F.Dockes"; static char rcsid[] = "@(#$Id: main.cpp,v 1.73 2008-12-17 08:01:40 dockes Exp $ (C) 2005 J.F.Dockes";
#endif #endif
/* /*
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
@ -87,7 +87,6 @@ RclConfig* RclConfig::getMainConfig()
RclHistory *g_dynconf; RclHistory *g_dynconf;
int recollNeedsExit; int recollNeedsExit;
int startIndexingAfterConfig; int startIndexingAfterConfig;
static string dbdir;
static RclMain *mainWindow; static RclMain *mainWindow;
static string recollsharedir; static string recollsharedir;
@ -106,10 +105,9 @@ bool maybeOpenDb(string &reason, bool force)
LOGDEB(("main: adding [%s]\n", it->c_str())); LOGDEB(("main: adding [%s]\n", it->c_str()));
rcldb->addQueryDb(*it); rcldb->addQueryDb(*it);
} }
if (!rcldb->isopen() && !rcldb->open(dbdir, rclconfig->getStopfile(), if (!rcldb->isopen() && !rcldb->open(Rcl::Db::DbRO)) {
Rcl::Db::DbRO)) {
reason = "Could not open database in " + reason = "Could not open database in " +
dbdir + " wait for indexing to complete?"; rclconfig->getDbDir() + " wait for indexing to complete?";
return false; return false;
} }
rcldb->setAbstractParams(-1, prefs.syntAbsLen, prefs.syntAbsCtx); rcldb->setAbstractParams(-1, prefs.syntAbsLen, prefs.syntAbsCtx);
@ -315,7 +313,7 @@ int main(int argc, char **argv)
} }
mainWindow->sSearch->searchTypCMB->setCurrentItem(prefs.ssearchTyp); mainWindow->sSearch->searchTypCMB->setCurrentItem(prefs.ssearchTyp);
dbdir = rclconfig->getDbDir(); string dbdir = rclconfig->getDbDir();
if (dbdir.empty()) { if (dbdir.empty()) {
// Note: this will have to be replaced by a call to a // Note: this will have to be replaced by a call to a
// configuration buildin dialog for initial configuration // configuration buildin dialog for initial configuration
@ -325,7 +323,7 @@ int main(int argc, char **argv)
exit(1); exit(1);
} }
rcldb = new Rcl::Db; rcldb = new Rcl::Db(rclconfig);
bool needindexconfig = false; bool needindexconfig = false;
if (!maybeOpenDb(reason)) { if (!maybeOpenDb(reason)) {

13
src/query/recollq.cpp
View File

@ -1,5 +1,5 @@
#ifndef lint #ifndef lint
static char rcsid[] = "@(#$Id: recollq.cpp,v 1.21 2008-12-05 11:09:31 dockes Exp $ (C) 2006 J.F.Dockes"; static char rcsid[] = "@(#$Id: recollq.cpp,v 1.22 2008-12-17 08:01:40 dockes Exp $ (C) 2006 J.F.Dockes";
#endif #endif
/* /*
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
@ -164,8 +164,6 @@ int recollq(RclConfig **cfp, int argc, char **argv)
while (argc > 0) { while (argc > 0) {
qs += string(" ") + *argv++;argc--; qs += string(" ") + *argv++;argc--;
} }
Rcl::Db rcldb;
string dbdir;
string reason; string reason;
*cfp = recollinit(0, 0, reason, &a_config); *cfp = recollinit(0, 0, reason, &a_config);
RclConfig *rclconfig = *cfp; RclConfig *rclconfig = *cfp;
@ -189,8 +187,12 @@ int recollq(RclConfig **cfp, int argc, char **argv)
} }
dbdir = rclconfig->getDbDir(); Rcl::Db rcldb(rclconfig);
rcldb.open(dbdir, rclconfig->getStopfile(), Rcl::Db::DbRO); if (!rcldb.open(Rcl::Db::DbRO)) {
cerr << "Cant open database in " << rclconfig->getDbDir() <<
" reason: " << rcldb.getReason() << endl;
exit(1);
}
Rcl::SearchData *sd = 0; Rcl::SearchData *sd = 0;
@ -297,4 +299,3 @@ int main(int argc, char **argv)
exit(recollq(&rclconfig, argc, argv)); exit(recollq(&rclconfig, argc, argv));
} }
#endif // TEST_RECOLLQ #endif // TEST_RECOLLQ

52
src/rcldb/rcldb.cpp
View File

@ -1,5 +1,5 @@
#ifndef lint #ifndef lint
static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.151 2008-12-12 11:53:45 dockes Exp $ (C) 2004 J.F.Dockes"; static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.152 2008-12-17 08:01:40 dockes Exp $ (C) 2004 J.F.Dockes";
#endif #endif
/* /*
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
@ -470,17 +470,16 @@ string Db::Native::makeAbstract(Xapian::docid docid, Query *query)
/* Rcl::Db methods ///////////////////////////////// */ /* Rcl::Db methods ///////////////////////////////// */
Db::Db() Db::Db(RclConfig *cfp)
: m_ndb(0), m_idxAbsTruncLen(250), m_synthAbsLen(250), : m_ndb(0), m_config(cfp), m_idxAbsTruncLen(250), m_synthAbsLen(250),
m_synthAbsWordCtxLen(4), m_flushMb(-1), m_synthAbsWordCtxLen(4), m_flushMb(-1),
m_curtxtsz(0), m_flushtxtsz(0), m_occtxtsz(0), m_curtxtsz(0), m_flushtxtsz(0), m_occtxtsz(0),
m_maxFsOccupPc(0), m_mode(Db::DbRO) m_maxFsOccupPc(0), m_mode(Db::DbRO)
{ {
m_ndb = new Native(this); m_ndb = new Native(this);
RclConfig *config = RclConfig::getMainConfig(); if (m_config) {
if (config) { m_config->getConfParam("maxfsoccuppc", &m_maxFsOccupPc);
config->getConfParam("maxfsoccuppc", &m_maxFsOccupPc); m_config->getConfParam("idxflushmb", &m_flushMb);
config->getConfParam("idxflushmb", &m_flushMb);
} }
} }
@ -494,18 +493,19 @@ Db::~Db()
i_close(true); i_close(true);
} }
list<string> Db::getStemmerNames() list<string> Db::getStemmerNames()
{ {
list<string> res; list<string> res;
stringToStrings(Xapian::Stem::get_available_languages(), res); stringToStrings(Xapian::Stem::get_available_languages(), res);
return res; return res;
} }
bool Db::open(const string& dir, const string &stops, OpenMode mode, bool Db::open(OpenMode mode, bool keep_updated)
bool keep_updated)
{ {
if (m_ndb == 0) if (m_ndb == 0 || m_config == 0) {
m_reason = "Null configuration or Xapian Db";
return false; return false;
}
LOGDEB(("Db::open: m_isopen %d m_iswritable %d\n", m_ndb->m_isopen, LOGDEB(("Db::open: m_isopen %d m_iswritable %d\n", m_ndb->m_isopen,
m_ndb->m_iswritable)); m_ndb->m_iswritable));
@ -514,9 +514,9 @@ bool Db::open(const string& dir, const string &stops, OpenMode mode,
if (!close()) if (!close())
return false; return false;
} }
if (!stops.empty()) if (m_config->getStopfile().empty())
m_stops.setFile(stops); m_stops.setFile(m_config->getStopfile());
string dir = m_config->getDbDir();
string ermsg; string ermsg;
try { try {
switch (mode) { switch (mode) {
@ -632,7 +632,7 @@ bool Db::reOpen()
if (m_ndb && m_ndb->m_isopen) { if (m_ndb && m_ndb->m_isopen) {
if (!close()) if (!close())
return false; return false;
if (!open(m_basedir, string(), m_mode, true)) { if (!open(m_mode, true)) {
return false; return false;
} }
} }
@ -742,8 +742,7 @@ bool Db::fieldToPrefix(const string& fld, string &pfx)
fldToPrefs["tags"] = "K"; fldToPrefs["tags"] = "K";
} }
RclConfig *config = RclConfig::getMainConfig(); if (m_config && m_config->getFieldPrefix(fld, pfx))
if (config && config->getFieldPrefix(fld, pfx))
return true; return true;
// No data in rclconfig? Check default values // No data in rclconfig? Check default values
@ -1052,17 +1051,14 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi,
if (!doc.meta[Doc::keyabs].empty()) if (!doc.meta[Doc::keyabs].empty())
RECORD_APPEND(record, Doc::keyabs, doc.meta[Doc::keyabs]); RECORD_APPEND(record, Doc::keyabs, doc.meta[Doc::keyabs]);
RclConfig *config = RclConfig::getMainConfig(); const set<string>& stored = m_config->getStoredFields();
if (config) { for (set<string>::const_iterator it = stored.begin();
const set<string>& stored = config->getStoredFields(); it != stored.end(); it++) {
for (set<string>::const_iterator it = stored.begin(); string nm = m_config->fieldCanon(*it);
it != stored.end(); it++) { if (!doc.meta[*it].empty()) {
string nm = config->fieldCanon(*it); string value =
if (!doc.meta[*it].empty()) { neutchars(truncate_to_word(doc.meta[*it], 150), nc);
string value = RECORD_APPEND(record, nm, value);
neutchars(truncate_to_word(doc.meta[*it], 150), nc);
RECORD_APPEND(record, nm, value);
}
} }
} }

12
src/rcldb/rcldb.h
View File

@ -16,7 +16,7 @@
*/ */
#ifndef _DB_H_INCLUDED_ #ifndef _DB_H_INCLUDED_
#define _DB_H_INCLUDED_ #define _DB_H_INCLUDED_
/* @(#$Id: rcldb.h,v 1.63 2008-09-29 08:59:20 dockes Exp $ (C) 2004 J.F.Dockes */ /* @(#$Id: rcldb.h,v 1.64 2008-12-17 08:01:40 dockes Exp $ (C) 2004 J.F.Dockes */
#include <string> #include <string>
#include <list> #include <list>
@ -50,6 +50,8 @@ using std::vector;
// big, cause it's stored as a Xapian term (< 150 bytes would be // big, cause it's stored as a Xapian term (< 150 bytes would be
// reasonable) // reasonable)
class RclConfig;
#ifndef NO_NAMESPACES #ifndef NO_NAMESPACES
namespace Rcl { namespace Rcl {
#endif #endif
@ -80,12 +82,11 @@ class Db {
friend class Native; friend class Native;
/* General stuff (valid for query or update) ****************************/ /* General stuff (valid for query or update) ****************************/
Db(); Db(RclConfig *cfp);
~Db(); ~Db();
enum OpenMode {DbRO, DbUpd, DbTrunc}; enum OpenMode {DbRO, DbUpd, DbTrunc};
bool open(const string &dbdir, const string &stoplistfn, bool open(OpenMode mode, bool keep_updated = false);
OpenMode mode, bool keep_updated = false);
bool close(); bool close();
bool isopen(); bool isopen();
@ -194,7 +195,8 @@ private:
// Internal form of close, can be called during destruction // Internal form of close, can be called during destruction
bool i_close(bool final); bool i_close(bool final);
string m_reason; // Error explanation RclConfig *m_config;
string m_reason; // Error explanation
/* Parameters cached out of the configuration files */ /* Parameters cached out of the configuration files */
// This is how long an abstract we keep or build from beginning of // This is how long an abstract we keep or build from beginning of