simplified db open by getting rid of the illusion that we could have several writeable dbs per config
This commit is contained in:
dockes
parent
befebb7371
commit
b9cc5fb1e0
@ -1,5 +1,5 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.70 2008-10-08 16:15:22 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.71 2008-12-17 08:01:40 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
@ -93,7 +93,7 @@ bool DbIndexer::indexDb(bool resetbefore, list<string> *topdirs)
|
||||
for (list<string>::const_iterator it = topdirs->begin();
|
||||
it != topdirs->end(); it++) {
|
||||
LOGDEB(("DbIndexer::index: Indexing %s into %s\n", it->c_str(),
|
||||
m_dbdir.c_str()));
|
||||
getDbDir().c_str()));
|
||||
|
||||
// Set the current directory in config so that subsequent
|
||||
// getConfParams() will get local values
|
||||
@ -143,7 +143,7 @@ bool DbIndexer::indexDb(bool resetbefore, list<string> *topdirs)
|
||||
}
|
||||
if (!m_db.close()) {
|
||||
LOGERR(("DbIndexer::index: error closing database in %s\n",
|
||||
m_dbdir.c_str()));
|
||||
getDbDir().c_str()));
|
||||
return false;
|
||||
}
|
||||
string missing;
|
||||
@ -197,8 +197,8 @@ bool DbIndexer::init(bool resetbefore, bool rdonly)
|
||||
}
|
||||
Rcl::Db::OpenMode mode = rdonly ? Rcl::Db::DbRO :
|
||||
resetbefore ? Rcl::Db::DbTrunc : Rcl::Db::DbUpd;
|
||||
if (!m_db.open(m_dbdir, m_config->getStopfile(), mode)) {
|
||||
LOGERR(("DbIndexer: error opening database in %s\n", m_dbdir.c_str()));
|
||||
if (!m_db.open(mode)) {
|
||||
LOGERR(("DbIndexer: error opening database %s\n", getDbDir().c_str()));
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -250,7 +250,7 @@ bool DbIndexer::createAspellDict()
|
||||
// The close would be done in our destructor, but we want status here
|
||||
if (!m_db.close()) {
|
||||
LOGERR(("DbIndexer::indexfiles: error closing database in %s\n",
|
||||
m_dbdir.c_str()));
|
||||
getDbDir().c_str()));
|
||||
noaspell = true;
|
||||
return false;
|
||||
}
|
||||
@ -322,7 +322,7 @@ bool DbIndexer::indexFiles(const list<string> &filenames)
|
||||
// The close would be done in our destructor, but we want status here
|
||||
if (!m_db.close()) {
|
||||
LOGERR(("DbIndexer::indexfiles: error closing database in %s\n",
|
||||
m_dbdir.c_str()));
|
||||
getDbDir().c_str()));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
@ -348,7 +348,7 @@ bool DbIndexer::purgeFiles(const list<string> &filenames)
|
||||
// The close would be done in our destructor, but we want status here
|
||||
if (!m_db.close()) {
|
||||
LOGERR(("DbIndexer::purgefiles: error closing database in %s\n",
|
||||
m_dbdir.c_str()));
|
||||
getDbDir().c_str()));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
*/
|
||||
#ifndef _INDEXER_H_INCLUDED_
|
||||
#define _INDEXER_H_INCLUDED_
|
||||
/* @(#$Id: indexer.h,v 1.26 2008-10-08 16:15:22 dockes Exp $ (C) 2004 J.F.Dockes */
|
||||
/* @(#$Id: indexer.h,v 1.27 2008-12-17 08:01:40 dockes Exp $ (C) 2004 J.F.Dockes */
|
||||
|
||||
#include <string>
|
||||
#include <list>
|
||||
@ -56,11 +56,14 @@ class DbIxStatusUpdater {
|
||||
/**
|
||||
The top level indexing object. Processes the configuration, then invokes
|
||||
file system walking to populate/update the database(s).
|
||||
|
||||
Multiple top-level directories can be listed in the
|
||||
configuration. Each can be indexed to a different
|
||||
database. Directories are first grouped by database, then an
|
||||
internal class (DbIndexer) is used to process each group.
|
||||
|
||||
Fiction:
|
||||
Multiple top-level directories can be listed in the
|
||||
configuration. Each can be indexed to a different
|
||||
database. Directories are first grouped by database, then an
|
||||
internal class (DbIndexer) is used to process each group.
|
||||
Fact: we've had one db per config forever. The multidb/config code has been
|
||||
kept around for no good reason, this fiction only affects indexer.cpp
|
||||
*/
|
||||
class ConfIndexer {
|
||||
public:
|
||||
@ -93,10 +96,11 @@ class DbIndexer : public FsTreeWalkerCB {
|
||||
public:
|
||||
/** Constructor does nothing but store parameters */
|
||||
DbIndexer(RclConfig *cnf, // Configuration data
|
||||
const string &dbd, // Place where the db lives
|
||||
// Db dir not used anymore, rcl::db gets it from the cfg
|
||||
const string &,
|
||||
DbIxStatusUpdater *updfunc = 0 // status updater callback
|
||||
)
|
||||
: m_config(cnf), m_dbdir(dbd), m_updater(updfunc)
|
||||
: m_config(cnf), m_db(cnf), m_updater(updfunc)
|
||||
{}
|
||||
|
||||
virtual ~DbIndexer();
|
||||
@ -134,7 +138,7 @@ class DbIndexer : public FsTreeWalkerCB {
|
||||
FsTreeWalker::CbFlag);
|
||||
|
||||
/** Return my db dir */
|
||||
string getDbDir() {return m_dbdir;}
|
||||
string getDbDir() {return m_config->getDbDir();}
|
||||
|
||||
/** List possible stemmer names */
|
||||
static list<string> getStemmerNames();
|
||||
@ -142,7 +146,6 @@ class DbIndexer : public FsTreeWalkerCB {
|
||||
private:
|
||||
FsTreeWalker m_walker;
|
||||
RclConfig *m_config;
|
||||
string m_dbdir;
|
||||
Rcl::Db m_db;
|
||||
string m_tmpdir;
|
||||
DbIxStatusUpdater *m_updater;
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: kio_recoll.cpp,v 1.25 2008-12-16 17:28:10 dockes Exp $ (C) 2005 J.F.Dockes";
|
||||
static char rcsid[] = "@(#$Id: kio_recoll.cpp,v 1.26 2008-12-17 08:01:40 dockes Exp $ (C) 2005 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
@ -77,7 +77,7 @@ RecollProtocol::RecollProtocol(const QByteArray &pool, const QByteArray &app)
|
||||
return;
|
||||
}
|
||||
|
||||
m_rcldb = new Rcl::Db;
|
||||
m_rcldb = new Rcl::Db(o_rclconfig);
|
||||
if (!m_rcldb) {
|
||||
m_reason = "Could not build database object. (out of memory ?)";
|
||||
return;
|
||||
@ -112,9 +112,7 @@ bool RecollProtocol::maybeOpenDb(string &reason)
|
||||
reason = "Internal error: initialization error";
|
||||
return false;
|
||||
}
|
||||
if (!m_rcldb->isopen() && !m_rcldb->open(o_rclconfig->getDbDir(),
|
||||
o_rclconfig->getStopfile(),
|
||||
Rcl::Db::DbRO)) {
|
||||
if (!m_rcldb->isopen() && !m_rcldb->open(Rcl::Db::DbRO)) {
|
||||
reason = "Could not open database in " + o_rclconfig->getDbDir();
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: pyrecoll.cpp,v 1.20 2008-10-10 08:18:27 dockes Exp $ (C) 2007 J.F.Dockes";
|
||||
static char rcsid[] = "@(#$Id: pyrecoll.cpp,v 1.21 2008-12-17 08:01:40 dockes Exp $ (C) 2007 J.F.Dockes";
|
||||
#endif
|
||||
|
||||
|
||||
@ -881,11 +881,8 @@ Db_init(recoll_DbObject *self, PyObject *args, PyObject *kwargs)
|
||||
if (self->db)
|
||||
the_dbs.erase(self->db);
|
||||
delete self->db;
|
||||
self->db = new Rcl::Db;
|
||||
string dbdir = rclconfig->getDbDir();
|
||||
|
||||
if (!self->db->open(dbdir, rclconfig->getStopfile(), writable ?
|
||||
Rcl::Db::DbUpd : Rcl::Db::DbRO)) {
|
||||
self->db = new Rcl::Db(rclconfig);
|
||||
if (!self->db->open(writable ? Rcl::Db::DbUpd : Rcl::Db::DbRO)) {
|
||||
LOGERR(("Db_init: db open error\n"));
|
||||
PyErr_SetString(PyExc_EnvironmentError, "Can't open index");
|
||||
return -1;
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: main.cpp,v 1.72 2008-12-12 11:00:27 dockes Exp $ (C) 2005 J.F.Dockes";
|
||||
static char rcsid[] = "@(#$Id: main.cpp,v 1.73 2008-12-17 08:01:40 dockes Exp $ (C) 2005 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
@ -87,7 +87,6 @@ RclConfig* RclConfig::getMainConfig()
|
||||
RclHistory *g_dynconf;
|
||||
int recollNeedsExit;
|
||||
int startIndexingAfterConfig;
|
||||
static string dbdir;
|
||||
static RclMain *mainWindow;
|
||||
static string recollsharedir;
|
||||
|
||||
@ -106,10 +105,9 @@ bool maybeOpenDb(string &reason, bool force)
|
||||
LOGDEB(("main: adding [%s]\n", it->c_str()));
|
||||
rcldb->addQueryDb(*it);
|
||||
}
|
||||
if (!rcldb->isopen() && !rcldb->open(dbdir, rclconfig->getStopfile(),
|
||||
Rcl::Db::DbRO)) {
|
||||
if (!rcldb->isopen() && !rcldb->open(Rcl::Db::DbRO)) {
|
||||
reason = "Could not open database in " +
|
||||
dbdir + " wait for indexing to complete?";
|
||||
rclconfig->getDbDir() + " wait for indexing to complete?";
|
||||
return false;
|
||||
}
|
||||
rcldb->setAbstractParams(-1, prefs.syntAbsLen, prefs.syntAbsCtx);
|
||||
@ -315,7 +313,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
|
||||
mainWindow->sSearch->searchTypCMB->setCurrentItem(prefs.ssearchTyp);
|
||||
dbdir = rclconfig->getDbDir();
|
||||
string dbdir = rclconfig->getDbDir();
|
||||
if (dbdir.empty()) {
|
||||
// Note: this will have to be replaced by a call to a
|
||||
// configuration buildin dialog for initial configuration
|
||||
@ -325,7 +323,7 @@ int main(int argc, char **argv)
|
||||
exit(1);
|
||||
}
|
||||
|
||||
rcldb = new Rcl::Db;
|
||||
rcldb = new Rcl::Db(rclconfig);
|
||||
|
||||
bool needindexconfig = false;
|
||||
if (!maybeOpenDb(reason)) {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: recollq.cpp,v 1.21 2008-12-05 11:09:31 dockes Exp $ (C) 2006 J.F.Dockes";
|
||||
static char rcsid[] = "@(#$Id: recollq.cpp,v 1.22 2008-12-17 08:01:40 dockes Exp $ (C) 2006 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
@ -164,8 +164,6 @@ int recollq(RclConfig **cfp, int argc, char **argv)
|
||||
while (argc > 0) {
|
||||
qs += string(" ") + *argv++;argc--;
|
||||
}
|
||||
Rcl::Db rcldb;
|
||||
string dbdir;
|
||||
string reason;
|
||||
*cfp = recollinit(0, 0, reason, &a_config);
|
||||
RclConfig *rclconfig = *cfp;
|
||||
@ -189,8 +187,12 @@ int recollq(RclConfig **cfp, int argc, char **argv)
|
||||
}
|
||||
|
||||
|
||||
dbdir = rclconfig->getDbDir();
|
||||
rcldb.open(dbdir, rclconfig->getStopfile(), Rcl::Db::DbRO);
|
||||
Rcl::Db rcldb(rclconfig);
|
||||
if (!rcldb.open(Rcl::Db::DbRO)) {
|
||||
cerr << "Cant open database in " << rclconfig->getDbDir() <<
|
||||
" reason: " << rcldb.getReason() << endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
Rcl::SearchData *sd = 0;
|
||||
|
||||
@ -297,4 +299,3 @@ int main(int argc, char **argv)
|
||||
exit(recollq(&rclconfig, argc, argv));
|
||||
}
|
||||
#endif // TEST_RECOLLQ
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.151 2008-12-12 11:53:45 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.152 2008-12-17 08:01:40 dockes Exp $ (C) 2004 J.F.Dockes";
|
||||
#endif
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
@ -470,17 +470,16 @@ string Db::Native::makeAbstract(Xapian::docid docid, Query *query)
|
||||
|
||||
/* Rcl::Db methods ///////////////////////////////// */
|
||||
|
||||
Db::Db()
|
||||
: m_ndb(0), m_idxAbsTruncLen(250), m_synthAbsLen(250),
|
||||
Db::Db(RclConfig *cfp)
|
||||
: m_ndb(0), m_config(cfp), m_idxAbsTruncLen(250), m_synthAbsLen(250),
|
||||
m_synthAbsWordCtxLen(4), m_flushMb(-1),
|
||||
m_curtxtsz(0), m_flushtxtsz(0), m_occtxtsz(0),
|
||||
m_maxFsOccupPc(0), m_mode(Db::DbRO)
|
||||
{
|
||||
m_ndb = new Native(this);
|
||||
RclConfig *config = RclConfig::getMainConfig();
|
||||
if (config) {
|
||||
config->getConfParam("maxfsoccuppc", &m_maxFsOccupPc);
|
||||
config->getConfParam("idxflushmb", &m_flushMb);
|
||||
if (m_config) {
|
||||
m_config->getConfParam("maxfsoccuppc", &m_maxFsOccupPc);
|
||||
m_config->getConfParam("idxflushmb", &m_flushMb);
|
||||
}
|
||||
}
|
||||
|
||||
@ -494,18 +493,19 @@ Db::~Db()
|
||||
i_close(true);
|
||||
}
|
||||
|
||||
list<string> Db::getStemmerNames()
|
||||
list<string> Db::getStemmerNames()
|
||||
{
|
||||
list<string> res;
|
||||
stringToStrings(Xapian::Stem::get_available_languages(), res);
|
||||
return res;
|
||||
}
|
||||
|
||||
bool Db::open(const string& dir, const string &stops, OpenMode mode,
|
||||
bool keep_updated)
|
||||
bool Db::open(OpenMode mode, bool keep_updated)
|
||||
{
|
||||
if (m_ndb == 0)
|
||||
if (m_ndb == 0 || m_config == 0) {
|
||||
m_reason = "Null configuration or Xapian Db";
|
||||
return false;
|
||||
}
|
||||
LOGDEB(("Db::open: m_isopen %d m_iswritable %d\n", m_ndb->m_isopen,
|
||||
m_ndb->m_iswritable));
|
||||
|
||||
@ -514,9 +514,9 @@ bool Db::open(const string& dir, const string &stops, OpenMode mode,
|
||||
if (!close())
|
||||
return false;
|
||||
}
|
||||
if (!stops.empty())
|
||||
m_stops.setFile(stops);
|
||||
|
||||
if (m_config->getStopfile().empty())
|
||||
m_stops.setFile(m_config->getStopfile());
|
||||
string dir = m_config->getDbDir();
|
||||
string ermsg;
|
||||
try {
|
||||
switch (mode) {
|
||||
@ -632,7 +632,7 @@ bool Db::reOpen()
|
||||
if (m_ndb && m_ndb->m_isopen) {
|
||||
if (!close())
|
||||
return false;
|
||||
if (!open(m_basedir, string(), m_mode, true)) {
|
||||
if (!open(m_mode, true)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -742,8 +742,7 @@ bool Db::fieldToPrefix(const string& fld, string &pfx)
|
||||
fldToPrefs["tags"] = "K";
|
||||
}
|
||||
|
||||
RclConfig *config = RclConfig::getMainConfig();
|
||||
if (config && config->getFieldPrefix(fld, pfx))
|
||||
if (m_config && m_config->getFieldPrefix(fld, pfx))
|
||||
return true;
|
||||
|
||||
// No data in rclconfig? Check default values
|
||||
@ -1052,17 +1051,14 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi,
|
||||
if (!doc.meta[Doc::keyabs].empty())
|
||||
RECORD_APPEND(record, Doc::keyabs, doc.meta[Doc::keyabs]);
|
||||
|
||||
RclConfig *config = RclConfig::getMainConfig();
|
||||
if (config) {
|
||||
const set<string>& stored = config->getStoredFields();
|
||||
for (set<string>::const_iterator it = stored.begin();
|
||||
it != stored.end(); it++) {
|
||||
string nm = config->fieldCanon(*it);
|
||||
if (!doc.meta[*it].empty()) {
|
||||
string value =
|
||||
neutchars(truncate_to_word(doc.meta[*it], 150), nc);
|
||||
RECORD_APPEND(record, nm, value);
|
||||
}
|
||||
const set<string>& stored = m_config->getStoredFields();
|
||||
for (set<string>::const_iterator it = stored.begin();
|
||||
it != stored.end(); it++) {
|
||||
string nm = m_config->fieldCanon(*it);
|
||||
if (!doc.meta[*it].empty()) {
|
||||
string value =
|
||||
neutchars(truncate_to_word(doc.meta[*it], 150), nc);
|
||||
RECORD_APPEND(record, nm, value);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
*/
|
||||
#ifndef _DB_H_INCLUDED_
|
||||
#define _DB_H_INCLUDED_
|
||||
/* @(#$Id: rcldb.h,v 1.63 2008-09-29 08:59:20 dockes Exp $ (C) 2004 J.F.Dockes */
|
||||
/* @(#$Id: rcldb.h,v 1.64 2008-12-17 08:01:40 dockes Exp $ (C) 2004 J.F.Dockes */
|
||||
|
||||
#include <string>
|
||||
#include <list>
|
||||
@ -50,6 +50,8 @@ using std::vector;
|
||||
// big, cause it's stored as a Xapian term (< 150 bytes would be
|
||||
// reasonable)
|
||||
|
||||
class RclConfig;
|
||||
|
||||
#ifndef NO_NAMESPACES
|
||||
namespace Rcl {
|
||||
#endif
|
||||
@ -80,12 +82,11 @@ class Db {
|
||||
friend class Native;
|
||||
|
||||
/* General stuff (valid for query or update) ****************************/
|
||||
Db();
|
||||
Db(RclConfig *cfp);
|
||||
~Db();
|
||||
|
||||
enum OpenMode {DbRO, DbUpd, DbTrunc};
|
||||
bool open(const string &dbdir, const string &stoplistfn,
|
||||
OpenMode mode, bool keep_updated = false);
|
||||
bool open(OpenMode mode, bool keep_updated = false);
|
||||
bool close();
|
||||
bool isopen();
|
||||
|
||||
@ -194,7 +195,8 @@ private:
|
||||
// Internal form of close, can be called during destruction
|
||||
bool i_close(bool final);
|
||||
|
||||
string m_reason; // Error explanation
|
||||
RclConfig *m_config;
|
||||
string m_reason; // Error explanation
|
||||
|
||||
/* Parameters cached out of the configuration files */
|
||||
// This is how long an abstract we keep or build from beginning of
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user