tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

additional search databases

Browse Source
This commit is contained in:
dockes 2006-04-05 12:50:42 +00:00
parent daa37c68f7
commit f2292dc951
7 changed files with 696 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
src/qtgui/guiutils.cpp
View File

@ -1,5 +1,5 @@
#ifndef lint #ifndef lint
static char rcsid[] = "@(#$Id: guiutils.cpp,v 1.5 2006-04-04 10:38:52 dockes Exp $ (C) 2005 Jean-Francois Dockes"; static char rcsid[] = "@(#$Id: guiutils.cpp,v 1.6 2006-04-05 12:50:42 dockes Exp $ (C) 2005 Jean-Francois Dockes";
#endif #endif
/* /*
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
@ -24,8 +24,10 @@ static char rcsid[] = "@(#$Id: guiutils.cpp,v 1.5 2006-04-04 10:38:52 dockes Exp
#include "recoll.h" #include "recoll.h"
#include "guiutils.h" #include "guiutils.h"
#include "pathut.h" #include "pathut.h"
#include "base64.h"
#include <qsettings.h> #include <qsettings.h>
#include <qstringlist.h>
const static char *htmlbrowserlist = const static char *htmlbrowserlist =
"opera konqueror firefox mozilla netscape"; "opera konqueror firefox mozilla netscape";
@ -129,6 +131,58 @@ void rwSettings(bool writing)
"/Recoll/prefs/query/buildAbstract", Bool, true); "/Recoll/prefs/query/buildAbstract", Bool, true);
SETTING_RW(prefs.queryReplaceAbstract, SETTING_RW(prefs.queryReplaceAbstract,
"/Recoll/prefs/query/replaceAbstract", Bool, false); "/Recoll/prefs/query/replaceAbstract", Bool, false);
QStringList qsl;
if (writing) {
for (list<string>::const_iterator it = prefs.allExtraDbs.begin();
it != prefs.allExtraDbs.end(); it++) {
string b64;
base64_encode(*it, b64);
qsl.push_back(QString::fromAscii(b64.c_str()));
}
settings.writeEntry("/Recoll/prefs/query/allExtraDbs", qsl);
qsl.clear();
for (list<string>::const_iterator it = prefs.activeExtraDbs.begin();
it != prefs.activeExtraDbs.end(); it++) {
string b64;
base64_encode(*it, b64);
qsl.push_back(QString::fromAscii(b64.c_str()));
}
settings.writeEntry("/Recoll/prefs/query/activeExtraDbs", qsl);
} else {
qsl = settings.readListEntry("/Recoll/prefs/query/allExtraDbs");
prefs.allExtraDbs.clear();
for (QStringList::iterator it = qsl.begin(); it != qsl.end(); it++) {
string dec;
base64_decode((*it).ascii(), dec);
prefs.allExtraDbs.push_back(dec);
}
qsl = settings.readListEntry("/Recoll/prefs/query/activeExtraDbs");
prefs.activeExtraDbs.clear();
for (QStringList::iterator it = qsl.begin(); it != qsl.end(); it++) {
string dec;
base64_decode((*it).ascii(), dec);
prefs.activeExtraDbs.push_back(dec);
}
}
#if 1
{
list<string>::const_iterator it;
fprintf(stderr, "All extra Dbs:\n");
for (it = prefs.allExtraDbs.begin();
it != prefs.allExtraDbs.end(); it++) {
fprintf(stderr, " [%s]\n", it->c_str());
}
fprintf(stderr, "Active extra Dbs:\n");
for (it = prefs.activeExtraDbs.begin();
it != prefs.activeExtraDbs.end(); it++) {
fprintf(stderr, " [%s]\n", it->c_str());
}
}
#endif
} }

13
src/qtgui/guiutils.h
View File

@ -17,7 +17,7 @@
#ifndef _GUIUTILS_H_INCLUDED_ #ifndef _GUIUTILS_H_INCLUDED_
#define _GUIUTILS_H_INCLUDED_ #define _GUIUTILS_H_INCLUDED_
/* /*
* @(#$Id: guiutils.h,v 1.3 2006-03-29 17:31:55 dockes Exp $ (C) 2005 Jean-Francois Dockes * @(#$Id: guiutils.h,v 1.4 2006-04-05 12:50:42 dockes Exp $ (C) 2005 Jean-Francois Dockes
* jean-francois.dockes@wanadoo.fr * jean-francois.dockes@wanadoo.fr
* *
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
@ -37,8 +37,14 @@
*/ */
#include <string> #include <string>
#include <list>
#include <qstring.h> #include <qstring.h>
#ifndef NO_NAMESPACES
using std::string;
using std::list;
#endif
/** Start a browser on the help document */ /** Start a browser on the help document */
extern bool startHelpBrowser(const string& url = ""); extern bool startHelpBrowser(const string& url = "");
@ -56,6 +62,11 @@ class PrefsPack {
QString htmlBrowser; QString htmlBrowser;
bool queryBuildAbstract; bool queryBuildAbstract;
bool queryReplaceAbstract; bool queryReplaceAbstract;
// Extra query databases. This are encoded to base64 before storing
// to the qt settings file to avoid any bin string/ charset conv issues
list<string> allExtraDbs;
list<string> activeExtraDbs;
PrefsPack() : PrefsPack() :
showicons(true), showicons(true),
respagesize(8), respagesize(8),

8
src/qtgui/main.cpp
View File

@ -1,5 +1,5 @@
#ifndef lint #ifndef lint
static char rcsid[] = "@(#$Id: main.cpp,v 1.39 2006-04-04 07:55:29 dockes Exp $ (C) 2005 J.F.Dockes"; static char rcsid[] = "@(#$Id: main.cpp,v 1.40 2006-04-05 12:50:42 dockes Exp $ (C) 2005 J.F.Dockes";
#endif #endif
/* /*
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
@ -90,6 +90,12 @@ bool maybeOpenDb(string &reason, bool force)
qopts |= Rcl::Db::QO_REPLACE_ABSTRACT; qopts |= Rcl::Db::QO_REPLACE_ABSTRACT;
if (force) if (force)
rcldb->close(); rcldb->close();
rcldb->rmQueryDb("");
for (list<string>::const_iterator it = prefs.activeExtraDbs.begin();
it != prefs.activeExtraDbs.end(); it++) {
LOGDEB(("main: adding [%s]\n", it->c_str()));
rcldb->addQueryDb(*it);
}
if (!rcldb->isopen() && !rcldb->open(dbdir, Rcl::Db::DbRO, qopts)) { if (!rcldb->isopen() && !rcldb->open(dbdir, Rcl::Db::DbRO, qopts)) {
reason = "Could not open database in " + reason = "Could not open database in " +
dbdir + " wait for indexing to complete?"; dbdir + " wait for indexing to complete?";

227
src/qtgui/uiprefs.ui
View File

@ -8,8 +8,8 @@
<rect> <rect>
<x>0</x> <x>0</x>
<y>0</y> <y>0</y>
<width>455</width> <width>533</width>
<height>238</height> <height>372</height>
</rect> </rect>
</property> </property>
<property name="caption"> <property name="caption">
@ -84,7 +84,7 @@
</widget> </widget>
<widget class="QLayoutWidget"> <widget class="QLayoutWidget">
<property name="name"> <property name="name">
<cstring>layout6</cstring> <cstring>layout4</cstring>
</property> </property>
<hbox> <hbox>
<property name="name"> <property name="name">
@ -260,6 +260,220 @@ May be slow for big documents.</string>
</widget> </widget>
</vbox> </vbox>
</widget> </widget>
<widget class="QWidget">
<property name="name">
<cstring>ExtraDb</cstring>
</property>
<attribute name="title">
<string>Extra Databases</string>
</attribute>
<vbox>
<property name="name">
<cstring>unnamed</cstring>
</property>
<widget class="QLayoutWidget">
<property name="name">
<cstring>layout12</cstring>
</property>
<hbox>
<property name="name">
<cstring>unnamed</cstring>
</property>
<widget class="QPushButton">
<property name="name">
<cstring>addExtraDbPB</cstring>
</property>
<property name="enabled">
<bool>false</bool>
</property>
<property name="text">
<string>Add database</string>
</property>
<property name="toolTip" stdset="0">
<string>Select the recoll.conf file defining the database you want to add, then click Add Database</string>
</property>
</widget>
<widget class="QLineEdit">
<property name="name">
<cstring>extraDbLE</cstring>
</property>
<property name="minimumSize">
<size>
<width>200</width>
<height>0</height>
</size>
</property>
<property name="toolTip" stdset="0">
<string>Select the recoll.conf file defining the database you want to add, then click Add Database</string>
</property>
</widget>
<widget class="QPushButton">
<property name="name">
<cstring>browseDbPB</cstring>
</property>
<property name="text">
<string>Browse</string>
</property>
<property name="toolTip" stdset="0">
<string>Select the recoll.conf file defining the database you want to add, then click Add Database</string>
</property>
</widget>
</hbox>
</widget>
<widget class="QLayoutWidget">
<property name="name">
<cstring>layout12</cstring>
</property>
<hbox>
<property name="name">
<cstring>unnamed</cstring>
</property>
<widget class="QLayoutWidget">
<property name="name">
<cstring>layout13</cstring>
</property>
<vbox>
<property name="name">
<cstring>unnamed</cstring>
</property>
<widget class="QLabel">
<property name="name">
<cstring>textLabel2_2</cstring>
</property>
<property name="text">
<string>All extra databases</string>
</property>
<property name="toolTip" stdset="0">
<string>All known extra databases</string>
</property>
</widget>
<widget class="QListBox">
<property name="name">
<cstring>allDbsLB</cstring>
</property>
<property name="enabled">
<bool>true</bool>
</property>
<property name="minimumSize">
<size>
<width>200</width>
<height>200</height>
</size>
</property>
<property name="selectionMode">
<enum>Extended</enum>
</property>
</widget>
</vbox>
</widget>
<widget class="QLayoutWidget">
<property name="name">
<cstring>layout11</cstring>
</property>
<vbox>
<property name="name">
<cstring>unnamed</cstring>
</property>
<widget class="QPushButton">
<property name="name">
<cstring>addAADbPB</cstring>
</property>
<property name="enabled">
<bool>true</bool>
</property>
<property name="text">
<string>All ----&gt;</string>
</property>
<property name="autoDefault">
<bool>false</bool>
</property>
</widget>
<widget class="QPushButton">
<property name="name">
<cstring>addADbPB</cstring>
</property>
<property name="enabled">
<bool>true</bool>
</property>
<property name="text">
<string>Sel -----&gt;</string>
</property>
<property name="autoDefault">
<bool>false</bool>
</property>
</widget>
<widget class="QPushButton">
<property name="name">
<cstring>delADbPB</cstring>
</property>
<property name="enabled">
<bool>true</bool>
</property>
<property name="text">
<string>&lt;----- Sel</string>
</property>
<property name="autoDefault">
<bool>false</bool>
</property>
</widget>
<widget class="QPushButton">
<property name="name">
<cstring>delAADbPB</cstring>
</property>
<property name="enabled">
<bool>true</bool>
</property>
<property name="text">
<string>&lt;----- All</string>
</property>
<property name="autoDefault">
<bool>false</bool>
</property>
</widget>
</vbox>
</widget>
<widget class="QLayoutWidget">
<property name="name">
<cstring>layout14</cstring>
</property>
<vbox>
<property name="name">
<cstring>unnamed</cstring>
</property>
<widget class="QLabel">
<property name="name">
<cstring>textLabel3</cstring>
</property>
<property name="text">
<string>Active extra databases</string>
</property>
<property name="toolTip" stdset="0">
<string>Extra databases that will be searched in addition to the main one</string>
</property>
</widget>
<widget class="QListBox">
<property name="name">
<cstring>actDbsLB</cstring>
</property>
<property name="enabled">
<bool>true</bool>
</property>
<property name="minimumSize">
<size>
<width>200</width>
<height>200</height>
</size>
</property>
<property name="selectionMode">
<enum>Extended</enum>
</property>
</widget>
</vbox>
</widget>
</hbox>
</widget>
</vbox>
</widget>
</widget> </widget>
<widget class="QLayoutWidget"> <widget class="QLayoutWidget">
<property name="name"> <property name="name">
@ -368,6 +582,13 @@ May be slow for big documents.</string>
<slot>showFontDialog()</slot> <slot>showFontDialog()</slot>
<slot>resetReslistFont()</slot> <slot>resetReslistFont()</slot>
<slot>showBrowserDialog()</slot> <slot>showBrowserDialog()</slot>
<slot>extraDbTextChanged( const QString &amp; text )</slot>
<slot>addAADbPB_clicked()</slot>
<slot>addADbPB_clicked()</slot>
<slot>delADbPB_clicked()</slot>
<slot>delAADbPB_clicked()</slot>
<slot>addExtraDbPB_clicked()</slot>
<slot>browseDbPB_clicked()</slot>
</slots> </slots>
<functions> <functions>
<function>init()</function> <function>init()</function>

132
src/qtgui/uiprefs.ui.h
View File

@ -25,6 +25,9 @@
** These will automatically be called by the form's constructor and ** These will automatically be called by the form's constructor and
** destructor. ** destructor.
*****************************************************************************/ *****************************************************************************/
#include <string>
#include <algorithm>
#include <list>
#include "qfontdialog.h" #include "qfontdialog.h"
#include "qfiledialog.h" #include "qfiledialog.h"
@ -33,6 +36,7 @@
#include "recoll.h" #include "recoll.h"
#include "guiutils.h" #include "guiutils.h"
#include "rcldb.h"
void UIPrefsDialog::init() void UIPrefsDialog::init()
{ {
@ -82,10 +86,35 @@ void UIPrefsDialog::init()
replAbsCB->setEnabled(false); replAbsCB->setEnabled(false);
} }
replAbsCB->setDown(prefs.queryReplaceAbstract); replAbsCB->setDown(prefs.queryReplaceAbstract);
// Initialize the extra databases listboxes
QStringList ql;
for (list<string>::iterator it = prefs.allExtraDbs.begin();
it != prefs.allExtraDbs.end(); it++) {
ql.append(QString::fromLocal8Bit(it->c_str()));
}
allDbsLB->insertStringList(ql);
ql.clear();
for (list<string>::iterator it = prefs.activeExtraDbs.begin();
it != prefs.activeExtraDbs.end(); it++) {
ql.append(QString::fromLocal8Bit(it->c_str()));
}
actDbsLB->insertStringList(ql);
ql.clear();
connect(reslistFontPB, SIGNAL(clicked()), this, SLOT(showFontDialog())); connect(reslistFontPB, SIGNAL(clicked()), this, SLOT(showFontDialog()));
connect(helpBrowserPB, SIGNAL(clicked()), this, SLOT(showBrowserDialog())); connect(helpBrowserPB, SIGNAL(clicked()), this, SLOT(showBrowserDialog()));
connect(resetFontPB, SIGNAL(clicked()), this, SLOT(resetReslistFont())); connect(resetFontPB, SIGNAL(clicked()), this, SLOT(resetReslistFont()));
connect(extraDbLE,SIGNAL(textChanged(const QString&)), this,
SLOT(extraDbTextChanged(const QString&)));
connect(addAADbPB, SIGNAL(clicked()), this, SLOT(addAADbPB_clicked()));
connect(addADbPB, SIGNAL(clicked()), this, SLOT(addADbPB_clicked()));
connect(delADbPB, SIGNAL(clicked()), this, SLOT(delADbPB_clicked()));
connect(delAADbPB, SIGNAL(clicked()), this, SLOT(delAADbPB_clicked()));
connect(addExtraDbPB, SIGNAL(clicked()), this, SLOT(addExtraDbPB_clicked()));
connect(browseDbPB, SIGNAL(clicked()), this, SLOT(browseDbPB_clicked()));
} }
void UIPrefsDialog::accept() void UIPrefsDialog::accept()
@ -159,3 +188,106 @@ void UIPrefsDialog::showBrowserDialog()
if (s) if (s)
helpBrowserLE->setText(s); helpBrowserLE->setText(s);
} }
////////////////////////////////////////////
// External / extra search databases setup: this should modify to take
// effect only when Ok is clicked. Currently modifs take effect as soon as
// done in the Gui
// Also needed: means to remove entry from 'all' list (del button? )
void UIPrefsDialog::extraDbTextChanged(const QString &text)
{
if (text.isEmpty()) {
addExtraDbPB->setEnabled(false);
} else {
addExtraDbPB->setEnabled(true);
}
}
// Add selected dbs to the active list
void UIPrefsDialog::addADbPB_clicked()
{
for (unsigned int i = 0; i < allDbsLB->count();i++) {
QListBoxItem *item = allDbsLB->item(i);
if (item && item->isSelected()) {
allDbsLB->setSelected(i, false);
string dbname = (const char*)item->text().local8Bit();
if (std::find(prefs.activeExtraDbs.begin(), prefs.activeExtraDbs.end(),
dbname) == prefs.activeExtraDbs.end()) {
actDbsLB->insertItem(item->text());
prefs.activeExtraDbs.push_back(dbname);
}
}
}
actDbsLB->sort();
}
void UIPrefsDialog::addAADbPB_clicked()
{
for (unsigned int i = 0; i < allDbsLB->count();i++) {
allDbsLB->setSelected(i, true);
}
addADbPB_clicked();
}
void UIPrefsDialog::delADbPB_clicked()
{
list<int> rmi;
for (unsigned int i = 0; i < actDbsLB->count(); i++) {
QListBoxItem *item = actDbsLB->item(i);
if (item && item->isSelected()) {
string dbname = (const char*)item->text().local8Bit();
list<string>::iterator sit;
if ((sit = ::std::find(prefs.activeExtraDbs.begin(),
prefs.activeExtraDbs.end(), dbname)) !=
prefs.activeExtraDbs.end()) {
prefs.activeExtraDbs.erase(sit);
}
rmi.push_front(i);
}
}
for (list<int>::iterator ii = rmi.begin(); ii != rmi.end(); ii++) {
actDbsLB->removeItem(*ii);
}
}
void UIPrefsDialog::delAADbPB_clicked()
{
for (unsigned int i = 0; i < actDbsLB->count(); i++) {
actDbsLB->setSelected(i, true);
}
delADbPB_clicked();
}
void UIPrefsDialog::addExtraDbPB_clicked()
{
string dbdir = (const char *)extraDbLE->text().local8Bit();
if (!Rcl::Db::testDbDir(dbdir)) {
QMessageBox::warning(0, "Recoll",
tr("The selected directory does not appear to be a Xapian database"));
return;
}
if (::std::find(prefs.allExtraDbs.begin(), prefs.allExtraDbs.end(),
dbdir) != prefs.allExtraDbs.end()) {
QMessageBox::warning(0, "Recoll",
tr("The selected directory is already in the database list"));
return;
}
prefs.allExtraDbs.push_back(dbdir);
allDbsLB->insertItem(extraDbLE->text());
allDbsLB->sort();
}
void UIPrefsDialog::browseDbPB_clicked()
{
QFileDialog fdia;
bool savedh = fdia.showHiddenFiles();
fdia.setShowHiddenFiles(true);
QString s = QFileDialog::getExistingDirectory("", this, 0,
tr("Select directory holding xapian database (ie: /home/someone/.recoll/xapiandb)"));
fdia.setShowHiddenFiles(savedh);
if (s)
extraDbLE->setText(s);
}

407
src/rcldb/rcldb.cpp
View File

@ -1,5 +1,5 @@
#ifndef lint #ifndef lint
static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.60 2006-04-05 06:26:56 dockes Exp $ (C) 2004 J.F.Dockes"; static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.61 2006-04-05 12:50:42 dockes Exp $ (C) 2004 J.F.Dockes";
#endif #endif
/* /*
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
@ -67,13 +67,26 @@ namespace Rcl {
// when building the abstract // when building the abstract
#define MA_EXTRACT_WIDTH 4 #define MA_EXTRACT_WIDTH 4
// Truncate longer path and uniquize with hash . The goal for this is
// to avoid xapian max term length limitations, not to gain space (we
// gain very little even with very short maxlens like 30)
#define PATHHASHLEN 150
// Synthetic abstract marker (to discriminate from abstract actually
// found in doc)
const static string rclSyntAbs = "?!#@";
// Data for a xapian database. There could actually be 2 different // Data for a xapian database. There could actually be 2 different
// ones for indexing or query as there is not much in common. // ones for indexing or query as there is not much in common.
class Native { class Native {
public: public:
bool isopen; bool m_isopen;
bool iswritable; bool m_iswritable;
string basedir; Db::OpenMode m_mode;
string m_basedir;
// List of directories for additional databases to query
list<string> m_extraDbs;
// Indexing // Indexing
Xapian::WritableDatabase wdb; Xapian::WritableDatabase wdb;
@ -92,7 +105,9 @@ class Native {
Xapian::docid docid, Xapian::docid docid,
const list<string>& terms); const list<string>& terms);
Native() : isopen(false), iswritable(false), enquire(0) { } Native()
: m_isopen(false), m_iswritable(false), m_mode(Db::DbRO), enquire(0)
{ }
~Native() { ~Native() {
delete enquire; delete enquire;
} }
@ -112,27 +127,28 @@ class Native {
}; };
Db::Db() Db::Db()
: m_qOpts(0)
{ {
ndb = new Native; m_ndb = new Native;
m_qOpts = 0;
} }
Db::~Db() Db::~Db()
{ {
LOGDEB1(("Db::~Db\n")); LOGDEB1(("Db::~Db\n"));
if (ndb == 0) if (m_ndb == 0)
return; return;
LOGDEB(("Db::~Db: isopen %d iswritable %d\n", ndb->isopen, LOGDEB(("Db::~Db: isopen %d m_iswritable %d\n", m_ndb->m_isopen,
ndb->iswritable)); m_ndb->m_iswritable));
if (ndb->isopen == false) if (m_ndb->m_isopen == false)
return; return;
const char *ermsg = "Unknown error"; const char *ermsg = "Unknown error";
try { try {
LOGDEB(("Db::~Db: closing native database\n")); LOGDEB(("Db::~Db: closing native database\n"));
if (ndb->iswritable == true) { if (m_ndb->m_iswritable == true) {
ndb->wdb.flush(); m_ndb->wdb.flush();
} }
delete ndb; delete m_ndb;
m_ndb = 0;
return; return;
} catch (const Xapian::Error &e) { } catch (const Xapian::Error &e) {
ermsg = e.get_msg().c_str(); ermsg = e.get_msg().c_str();
@ -148,15 +164,15 @@ Db::~Db()
bool Db::open(const string& dir, OpenMode mode, int qops) bool Db::open(const string& dir, OpenMode mode, int qops)
{ {
if (ndb == 0) if (m_ndb == 0)
return false; return false;
LOGDEB(("Db::open: isopen %d iswritable %d\n", ndb->isopen, LOGDEB(("Db::open: m_isopen %d m_iswritable %d\n", m_ndb->m_isopen,
ndb->iswritable)); m_ndb->m_iswritable));
m_qOpts = qops;
if (ndb->isopen) { if (m_ndb->m_isopen) {
LOGERR(("Db::open: already open\n")); // We used to return an error here but I see no reason to
return false; if (!close())
return false;
} }
const char *ermsg = "Unknown"; const char *ermsg = "Unknown";
try { try {
@ -166,23 +182,46 @@ bool Db::open(const string& dir, OpenMode mode, int qops)
{ {
int action = (mode == DbUpd) ? Xapian::DB_CREATE_OR_OPEN : int action = (mode == DbUpd) ? Xapian::DB_CREATE_OR_OPEN :
Xapian::DB_CREATE_OR_OVERWRITE; Xapian::DB_CREATE_OR_OVERWRITE;
ndb->wdb = Xapian::WritableDatabase(dir, action); m_ndb->wdb = Xapian::WritableDatabase(dir, action);
LOGDEB(("Db::open: lastdocid: %d\n", LOGDEB(("Db::open: lastdocid: %d\n",
ndb->wdb.get_lastdocid())); m_ndb->wdb.get_lastdocid()));
ndb->updated.resize(ndb->wdb.get_lastdocid() + 1); m_ndb->updated.resize(m_ndb->wdb.get_lastdocid() + 1);
for (unsigned int i = 0; i < ndb->updated.size(); i++) for (unsigned int i = 0; i < m_ndb->updated.size(); i++)
ndb->updated[i] = false; m_ndb->updated[i] = false;
ndb->iswritable = true; m_ndb->m_iswritable = true;
} }
break; break;
case DbRO: case DbRO:
default: default:
ndb->iswritable = false; m_ndb->m_iswritable = false;
ndb->db = Xapian::Database(dir); m_ndb->db = Xapian::Database(dir);
for (list<string>::iterator it = m_ndb->m_extraDbs.begin();
it != m_ndb->m_extraDbs.end(); it++) {
string aerr;
LOGDEB(("Db::Open: adding query db [%s]\n", it->c_str()));
aerr.clear();
try {
// Make this non-fatal
m_ndb->db.add_database(Xapian::Database(*it));
} catch (const Xapian::Error &e) {
aerr = e.get_msg().c_str();
} catch (const string &s) {
aerr = s.c_str();
} catch (const char *s) {
aerr = s;
} catch (...) {
aerr = "Caught unknown exception";
}
if (!aerr.empty())
LOGERR(("Db::Open: error while trying to add database "
"from [%s]: %s\n", it->c_str(), aerr.c_str()));
}
break; break;
} }
ndb->isopen = true; m_qOpts = qops;
ndb->basedir = dir; m_ndb->m_mode = mode;
m_ndb->m_isopen = true;
m_ndb->m_basedir = dir;
return true; return true;
} catch (const Xapian::Error &e) { } catch (const Xapian::Error &e) {
ermsg = e.get_msg().c_str(); ermsg = e.get_msg().c_str();
@ -201,21 +240,21 @@ bool Db::open(const string& dir, OpenMode mode, int qops)
// Note: xapian has no close call, we delete and recreate the db // Note: xapian has no close call, we delete and recreate the db
bool Db::close() bool Db::close()
{ {
if (ndb == 0) if (m_ndb == 0)
return false; return false;
LOGDEB(("Db::close(): isopen %d iswritable %d\n", ndb->isopen, LOGDEB(("Db::close(): m_isopen %d m_iswritable %d\n", m_ndb->m_isopen,
ndb->iswritable)); m_ndb->m_iswritable));
if (ndb->isopen == false) if (m_ndb->m_isopen == false)
return true; return true;
const char *ermsg = "Unknown"; const char *ermsg = "Unknown";
try { try {
if (ndb->iswritable == true) { if (m_ndb->m_iswritable == true) {
ndb->wdb.flush(); m_ndb->wdb.flush();
LOGDEB(("Rcl:Db: Called xapian flush\n")); LOGDEB(("Rcl:Db: Called xapian flush\n"));
} }
delete ndb; delete m_ndb;
ndb = new Native; m_ndb = new Native;
if (ndb) if (m_ndb)
return true; return true;
} catch (const Xapian::Error &e) { } catch (const Xapian::Error &e) {
ermsg = e.get_msg().c_str(); ermsg = e.get_msg().c_str();
@ -229,12 +268,77 @@ bool Db::close()
LOGERR(("Db:close: exception while deleting db: %s\n", ermsg)); LOGERR(("Db:close: exception while deleting db: %s\n", ermsg));
return false; return false;
} }
bool Db::reOpen()
{
if (m_ndb->m_isopen) {
if (!close())
return false;
if (!open(m_ndb->m_basedir, m_ndb->m_mode, m_qOpts)) {
return false;
}
}
return true;
}
bool Db::addQueryDb(const string &dir)
{
LOGDEB(("Db::addQueryDb: ndb %p iswritable %d db [%s]\n", m_ndb,
(m_ndb)?m_ndb->m_iswritable:0, dir.c_str()));
if (!m_ndb)
return false;
if (m_ndb->m_iswritable)
return false;
if (find(m_ndb->m_extraDbs.begin(), m_ndb->m_extraDbs.end(), dir) ==
m_ndb->m_extraDbs.end()) {
m_ndb->m_extraDbs.push_back(dir);
}
return reOpen();
}
bool Db::rmQueryDb(const string &dir)
{
if (!m_ndb)
return false;
if (m_ndb->m_iswritable)
return false;
if (dir.empty()) {
m_ndb->m_extraDbs.clear();
} else {
list<string>::iterator it = find(m_ndb->m_extraDbs.begin(),
m_ndb->m_extraDbs.end(), dir);
if (it != m_ndb->m_extraDbs.end()) {
m_ndb->m_extraDbs.erase(it);
}
}
return reOpen();
}
bool Db::testDbDir(const string &dir)
{
string aerr;
LOGDEB(("Db::testDbDir: [%s]\n", dir.c_str()));
try {
Xapian::Database db(dir);
} catch (const Xapian::Error &e) {
aerr = e.get_msg().c_str();
} catch (const string &s) {
aerr = s.c_str();
} catch (const char *s) {
aerr = s;
} catch (...) {
aerr = "Caught unknown exception";
}
if (!aerr.empty()) {
LOGERR(("Db::Open: error while trying to open database "
"from [%s]: %s\n", dir.c_str(), aerr.c_str()));
return false;
}
return true;
}
bool Db::isopen() bool Db::isopen()
{ {
if (ndb == 0) if (m_ndb == 0)
return false; return false;
return ndb->isopen; return m_ndb->m_isopen;
} }
// A small class to hold state while splitting text // A small class to hold state while splitting text
@ -335,7 +439,7 @@ truncate_to_word(string & input, string::size_type maxlen)
return output; return output;
} }
// remove some chars and replace them with spaces // Remove some chars and replace them with spaces
static string stripchars(const string &str, string delims) static string stripchars(const string &str, string delims)
{ {
string out; string out;
@ -357,13 +461,6 @@ static string stripchars(const string &str, string delims)
return out; return out;
} }
// Truncate longer path and uniquize with hash . The goal for this is
// to avoid xapian max term length limitations, not to gain space (we
// gain very little even with very short maxlens like 30)
#define PATHHASHLEN 150
const static string rclSyntAbs = "?!#@";
// Add document in internal form to the database: index the terms in // Add document in internal form to the database: index the terms in
// the title abstract and body and add special terms for file name, // the title abstract and body and add special terms for file name,
// date, mime type ... , create the document data record (more // date, mime type ... , create the document data record (more
@ -372,7 +469,7 @@ bool Db::add(const string &fn, const Doc &idoc,
const struct stat *stp) const struct stat *stp)
{ {
LOGDEB1(("Db::add: fn %s\n", fn.c_str())); LOGDEB1(("Db::add: fn %s\n", fn.c_str()));
if (ndb == 0) if (m_ndb == 0)
return false; return false;
Doc doc = idoc; Doc doc = idoc;
@ -513,10 +610,10 @@ bool Db::add(const string &fn, const Doc &idoc,
// Add db entry or update existing entry: // Add db entry or update existing entry:
try { try {
Xapian::docid did = Xapian::docid did =
ndb->wdb.replace_document(uniterm.empty() ? pathterm : uniterm, m_ndb->wdb.replace_document(uniterm.empty() ? pathterm : uniterm,
newdocument); newdocument);
if (did < ndb->updated.size()) { if (did < m_ndb->updated.size()) {
ndb->updated[did] = true; m_ndb->updated[did] = true;
LOGDEB(("Db::add: docid %d updated [%s , %s]\n", did, fnc, LOGDEB(("Db::add: docid %d updated [%s , %s]\n", did, fnc,
doc.ipath.c_str())); doc.ipath.c_str()));
} else { } else {
@ -526,7 +623,7 @@ bool Db::add(const string &fn, const Doc &idoc,
} catch (...) { } catch (...) {
// FIXME: is this ever actually needed? // FIXME: is this ever actually needed?
try { try {
ndb->wdb.add_document(newdocument); m_ndb->wdb.add_document(newdocument);
LOGDEB(("Db::add: %s added (failed re-seek for duplicate)\n", LOGDEB(("Db::add: %s added (failed re-seek for duplicate)\n",
fnc)); fnc));
} catch (...) { } catch (...) {
@ -540,7 +637,7 @@ bool Db::add(const string &fn, const Doc &idoc,
// Test if given filename has changed since last indexed: // Test if given filename has changed since last indexed:
bool Db::needUpdate(const string &filename, const struct stat *stp) bool Db::needUpdate(const string &filename, const struct stat *stp)
{ {
if (ndb == 0) if (m_ndb == 0)
return false; return false;
// If no document exist with this path, we do need update // If no document exist with this path, we do need update
@ -556,16 +653,16 @@ bool Db::needUpdate(const string &filename, const struct stat *stp)
// file changed) // file changed)
Xapian::PostingIterator doc; Xapian::PostingIterator doc;
try { try {
if (!ndb->wdb.term_exists(pathterm)) { if (!m_ndb->wdb.term_exists(pathterm)) {
LOGDEB1(("Db::needUpdate: no such path: %s\n", pathterm.c_str())); LOGDEB1(("Db::needUpdate: no such path: %s\n", pathterm.c_str()));
return true; return true;
} }
Xapian::PostingIterator docid0 = ndb->wdb.postlist_begin(pathterm); Xapian::PostingIterator docid0 = m_ndb->wdb.postlist_begin(pathterm);
for (Xapian::PostingIterator docid = docid0; for (Xapian::PostingIterator docid = docid0;
docid != ndb->wdb.postlist_end(pathterm); docid++) { docid != m_ndb->wdb.postlist_end(pathterm); docid++) {
Xapian::Document doc = ndb->wdb.get_document(*docid); Xapian::Document doc = m_ndb->wdb.get_document(*docid);
// Check the date once. no need to look at the others if // Check the date once. no need to look at the others if
// the db needs updating. Note that the fmtime used to be // the db needs updating. Note that the fmtime used to be
@ -590,8 +687,8 @@ bool Db::needUpdate(const string &filename, const struct stat *stp)
} }
// Db is up to date. Make a note that this document exists. // Db is up to date. Make a note that this document exists.
if (*docid < ndb->updated.size()) if (*docid < m_ndb->updated.size())
ndb->updated[*docid] = true; m_ndb->updated[*docid] = true;
} }
return false; return false;
} catch (const Xapian::Error &e) { } catch (const Xapian::Error &e) {
@ -627,12 +724,12 @@ p_notlowerorutf(unsigned int c)
bool Db::deleteStemDb(const string& lang) bool Db::deleteStemDb(const string& lang)
{ {
LOGDEB(("Db::deleteStemDb(%s)\n", lang.c_str())); LOGDEB(("Db::deleteStemDb(%s)\n", lang.c_str()));
if (ndb == 0) if (m_ndb == 0)
return false; return false;
if (ndb->isopen == false) if (m_ndb->m_isopen == false)
return false; return false;
string dir = stemdbname(ndb->basedir, lang); string dir = stemdbname(m_ndb->m_basedir, lang);
if (wipedir(dir) == 0 && rmdir(dir.c_str()) == 0) if (wipedir(dir) == 0 && rmdir(dir.c_str()) == 0)
return true; return true;
return false; return false;
@ -647,9 +744,9 @@ bool Db::deleteStemDb(const string& lang)
bool Db::createStemDb(const string& lang) bool Db::createStemDb(const string& lang)
{ {
LOGDEB(("Db::createStemDb(%s)\n", lang.c_str())); LOGDEB(("Db::createStemDb(%s)\n", lang.c_str()));
if (ndb == 0) if (m_ndb == 0)
return false; return false;
if (ndb->isopen == false) if (m_ndb->m_isopen == false)
return false; return false;
// First build the in-memory stem database: // First build the in-memory stem database:
@ -667,8 +764,8 @@ bool Db::createStemDb(const string& lang)
try { try {
Xapian::Stem stemmer(lang); Xapian::Stem stemmer(lang);
Xapian::TermIterator it; Xapian::TermIterator it;
for (it = ndb->wdb.allterms_begin(); for (it = m_ndb->wdb.allterms_begin();
it != ndb->wdb.allterms_end(); it++) { it != m_ndb->wdb.allterms_end(); it++) {
// If it has any non-lowercase 7bit char, cant be stemmable // If it has any non-lowercase 7bit char, cant be stemmable
string::iterator sit = (*it).begin(), eit = sit + (*it).length(); string::iterator sit = (*it).begin(), eit = sit + (*it).length();
if ((sit = find_if(sit, eit, p_notlowerorutf)) != eit) { if ((sit = find_if(sit, eit, p_notlowerorutf)) != eit) {
@ -707,7 +804,7 @@ bool Db::createStemDb(const string& lang)
} }
}; };
// Create xapian database for stem relations // Create xapian database for stem relations
string stemdbdir = stemdbname(ndb->basedir, lang); string stemdbdir = stemdbname(m_ndb->m_basedir, lang);
// We want to get rid of the db dir in case of error. This gets disarmed // We want to get rid of the db dir in case of error. This gets disarmed
// just before success return. // just before success return.
DirWiper wiper(stemdbdir); DirWiper wiper(stemdbdir);
@ -781,10 +878,10 @@ list<string> Db::getStemLangs()
{ {
list<string> dirs; list<string> dirs;
LOGDEB(("Db::getStemLang\n")); LOGDEB(("Db::getStemLang\n"));
if (ndb == 0) if (m_ndb == 0)
return dirs; return dirs;
string pattern = stemdirstem + "*"; string pattern = stemdirstem + "*";
dirs = path_dirglob(ndb->basedir, pattern); dirs = path_dirglob(m_ndb->m_basedir, pattern);
for (list<string>::iterator it = dirs.begin(); it != dirs.end(); it++) { for (list<string>::iterator it = dirs.begin(); it != dirs.end(); it++) {
*it = path_basename(*it); *it = path_basename(*it);
*it = it->substr(stemdirstem.length(), string::npos); *it = it->substr(stemdirstem.length(), string::npos);
@ -801,11 +898,11 @@ list<string> Db::getStemLangs()
bool Db::purge() bool Db::purge()
{ {
LOGDEB(("Db::purge\n")); LOGDEB(("Db::purge\n"));
if (ndb == 0) if (m_ndb == 0)
return false; return false;
LOGDEB(("Db::purge: isopen %d iswritable %d\n", ndb->isopen, LOGDEB(("Db::purge: m_isopen %d m_iswritable %d\n", m_ndb->m_isopen,
ndb->iswritable)); m_ndb->m_iswritable));
if (ndb->isopen == false || ndb->iswritable == false) if (m_ndb->m_isopen == false || m_ndb->m_iswritable == false)
return false; return false;
// There seems to be problems with the document delete code, when // There seems to be problems with the document delete code, when
@ -816,14 +913,14 @@ bool Db::purge()
// trying to delete an unexistant document ? // trying to delete an unexistant document ?
// Flushing before trying the deletes seeems to work around the problem // Flushing before trying the deletes seeems to work around the problem
try { try {
ndb->wdb.flush(); m_ndb->wdb.flush();
} catch (...) { } catch (...) {
LOGDEB(("Db::purge: 1st flush failed\n")); LOGDEB(("Db::purge: 1st flush failed\n"));
} }
for (Xapian::docid docid = 1; docid < ndb->updated.size(); ++docid) { for (Xapian::docid docid = 1; docid < m_ndb->updated.size(); ++docid) {
if (!ndb->updated[docid]) { if (!m_ndb->updated[docid]) {
try { try {
ndb->wdb.delete_document(docid); m_ndb->wdb.delete_document(docid);
LOGDEB(("Db::purge: deleted document #%d\n", docid)); LOGDEB(("Db::purge: deleted document #%d\n", docid));
} catch (const Xapian::DocNotFoundError &) { } catch (const Xapian::DocNotFoundError &) {
LOGDEB(("Db::purge: document #%d not found\n", docid)); LOGDEB(("Db::purge: document #%d not found\n", docid));
@ -831,7 +928,7 @@ bool Db::purge()
} }
} }
try { try {
ndb->wdb.flush(); m_ndb->wdb.flush();
} catch (...) { } catch (...) {
LOGDEB(("Db::purge: 2nd flush failed\n")); LOGDEB(("Db::purge: 2nd flush failed\n"));
} }
@ -841,7 +938,7 @@ bool Db::purge()
/** /**
* Expand term to list of all terms which stem to the same term. * Expand term to list of all terms which stem to the same term.
*/ */
static list<string> stemexpand(Native *ndb, string term, const string& lang) static list<string> stemexpand(Native *m_ndb, string term, const string& lang)
{ {
list<string> explist; list<string> explist;
try { try {
@ -849,7 +946,7 @@ static list<string> stemexpand(Native *ndb, string term, const string& lang)
string stem = stemmer.stem_word(term); string stem = stemmer.stem_word(term);
LOGDEB(("stemexpand: [%s] stem-> [%s]\n", term.c_str(), stem.c_str())); LOGDEB(("stemexpand: [%s] stem-> [%s]\n", term.c_str(), stem.c_str()));
// Try to fetch the doc from the stem db // Try to fetch the doc from the stem db
string stemdbdir = stemdbname(ndb->basedir, lang); string stemdbdir = stemdbname(m_ndb->m_basedir, lang);
Xapian::Database sdb(stemdbdir); Xapian::Database sdb(stemdbdir);
LOGDEB1(("stemexpand: %s lastdocid: %d\n", LOGDEB1(("stemexpand: %s lastdocid: %d\n",
stemdbdir.c_str(), sdb.get_lastdocid())); stemdbdir.c_str(), sdb.get_lastdocid()));
@ -925,7 +1022,7 @@ class wsQData : public TextSplitCB {
// phrase terms (no stem expansion in this case) // phrase terms (no stem expansion in this case)
static void stringToXapianQueries(const string &iq, static void stringToXapianQueries(const string &iq,
const string& stemlang, const string& stemlang,
Native *ndb, Native *m_ndb,
list<Xapian::Query> &pqueries, list<Xapian::Query> &pqueries,
Db::QueryOpts opts = Db::QO_NONE) Db::QueryOpts opts = Db::QO_NONE)
{ {
@ -973,7 +1070,7 @@ static void stringToXapianQueries(const string &iq,
dumb_string(term, term1); dumb_string(term, term1);
// Possibly perform stem compression/expansion // Possibly perform stem compression/expansion
if (!nostemexp && (opts & Db::QO_STEM)) { if (!nostemexp && (opts & Db::QO_STEM)) {
exp = stemexpand(ndb, term1, stemlang); exp = stemexpand(m_ndb, term1, stemlang);
} else { } else {
exp.push_back(term1); exp.push_back(term1);
} }
@ -1001,18 +1098,18 @@ bool Db::setQuery(const std::string &iqstring, QueryOpts opts,
{ {
LOGDEB(("Db::setQuery: q: [%s], opts 0x%x, stemlang %s\n", LOGDEB(("Db::setQuery: q: [%s], opts 0x%x, stemlang %s\n",
iqstring.c_str(), (unsigned int)opts, stemlang.c_str())); iqstring.c_str(), (unsigned int)opts, stemlang.c_str()));
if (!ndb) if (!m_ndb)
return false; return false;
m_asdata.erase(); m_asdata.erase();
dbindices.clear(); m_dbindices.clear();
list<Xapian::Query> pqueries; list<Xapian::Query> pqueries;
stringToXapianQueries(iqstring, stemlang, ndb, pqueries, opts); stringToXapianQueries(iqstring, stemlang, m_ndb, pqueries, opts);
ndb->query = Xapian::Query(Xapian::Query::OP_OR, pqueries.begin(), m_ndb->query = Xapian::Query(Xapian::Query::OP_OR, pqueries.begin(),
pqueries.end()); pqueries.end());
delete ndb->enquire; delete m_ndb->enquire;
ndb->enquire = new Xapian::Enquire(ndb->db); m_ndb->enquire = new Xapian::Enquire(m_ndb->db);
ndb->enquire->set_query(ndb->query); m_ndb->enquire->set_query(m_ndb->query);
ndb->mset = Xapian::MSet(); m_ndb->mset = Xapian::MSet();
return true; return true;
} }
@ -1035,9 +1132,9 @@ bool Db::setQuery(AdvSearchData &sdata, QueryOpts opts,
LOGDEB((" restricted to: %s\n", sdata.topdir.c_str())); LOGDEB((" restricted to: %s\n", sdata.topdir.c_str()));
m_asdata = sdata; m_asdata = sdata;
dbindices.clear(); m_dbindices.clear();
if (!ndb) if (!m_ndb)
return false; return false;
list<Xapian::Query> pqueries; list<Xapian::Query> pqueries;
Xapian::Query xq; Xapian::Query xq;
@ -1062,10 +1159,10 @@ bool Db::setQuery(AdvSearchData &sdata, QueryOpts opts,
LOGDEB((" pattern: [%s]\n", pattern.c_str())); LOGDEB((" pattern: [%s]\n", pattern.c_str()));
// Match pattern against all file names in the db // Match pattern against all file names in the db
Xapian::TermIterator it = ndb->db.allterms_begin(); Xapian::TermIterator it = m_ndb->db.allterms_begin();
it.skip_to("XSFN"); it.skip_to("XSFN");
list<string> names; list<string> names;
for (;it != ndb->db.allterms_end(); it++) { for (;it != m_ndb->db.allterms_end(); it++) {
if ((*it).find("XSFN") != 0) if ((*it).find("XSFN") != 0)
break; break;
string fn = (*it).substr(4); string fn = (*it).substr(4);
@ -1089,7 +1186,7 @@ bool Db::setQuery(AdvSearchData &sdata, QueryOpts opts,
} }
if (!sdata.allwords.empty()) { if (!sdata.allwords.empty()) {
stringToXapianQueries(sdata.allwords, stemlang, ndb, pqueries, opts); stringToXapianQueries(sdata.allwords, stemlang, m_ndb, pqueries, opts);
if (!pqueries.empty()) { if (!pqueries.empty()) {
Xapian::Query nq = Xapian::Query nq =
Xapian::Query(Xapian::Query::OP_AND, pqueries.begin(), Xapian::Query(Xapian::Query::OP_AND, pqueries.begin(),
@ -1101,7 +1198,7 @@ bool Db::setQuery(AdvSearchData &sdata, QueryOpts opts,
} }
if (!sdata.orwords.empty()) { if (!sdata.orwords.empty()) {
stringToXapianQueries(sdata.orwords, stemlang, ndb, pqueries, opts); stringToXapianQueries(sdata.orwords, stemlang, m_ndb, pqueries, opts);
if (!pqueries.empty()) { if (!pqueries.empty()) {
Xapian::Query nq = Xapian::Query nq =
Xapian::Query(Xapian::Query::OP_OR, pqueries.begin(), Xapian::Query(Xapian::Query::OP_OR, pqueries.begin(),
@ -1114,7 +1211,7 @@ bool Db::setQuery(AdvSearchData &sdata, QueryOpts opts,
// We do no stem expansion on 'No' words. Should we ? // We do no stem expansion on 'No' words. Should we ?
if (!sdata.nowords.empty()) { if (!sdata.nowords.empty()) {
stringToXapianQueries(sdata.nowords, stemlang, ndb, pqueries); stringToXapianQueries(sdata.nowords, stemlang, m_ndb, pqueries);
if (!pqueries.empty()) { if (!pqueries.empty()) {
Xapian::Query nq; Xapian::Query nq;
nq = Xapian::Query(Xapian::Query::OP_OR, pqueries.begin(), nq = Xapian::Query(Xapian::Query::OP_OR, pqueries.begin(),
@ -1128,7 +1225,7 @@ bool Db::setQuery(AdvSearchData &sdata, QueryOpts opts,
if (!sdata.phrase.empty()) { if (!sdata.phrase.empty()) {
Xapian::Query nq; Xapian::Query nq;
string s = string("\"") + sdata.phrase + string("\""); string s = string("\"") + sdata.phrase + string("\"");
stringToXapianQueries(s, stemlang, ndb, pqueries); stringToXapianQueries(s, stemlang, m_ndb, pqueries);
if (!pqueries.empty()) { if (!pqueries.empty()) {
// There should be a single list element phrase query. // There should be a single list element phrase query.
xq = xq.empty() ? *pqueries.begin() : xq = xq.empty() ? *pqueries.begin() :
@ -1149,13 +1246,13 @@ bool Db::setQuery(AdvSearchData &sdata, QueryOpts opts,
xq = xq.empty() ? tq : Xapian::Query(Xapian::Query::OP_FILTER, xq, tq); xq = xq.empty() ? tq : Xapian::Query(Xapian::Query::OP_FILTER, xq, tq);
} }
ndb->query = xq; m_ndb->query = xq;
delete ndb->enquire; delete m_ndb->enquire;
ndb->enquire = new Xapian::Enquire(ndb->db); m_ndb->enquire = new Xapian::Enquire(m_ndb->db);
ndb->enquire->set_query(ndb->query); m_ndb->enquire->set_query(m_ndb->query);
ndb->mset = Xapian::MSet(); m_ndb->mset = Xapian::MSet();
// Get the query description and trim the "Xapian::Query" // Get the query description and trim the "Xapian::Query"
sdata.description = ndb->query.get_description(); sdata.description = m_ndb->query.get_description();
if (sdata.description.find("Xapian::Query") == 0) if (sdata.description.find("Xapian::Query") == 0)
sdata.description = sdata.description.substr(strlen("Xapian::Query")); sdata.description = sdata.description.substr(strlen("Xapian::Query"));
LOGDEB(("Db::SetQuery: Q: %s\n", sdata.description.c_str())); LOGDEB(("Db::SetQuery: Q: %s\n", sdata.description.c_str()));
@ -1164,32 +1261,33 @@ bool Db::setQuery(AdvSearchData &sdata, QueryOpts opts,
bool Db::getQueryTerms(list<string>& terms) bool Db::getQueryTerms(list<string>& terms)
{ {
if (!ndb) if (!m_ndb)
return false; return false;
terms.clear(); terms.clear();
Xapian::TermIterator it; Xapian::TermIterator it;
for (it = ndb->query.get_terms_begin(); it != ndb->query.get_terms_end(); for (it = m_ndb->query.get_terms_begin(); it != m_ndb->query.get_terms_end();
it++) { it++) {
terms.push_back(*it); terms.push_back(*it);
} }
return true; return true;
} }
// Mset size
static const int qquantum = 30; static const int qquantum = 30;
int Db::getResCnt() int Db::getResCnt()
{ {
if (!ndb || !ndb->enquire) { if (!m_ndb || !m_ndb->enquire) {
LOGERR(("Db::getResCnt: no query opened\n")); LOGERR(("Db::getResCnt: no query opened\n"));
return -1; return -1;
} }
if (ndb->mset.size() <= 0) { if (m_ndb->mset.size() <= 0) {
try { try {
ndb->mset = ndb->enquire->get_mset(0, qquantum); m_ndb->mset = m_ndb->enquire->get_mset(0, qquantum);
} catch (const Xapian::DatabaseModifiedError &error) { } catch (const Xapian::DatabaseModifiedError &error) {
ndb->db.reopen(); m_ndb->db.reopen();
ndb->mset = ndb->enquire->get_mset(0, qquantum); m_ndb->mset = m_ndb->enquire->get_mset(0, qquantum);
} catch (const Xapian::Error & error) { } catch (const Xapian::Error & error) {
LOGERR(("enquire->get_mset: exception: %s\n", LOGERR(("enquire->get_mset: exception: %s\n",
error.get_msg().c_str())); error.get_msg().c_str()));
@ -1197,16 +1295,9 @@ int Db::getResCnt()
} }
} }
return ndb->mset.get_matches_lower_bound(); return m_ndb->mset.get_matches_lower_bound();
} }
// This class (friend to RclDb) exists so that we can have functions that
// access private RclDb data and have Xapian-specific parameters (so that we
// don't want them to appear in the public rcldb.h).
class DbPops {
public:
};
bool Native::dbDataToRclDoc(std::string &data, Doc &doc, bool Native::dbDataToRclDoc(std::string &data, Doc &doc,
int qopts, int qopts,
Xapian::docid docid, const list<string>& terms) Xapian::docid docid, const list<string>& terms)
@ -1252,7 +1343,7 @@ bool Native::dbDataToRclDoc(std::string &data, Doc &doc,
bool Db::getDoc(int exti, Doc &doc, int *percent) bool Db::getDoc(int exti, Doc &doc, int *percent)
{ {
LOGDEB1(("Db::getDoc: exti %d\n", exti)); LOGDEB1(("Db::getDoc: exti %d\n", exti));
if (!ndb || !ndb->enquire) { if (!m_ndb || !m_ndb->enquire) {
LOGERR(("Db::getDoc: no query opened\n")); LOGERR(("Db::getDoc: no query opened\n"));
return false; return false;
} }
@ -1264,85 +1355,85 @@ bool Db::getDoc(int exti, Doc &doc, int *percent)
int xapi; int xapi;
if (postqfilter) { if (postqfilter) {
// There is a postquery filter, does this fall in already known area ? // There is a postquery filter, does this fall in already known area ?
if (exti >= (int)dbindices.size()) { if (exti >= (int)m_dbindices.size()) {
// Have to fetch xapian docs and filter until we get // Have to fetch xapian docs and filter until we get
// enough or fail // enough or fail
dbindices.reserve(exti+1); m_dbindices.reserve(exti+1);
// First xapian doc we fetch is the one after last stored // First xapian doc we fetch is the one after last stored
int first = dbindices.size() > 0 ? dbindices.back() + 1 : 0; int first = m_dbindices.size() > 0 ? m_dbindices.back() + 1 : 0;
// Loop until we get enough docs // Loop until we get enough docs
while (exti >= (int)dbindices.size()) { while (exti >= (int)m_dbindices.size()) {
LOGDEB(("Db::getDoc: fetching %d starting at %d\n", LOGDEB(("Db::getDoc: fetching %d starting at %d\n",
qquantum, first)); qquantum, first));
try { try {
ndb->mset = ndb->enquire->get_mset(first, qquantum); m_ndb->mset = m_ndb->enquire->get_mset(first, qquantum);
} catch (const Xapian::DatabaseModifiedError &error) { } catch (const Xapian::DatabaseModifiedError &error) {
ndb->db.reopen(); m_ndb->db.reopen();
ndb->mset = ndb->enquire->get_mset(first, qquantum); m_ndb->mset = m_ndb->enquire->get_mset(first, qquantum);
} catch (const Xapian::Error & error) { } catch (const Xapian::Error & error) {
LOGERR(("enquire->get_mset: exception: %s\n", LOGERR(("enquire->get_mset: exception: %s\n",
error.get_msg().c_str())); error.get_msg().c_str()));
abort(); abort();
} }
if (ndb->mset.empty()) { if (m_ndb->mset.empty()) {
LOGDEB(("Db::getDoc: got empty mset\n")); LOGDEB(("Db::getDoc: got empty mset\n"));
return false; return false;
} }
first = ndb->mset.get_firstitem(); first = m_ndb->mset.get_firstitem();
for (unsigned int i = 0; i < ndb->mset.size() ; i++) { for (unsigned int i = 0; i < m_ndb->mset.size() ; i++) {
LOGDEB(("Db::getDoc: [%d]\n", i)); LOGDEB(("Db::getDoc: [%d]\n", i));
Xapian::Document xdoc = ndb->mset[i].get_document(); Xapian::Document xdoc = m_ndb->mset[i].get_document();
if (ndb->filterMatch(this, xdoc)) { if (m_ndb->filterMatch(this, xdoc)) {
dbindices.push_back(first + i); m_dbindices.push_back(first + i);
} }
} }
first = first + ndb->mset.size(); first = first + m_ndb->mset.size();
} }
} }
xapi = dbindices[exti]; xapi = m_dbindices[exti];
} else { } else {
xapi = exti; xapi = exti;
} }
// From there on, we work with a xapian enquire item number. Fetch it // From there on, we work with a xapian enquire item number. Fetch it
int first = ndb->mset.get_firstitem(); int first = m_ndb->mset.get_firstitem();
int last = first + ndb->mset.size() -1; int last = first + m_ndb->mset.size() -1;
if (!(xapi >= first && xapi <= last)) { if (!(xapi >= first && xapi <= last)) {
LOGDEB(("Fetching for first %d, count %d\n", xapi, qquantum)); LOGDEB(("Fetching for first %d, count %d\n", xapi, qquantum));
try { try {
ndb->mset = ndb->enquire->get_mset(xapi, qquantum); m_ndb->mset = m_ndb->enquire->get_mset(xapi, qquantum);
} catch (const Xapian::DatabaseModifiedError &error) { } catch (const Xapian::DatabaseModifiedError &error) {
ndb->db.reopen(); m_ndb->db.reopen();
ndb->mset = ndb->enquire->get_mset(xapi, qquantum); m_ndb->mset = m_ndb->enquire->get_mset(xapi, qquantum);
} catch (const Xapian::Error & error) { } catch (const Xapian::Error & error) {
LOGERR(("enquire->get_mset: exception: %s\n", LOGERR(("enquire->get_mset: exception: %s\n",
error.get_msg().c_str())); error.get_msg().c_str()));
abort(); abort();
} }
if (ndb->mset.empty()) if (m_ndb->mset.empty())
return false; return false;
first = ndb->mset.get_firstitem(); first = m_ndb->mset.get_firstitem();
last = first + ndb->mset.size() -1; last = first + m_ndb->mset.size() -1;
} }
LOGDEB1(("Db::getDoc: Qry [%s] win [%d-%d] Estimated results: %d", LOGDEB1(("Db::getDoc: Qry [%s] win [%d-%d] Estimated results: %d",
ndb->query.get_description().c_str(), m_ndb->query.get_description().c_str(),
first, last, first, last,
ndb->mset.get_matches_lower_bound())); m_ndb->mset.get_matches_lower_bound()));
Xapian::Document xdoc = ndb->mset[xapi-first].get_document(); Xapian::Document xdoc = m_ndb->mset[xapi-first].get_document();
Xapian::docid docid = *(ndb->mset[xapi-first]); Xapian::docid docid = *(m_ndb->mset[xapi-first]);
if (percent) if (percent)
*percent = ndb->mset.convert_to_percent(ndb->mset[xapi-first]); *percent = m_ndb->mset.convert_to_percent(m_ndb->mset[xapi-first]);
// Parse xapian document's data and populate doc fields // Parse xapian document's data and populate doc fields
string data = xdoc.get_data(); string data = xdoc.get_data();
list<string> terms; list<string> terms;
getQueryTerms(terms); getQueryTerms(terms);
return ndb->dbDataToRclDoc(data, doc, m_qOpts, docid, terms); return m_ndb->dbDataToRclDoc(data, doc, m_qOpts, docid, terms);
} }
// Retrieve document defined by file name and internal path. Very inefficient, // Retrieve document defined by file name and internal path. Very inefficient,
@ -1352,7 +1443,7 @@ bool Db::getDoc(const string &fn, const string &ipath, Doc &doc, int *pc)
{ {
LOGDEB(("Db:getDoc: [%s] (%d) [%s]\n", fn.c_str(), fn.length(), LOGDEB(("Db:getDoc: [%s] (%d) [%s]\n", fn.c_str(), fn.length(),
ipath.c_str())); ipath.c_str()));
if (ndb == 0) if (m_ndb == 0)
return false; return false;
// Initialize what we can in any case. If this is history, caller // Initialize what we can in any case. If this is history, caller
@ -1369,7 +1460,7 @@ bool Db::getDoc(const string &fn, const string &ipath, Doc &doc, int *pc)
// with the appropriate ipath. This is very inefficient. // with the appropriate ipath. This is very inefficient.
const char *ermsg = ""; const char *ermsg = "";
try { try {
if (!ndb->db.term_exists(pathterm)) { if (!m_ndb->db.term_exists(pathterm)) {
// Document found in history no longer in the database. // Document found in history no longer in the database.
// We return true (because their might be other ok docs further) // We return true (because their might be other ok docs further)
// but indicate the error with pc = -1 // but indicate the error with pc = -1
@ -1380,13 +1471,13 @@ bool Db::getDoc(const string &fn, const string &ipath, Doc &doc, int *pc)
return true; return true;
} }
for (Xapian::PostingIterator docid = for (Xapian::PostingIterator docid =
ndb->db.postlist_begin(pathterm); m_ndb->db.postlist_begin(pathterm);
docid != ndb->db.postlist_end(pathterm); docid++) { docid != m_ndb->db.postlist_end(pathterm); docid++) {
Xapian::Document xdoc = ndb->db.get_document(*docid); Xapian::Document xdoc = m_ndb->db.get_document(*docid);
string data = xdoc.get_data(); string data = xdoc.get_data();
list<string> terms; list<string> terms;
if (ndb->dbDataToRclDoc(data, doc, QO_NONE, *docid, terms) if (m_ndb->dbDataToRclDoc(data, doc, QO_NONE, *docid, terms)
&& doc.ipath == ipath) && doc.ipath == ipath)
return true; return true;
} }

24
src/rcldb/rcldb.h
View File

@ -16,11 +16,12 @@
*/ */
#ifndef _DB_H_INCLUDED_ #ifndef _DB_H_INCLUDED_
#define _DB_H_INCLUDED_ #define _DB_H_INCLUDED_
/* @(#$Id: rcldb.h,v 1.28 2006-04-05 06:26:56 dockes Exp $ (C) 2004 J.F.Dockes */ /* @(#$Id: rcldb.h,v 1.29 2006-04-05 12:50:42 dockes Exp $ (C) 2004 J.F.Dockes */
#include <string> #include <string>
#include <list> #include <list>
#include <vector> #include <vector>
#ifndef NO_NAMESPACES #ifndef NO_NAMESPACES
using std::string; using std::string;
using std::list; using std::list;
@ -156,6 +157,13 @@ class Db {
const string& stemlang = "english"); const string& stemlang = "english");
bool getQueryTerms(list<string>& terms); bool getQueryTerms(list<string>& terms);
/// Add extra database for querying
bool addQueryDb(const string &dir);
/// Remove extra database. if dir == "", remove all.
bool rmQueryDb(const string &dir);
/// Tell if directory seems to hold xapian db
static bool testDbDir(const string &dir);
/** Get document at rank i in current query. /** Get document at rank i in current query.
This is probably vastly inferior to the type of interface in This is probably vastly inferior to the type of interface in
@ -173,18 +181,20 @@ class Db {
/** Get a list of existing stemming databases */ /** Get a list of existing stemming databases */
std::list<std::string> getStemLangs(); std::list<std::string> getStemLangs();
/** Things we don't want to have here. */
friend class Native;
private: private:
AdvSearchData m_asdata; AdvSearchData m_asdata;
vector<int> dbindices; // In case there is a postq filter: sequence of vector<int> m_dbindices; // In case there is a postq filter: sequence of
// db indices that match // db indices that match
Native *ndb; // Pointer to private data. We don't want db(ie
// Things we don't want to have here.
friend class Native;
Native *m_ndb; // Pointer to private data. We don't want db(ie
// xapian)-specific defs to show in here // xapian)-specific defs to show in here
unsigned int m_qOpts; unsigned int m_qOpts;
bool reOpen(); // Close/open, same mode/opts
/* Copyconst and assignemt private and forbidden */ /* Copyconst and assignemt private and forbidden */
Db(const Db &) {} Db(const Db &) {}
Db & operator=(const Db &) {return *this;}; Db & operator=(const Db &) {return *this;};