tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added field/prefixes for author and title + command line query language

Browse Source
This commit is contained in:
dockes
2007-01-17 13:53:41 +00:00
parent ee85be5c61
commit 1d683ad411
15 changed files with 256 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rcldb/rcldb.cpp
+59 -20
View File
@@ -1,5 +1,5 @@
#ifndef lint
static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.101 2006-12-19 12:11:21 dockes Exp $ (C) 2004 J.F.Dockes";
static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.102 2007-01-17 13:53:41 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
@@ -80,6 +80,7 @@ namespace Rcl {
// Synthetic abstract marker (to discriminate from abstract actually
// found in doc)
const static string rclSyntAbs = "?!#@";
const static string emptystring;
// A class for data and methods that would have to expose
// Xapian-specific stuff if they were in Rcl::Db. There could actually be
@@ -703,15 +704,24 @@ bool Db::isopen()
return m_ndb->m_isopen;
}
// A small class to hold state while splitting text
// The text splitter callback class which receives words from the
// splitter and adds postings to the Xapian document.
class mySplitterCB : public TextSplitCB {
public:
Xapian::Document &doc;
Xapian::Document &doc; // Xapian document
Xapian::termpos basepos; // Base for document section
Xapian::termpos curpos; // Last position sent to callback
mySplitterCB(Xapian::Document &d) : doc(d), basepos(1), curpos(0)
Xapian::termpos curpos; // Current position. Used to set basepos for the
// following section
mySplitterCB(Xapian::Document &d)
: doc(d), basepos(1), curpos(0)
{}
bool takeword(const std::string &term, int pos, int, int);
void setprefix(const string& pref) {prefix = pref;}
private:
// If prefix is set, we also add a posting for the prefixed terms
// (ie: for titles, add postings for both "term" and "Sterm")
string prefix;
};
// Callback for the document to word splitting class during indexation
@@ -731,7 +741,11 @@ bool mySplitterCB::takeword(const std::string &term, int pos, int, int)
// be possible to assign different weigths to doc parts (ie title)
// by using a higher value
curpos = pos;
doc.add_posting(term, basepos + curpos, 1);
pos += basepos;
doc.add_posting(term, pos, 1);
if (!prefix.empty()) {
doc.add_posting(prefix + term, pos, 1);
}
return true;
} catch (const Xapian::Error &e) {
ermsg = e.get_msg().c_str();
@@ -804,8 +818,9 @@ bool Db::add(const string &fn, const Doc &idoc,
doc.abstract = truncate_to_word(doc.abstract, m_idxAbsTruncLen);
}
doc.abstract = neutchars(doc.abstract, "\n\r");
doc.title = truncate_to_word(doc.title, 100);
doc.keywords = truncate_to_word(doc.keywords, 300);
doc.title = neutchars(truncate_to_word(doc.title, 150), "\n\r");
doc.author = neutchars(truncate_to_word(doc.author, 150), "\n\r");
doc.keywords = neutchars(truncate_to_word(doc.keywords, 300), "\n\r");
Xapian::Document newdocument;
@@ -824,13 +839,30 @@ bool Db::add(const string &fn, const Doc &idoc,
}
// Split and index title
LOGDEB2(("Db::add: split title [%s]\n", doc.title.c_str()));
if (!dumb_string(doc.title, noacc)) {
LOGERR(("Db::add: dumb_string failed\n"));
return false;
if (!doc.title.empty()) {
LOGDEB2(("Db::add: split title [%s]\n", doc.title.c_str()));
if (!dumb_string(doc.title, noacc)) {
LOGERR(("Db::add: dumb_string failed\n"));
return false;
}
splitData.setprefix("S"); // Subject
splitter.text_to_words(noacc);
splitData.setprefix(emptystring);
splitData.basepos += splitData.curpos + 100;
}
// Split and index author
if (!doc.author.empty()) {
LOGDEB2(("Db::add: split author [%s]\n", doc.author.c_str()));
if (!dumb_string(doc.author, noacc)) {
LOGERR(("Db::add: dumb_string failed\n"));
return false;
}
splitData.setprefix("A");
splitter.text_to_words(noacc);
splitData.setprefix(emptystring);
splitData.basepos += splitData.curpos + 100;
}
splitter.text_to_words(noacc);
splitData.basepos += splitData.curpos + 100;
// Split and index body
LOGDEB2(("Db::add: split body\n"));
@@ -842,13 +874,17 @@ bool Db::add(const string &fn, const Doc &idoc,
splitData.basepos += splitData.curpos + 100;
// Split and index keywords
LOGDEB2(("Db::add: split kw [%s]\n", doc.keywords.c_str()));
if (!dumb_string(doc.keywords, noacc)) {
LOGERR(("Db::add: dumb_string failed\n"));
return false;
if (!doc.keywords.empty()) {
LOGDEB2(("Db::add: split kw [%s]\n", doc.keywords.c_str()));
if (!dumb_string(doc.keywords, noacc)) {
LOGERR(("Db::add: dumb_string failed\n"));
return false;
}
splitData.setprefix("K");
splitter.text_to_words(noacc);
splitData.setprefix(emptystring);
splitData.basepos += splitData.curpos + 100;
}
splitter.text_to_words(noacc);
splitData.basepos += splitData.curpos + 100;
// Split and index abstract. We don't do this if it is synthetic
// any more (this used to give a relevance boost to the beginning
@@ -946,6 +982,9 @@ bool Db::add(const string &fn, const Doc &idoc,
record += "\ncaption=" + doc.title;
record += "\nkeywords=" + doc.keywords;
record += "\nabstract=" + doc.abstract;
if (!doc.author.empty()) {
record += "\nauthor=" + doc.author;
}
record += "\n";
LOGDEB1(("Newdocument data: %s\n", record.c_str()));
newdocument.set_data(record);
src/rcldb/rcldoc.h
+2 -1
View File
@@ -16,7 +16,7 @@
*/
#ifndef _RCLDOC_H_INCLUDED_
#define _RCLDOC_H_INCLUDED_
/* @(#$Id: rcldoc.h,v 1.1 2006-12-14 14:54:13 dockes Exp $ (C) 2006 J.F.Dockes */
/* @(#$Id: rcldoc.h,v 1.2 2007-01-17 13:53:41 dockes Exp $ (C) 2006 J.F.Dockes */
#include <string>
@@ -48,6 +48,7 @@ class Doc {
string origcharset; // Charset we transcoded from (in case we want back)
// Possibly set by handler
string title; // Possibly set by handler
string author; // Possibly set by handler
string keywords; // Possibly set by handler
string abstract; // Possibly set by handler
bool syntabs; // true if abstract is just the top of doc, not an
src/rcldb/searchdata.cpp
+55 -14
View File
@@ -1,5 +1,5 @@
#ifndef lint
static char rcsid[] = "@(#$Id: searchdata.cpp,v 1.7 2006-12-19 12:11:21 dockes Exp $ (C) 2006 J.F.Dockes";
static char rcsid[] = "@(#$Id: searchdata.cpp,v 1.8 2007-01-17 13:53:41 dockes Exp $ (C) 2006 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
@@ -154,6 +154,7 @@ public:
bool translate(const string &iq,
const string &prefix,
string &ermsg,
list<Xapian::Query> &pqueries,
int slack = 0, bool useNear = false);
@@ -257,6 +258,14 @@ void multiply_groups(vector<vector<string> >::const_iterator vvit,
}
}
static void addPrefix(list<string>& terms, const string& prefix)
{
if (prefix.empty())
return;
for (list<string>::iterator it = terms.begin(); it != terms.end(); it++)
it->insert(0, prefix);
}
/**
* Turn string into list of xapian queries. There is little
* interpretation done on the string (no +term -term or filename:term
@@ -271,6 +280,7 @@ void multiply_groups(vector<vector<string> >::const_iterator vvit,
* count)
*/
bool StringToXapianQ::translate(const string &iq,
const string &prefix,
string &ermsg,
list<Xapian::Query> &pqueries,
int slack, bool useNear)
@@ -301,24 +311,25 @@ bool StringToXapianQ::translate(const string &iq,
splitterS.text_to_words(*it);
TextSplit splitterW(&splitDataW, TextSplit::TXTS_NOSPANS);
splitterW.text_to_words(*it);
wsQData& splitData = splitDataS;
if (splitDataS.terms.size() > 1 && splitDataS.terms.size() !=
splitDataW.terms.size())
splitData = splitDataW;
wsQData *splitData = &splitDataS;
if (splitDataS.terms.size() > 1 &&
splitDataS.terms.size() != splitDataW.terms.size())
splitData = &splitDataW;
LOGDEB1(("strToXapianQ: splitter term count: %d\n",
splitData.terms.size()));
switch(splitData.terms.size()) {
splitData->terms.size()));
switch(splitData->terms.size()) {
case 0: continue;// ??
case 1: // Not a real phrase: one term
{
string term = splitData.terms.front();
string term = splitData->terms.front();
list<string> exp;
maybeStemExp(false, term, exp);
m_terms.insert(m_terms.end(), exp.begin(), exp.end());
// Push either term or OR of stem-expanded set
addPrefix(exp, prefix);
pqueries.push_back(Xapian::Query(Xapian::Query::OP_OR,
exp.begin(), exp.end()));
m_terms.insert(m_terms.end(), exp.begin(), exp.end());
}
break;
@@ -329,8 +340,8 @@ bool StringToXapianQ::translate(const string &iq,
list<Xapian::Query> orqueries;
bool hadmultiple = false;
vector<vector<string> >groups;
for (vector<string>::iterator it = splitData.terms.begin();
it != splitData.terms.end(); it++) {
for (vector<string>::iterator it = splitData->terms.begin();
it != splitData->terms.end(); it++) {
// Some version of xapian will accept only one OR clause
// inside NEAR, all others must be leafs
bool nostemexp =
@@ -341,6 +352,7 @@ bool StringToXapianQ::translate(const string &iq,
maybeStemExp(nostemexp, *it, exp);
groups.push_back(vector<string>(exp.begin(), exp.end()));
addPrefix(exp, prefix);
orqueries.push_back(Xapian::Query(Xapian::Query::OP_OR,
exp.begin(), exp.end()));
#ifdef XAPIAN_NEAR_EXPAND_SINGLE_BUF
@@ -352,7 +364,7 @@ bool StringToXapianQ::translate(const string &iq,
pqueries.push_back(Xapian::Query(op,
orqueries.begin(),
orqueries.end(),
splitData.terms.size() + slack));
splitData->terms.size() + slack));
// Add NEAR/PHRASE groups to the highlighting data. Must
// push all combinations
vector<vector<string> > allcombs;
@@ -378,6 +390,28 @@ bool StringToXapianQ::translate(const string &iq,
return true;
}
// Try to translate field specification into field prefix. This should
// probably be an Rcl::Db method and much more configurable (store
// prefix translation list in config ?)
static string fieldToPrefix(const string& i_field)
{
static map<string, string> fldToPrefs;
if (fldToPrefs.empty()) {
fldToPrefs["title"] = "S";
fldToPrefs["caption"] = "S";
fldToPrefs["subject"] = "S";
fldToPrefs["author"] = "A";
fldToPrefs["from"] = "A";
fldToPrefs["keyword"] = "K";
}
string fld(i_field);
stringtolower(fld);
map<string, string>::const_iterator it = fldToPrefs.find(fld);
if (it != fldToPrefs.end())
return it->second;
return "";
}
// Translate a simple OR, AND, or EXCL search clause.
bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p,
const string& stemlang)
@@ -397,9 +431,12 @@ bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p,
LOGERR(("SearchDataClauseSimple: bad m_tp %d\n", m_tp));
return false;
}
string prefix;
if (!m_field.empty())
prefix = fieldToPrefix(m_field);
list<Xapian::Query> pqueries;
StringToXapianQ tr(db, stemlang);
if (!tr.translate(m_text, m_reason, pqueries))
if (!tr.translate(m_text, prefix, m_reason, pqueries))
return false;
if (pqueries.empty()) {
LOGERR(("SearchDataClauseSimple: resolved to null query\n"));
@@ -437,12 +474,16 @@ bool SearchDataClauseDist::toNativeQuery(Rcl::Db &db, void *p,
list<Xapian::Query> pqueries;
Xapian::Query nq;
string prefix;
if (!m_field.empty())
prefix = fieldToPrefix(m_field);
// Use stringToXapianQueries to lowercase and simplify the phrase
// terms etc. The result should be a single element list
string s = string("\"") + m_text + string("\"");
bool useNear = m_tp == SCLT_NEAR;
StringToXapianQ tr(db, stemlang);
if (!tr.translate(s, m_reason, pqueries, m_slack, useNear))
if (!tr.translate(s, prefix, m_reason, pqueries, m_slack, useNear))
return false;
if (pqueries.empty()) {
LOGERR(("SearchDataClauseDist: resolved to null query\n"));
src/rcldb/searchdata.h
+33 -16
View File
@@ -16,7 +16,7 @@
*/
#ifndef _SEARCHDATA_H_INCLUDED_
#define _SEARCHDATA_H_INCLUDED_
/* @(#$Id: searchdata.h,v 1.7 2006-12-05 15:17:13 dockes Exp $ (C) 2004 J.F.Dockes */
/* @(#$Id: searchdata.h,v 1.8 2007-01-17 13:53:41 dockes Exp $ (C) 2004 J.F.Dockes */
/**
* Structures to hold data coming almost directly from the gui
@@ -46,11 +46,22 @@ enum SClType {
class SearchDataClause;
/**
* Holder for a list of search clauses. Some of the clauses may be be reference
* to other subqueries in the future. For now, they just reflect user entry in
* a query field: type, some text and possibly a distance. Each clause may
* hold several queries in the Xapian sense, for exemple several terms
* and phrases as would result from ["this is a phrase" term1 term2]
Data structure representing A Recoll query.
This is currently simply a list of search clauses.
For now, clauses in the list just reflect user entry in a query
field: some text, a clause type (AND/OR/NEAR etc.) and possibly a
distance. Each clause may hold several queries in the Xapian sense,
for exemple several terms and phrases as would result from
["this is a phrase" term1 term2]
This means that SearchData will be translated into a Xapian
Query tree of depth 2.
The structure might be extended in the future so that some of the
clauses may be references to other subqueries (there doesn't seem to
be an urgent need for this)
*/
class SearchData {
public:
@@ -134,15 +145,19 @@ protected:
*/
class SearchDataClauseSimple : public SearchDataClause {
public:
SearchDataClauseSimple(SClType tp, string txt)
: SearchDataClause(tp), m_text(txt), m_slack(0) {}
SearchDataClauseSimple(SClType tp, const string& txt,
const string& fld = "")
: SearchDataClause(tp), m_text(txt), m_field(fld), m_slack(0) {}
virtual ~SearchDataClauseSimple() {}
/** Translate to Xapian query */
virtual bool toNativeQuery(Rcl::Db &db, void *, const string& stemlang);
virtual bool getTerms(vector<string>& terms,
vector<vector<string> >& groups,
vector<int>& gslks) const
/** Retrieve query terms and term groups. This is used for highlighting */
virtual bool getTerms(vector<string>& terms, /* Single terms */
vector<vector<string> >& groups, /* Prox grps */
vector<int>& gslks) const /* Prox slacks */
{
terms.insert(terms.end(), m_terms.begin(), m_terms.end());
groups.insert(groups.end(), m_groups.begin(), m_groups.end());
@@ -151,7 +166,8 @@ public:
}
protected:
string m_text;
string m_text; // Raw user entry text.
string m_field; // Field specification if any
// Single terms and phrases resulting from breaking up m_text;
// valid after toNativeQuery() call
vector<string> m_terms;
@@ -161,10 +177,10 @@ protected:
int m_slack;
};
/** Filename search. */
/** Filename search clause. */
class SearchDataClauseFilename : public SearchDataClauseSimple {
public:
SearchDataClauseFilename(string txt)
SearchDataClauseFilename(const string& txt)
: SearchDataClauseSimple(SCLT_FILENAME, txt) {}
virtual ~SearchDataClauseFilename() {}
virtual bool toNativeQuery(Rcl::Db &db, void *, const string& stemlang);
@@ -176,8 +192,9 @@ public:
*/
class SearchDataClauseDist : public SearchDataClauseSimple {
public:
SearchDataClauseDist(SClType tp, string txt, int slack)
: SearchDataClauseSimple(tp, txt) {m_slack = slack;}
SearchDataClauseDist(SClType tp, const string& txt, int slack,
const string& fld = "")
: SearchDataClauseSimple(tp, txt, fld) {m_slack = slack;}
virtual ~SearchDataClauseDist() {}
virtual bool toNativeQuery(Rcl::Db &db, void *, const string& stemlang);