tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

merged the case/diac sensitivity code back into trunk

Browse Source
This commit is contained in:
"Jean-Francois Dockes ext:(%22)
2012-09-25 19:20:24 +02:00
parent a2be0d36ef 1bab5ae658
commit 52bc9f4aa3
40 changed files with 1155 additions and 595 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/query/docseqdb.cpp
+3 -1
View File
@@ -146,7 +146,8 @@ bool DocSequenceDb::setFiltSpec(const DocSeqFiltSpec &fs)
LOGDEB(("DocSequenceDb::setFiltSpec\n"));
if (fs.isNotNull()) {
// We build a search spec by adding a filtering layer to the base one.
m_fsdata = RefCntr<Rcl::SearchData>(new Rcl::SearchData(Rcl::SCLT_AND));
m_fsdata = RefCntr<Rcl::SearchData>(
new Rcl::SearchData(Rcl::SCLT_AND, m_sdata->getStemLang()));
Rcl::SearchDataClauseSub *cl =
new Rcl::SearchDataClauseSub(Rcl::SCLT_SUB, m_sdata);
m_fsdata->addClause(cl);
@@ -164,6 +165,7 @@ bool DocSequenceDb::setFiltSpec(const DocSeqFiltSpec &fs)
string reason;
Rcl::SearchData *sd =
wasaStringToRcl(m_q->whatDb()->getConf(),
m_sdata->getStemLang(),
fs.values[i], reason);
if (sd) {
Rcl::SearchDataClauseSub *cl1 =
src/query/plaintorich.cpp
+24 -1
View File
@@ -50,7 +50,10 @@ static string vecStringToString(const vector<string>& t)
}
struct MatchEntry {
// Start/End byte offsets in the document text
pair<int, int> offs;
// Index of the search group this comes from: this is to relate a
// match to the original user input.
unsigned int grpidx;
MatchEntry(int sta, int sto, unsigned int idx)
: offs(sta, sto), grpidx(idx)
@@ -76,11 +79,31 @@ class TextSplitPTR : public TextSplit {
for (vector<vector<string> >::const_iterator vit = hdata.groups.begin();
vit != hdata.groups.end(); vit++) {
if (vit->size() == 1) {
m_terms[vit->front()] = vit - hdata.groups.begin();
#ifndef RCL_INDEX_STRIPCHARS
if (o_index_stripchars) {
#endif
m_terms[vit->front()] = vit - hdata.groups.begin();
#ifndef RCL_INDEX_STRIPCHARS
} else {
string dumb = vit->front();
unacmaybefold(vit->front(), dumb, "UTF-8", UNACOP_UNACFOLD);
m_terms[dumb] = vit - hdata.groups.begin();
}
#endif
} else if (vit->size() > 1) {
for (vector<string>::const_iterator it = vit->begin();
it != vit->end(); it++) {
#ifndef RCL_INDEX_STRIPCHARS
if (o_index_stripchars) {
#endif
m_gterms.insert(*it);
#ifndef RCL_INDEX_STRIPCHARS
} else {
string dumb = *it;
unacmaybefold(*it, dumb, "UTF-8", UNACOP_UNACFOLD);
m_gterms.insert(dumb);
}
#endif
}
}
}
src/query/recollq.cpp
+2 -3
View File
@@ -286,7 +286,7 @@ int recollq(RclConfig **cfp, int argc, char **argv)
Rcl::SearchData *sd = 0;
if (op_flags & (OPT_a|OPT_o|OPT_f)) {
sd = new Rcl::SearchData(Rcl::SCLT_OR);
sd = new Rcl::SearchData(Rcl::SCLT_OR, stemlang);
Rcl::SearchDataClause *clp = 0;
if (op_flags & OPT_f) {
clp = new Rcl::SearchDataClauseFilename(qs);
@@ -305,14 +305,13 @@ int recollq(RclConfig **cfp, int argc, char **argv)
if (sd)
sd->addClause(clp);
} else {
sd = wasaStringToRcl(rclconfig, qs, reason);
sd = wasaStringToRcl(rclconfig, stemlang, qs, reason);
}
if (!sd) {
cerr << "Query string interpretation failed: " << reason << endl;
return 1;
}
sd->setStemlang(stemlang);
RefCntr<Rcl::SearchData> rq(sd);
Rcl::Query query(&rcldb);
src/query/reslistpager.cpp
+10 -3
View File
@@ -320,9 +320,16 @@ void ResListPager::displayPage(RclConfig *config)
map<string, vector<string> > spellings;
suggest(uterms, spellings);
if (!spellings.empty()) {
chunk <<
trans("<p><i>Alternate spellings (accents suppressed): </i>")
<< "<br /><blockquote>";
if (o_index_stripchars) {
chunk <<
trans("<p><i>Alternate spellings (accents suppressed): </i>")
<< "<br /><blockquote>";
} else {
chunk <<
trans("<p><i>Alternate spellings: </i>")
<< "<br /><blockquote>";
}
for (map<string, vector<string> >::const_iterator it0 =
spellings.begin(); it0 != spellings.end(); it0++) {
src/query/wasatorcl.cpp
+9 -6
View File
@@ -32,7 +32,9 @@ using std::list;
#include "refcntr.h"
#include "textsplit.h"
static Rcl::SearchData *wasaQueryToRcl(RclConfig *config, WasaQuery *wasa,
static Rcl::SearchData *wasaQueryToRcl(RclConfig *config,
const string& stemlang,
WasaQuery *wasa,
const string& autosuffs, string& reason)
{
if (wasa == 0) {
@@ -47,7 +49,7 @@ static Rcl::SearchData *wasaQueryToRcl(RclConfig *config, WasaQuery *wasa,
Rcl::SearchData *sdata = new
Rcl::SearchData(wasa->m_op == WasaQuery::OP_AND ? Rcl::SCLT_AND :
Rcl::SCLT_OR);
Rcl::SCLT_OR, stemlang);
LOGDEB2(("wasaQueryToRcl: %s chain\n", wasa->m_op == WasaQuery::OP_AND ?
"AND" : "OR"));
@@ -167,6 +169,7 @@ static Rcl::SearchData *wasaQueryToRcl(RclConfig *config, WasaQuery *wasa,
// "Regular" processing follows:
unsigned int mods = (unsigned int)(*it)->m_modifiers;
LOGDEB0(("wasaQueryToRcl: clause modifiers 0x%x\n", mods));
nclause = 0;
switch ((*it)->m_op) {
@@ -178,7 +181,7 @@ static Rcl::SearchData *wasaQueryToRcl(RclConfig *config, WasaQuery *wasa,
continue;
case WasaQuery::OP_LEAF: {
LOGDEB(("wasaQueryToRcl: leaf clause [%s]:[%s] slack %d\n",
LOGDEB0(("wasaQueryToRcl: leaf clause [%s]:[%s] slack %d\n",
(*it)->m_fieldspec.c_str(), (*it)->m_value.c_str(),
(*it)->m_slack));
@@ -250,7 +253,7 @@ static Rcl::SearchData *wasaQueryToRcl(RclConfig *config, WasaQuery *wasa,
(*it)->m_fieldspec.c_str(), (*it)->m_value.c_str()));
// Create a subquery.
Rcl::SearchData *sub =
wasaQueryToRcl(config, *it, autosuffs, reason);
wasaQueryToRcl(config, stemlang, *it, autosuffs, reason);
if (sub == 0) {
continue;
}
@@ -278,7 +281,7 @@ static Rcl::SearchData *wasaQueryToRcl(RclConfig *config, WasaQuery *wasa,
return sdata;
}
Rcl::SearchData *wasaStringToRcl(RclConfig *config,
Rcl::SearchData *wasaStringToRcl(RclConfig *config, const string& stemlang,
const string &qs, string &reason,
const string& autosuffs)
{
@@ -286,5 +289,5 @@ Rcl::SearchData *wasaStringToRcl(RclConfig *config,
WasaQuery *wq = parser.stringToQuery(qs, reason);
if (wq == 0)
return 0;
return wasaQueryToRcl(config, wq, autosuffs, reason);
return wasaQueryToRcl(config, stemlang, wq, autosuffs, reason);
}
src/query/wasatorcl.h
+1 -1
View File
@@ -25,7 +25,7 @@ using std::string;
class RclConfig;
extern Rcl::SearchData *wasaStringToRcl(RclConfig *,
extern Rcl::SearchData *wasaStringToRcl(RclConfig *, const string& stemlang,
const string& query, string &reason,
const string& autosuffs = string());
#endif /* _WASATORCL_H_INCLUDED_ */
src/query/xadump.cpp
+38 -7
View File
@@ -14,6 +14,9 @@
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "autoconfig.h"
#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
@@ -36,7 +39,8 @@ using namespace std;
static string thisprog;
static string usage =
" -d <dbdir> -e <output encoding>\n"
" -d <dbdir> \n"
"-e <output encoding>\n"
" -i docid -D : get document data for docid\n"
" -i docid -X : delete document docid\n"
" -i docid -b : 'rebuild' document from term positions\n"
@@ -112,6 +116,23 @@ static void sigcleanup(int sig)
exit(1);
}
#ifndef RCL_INDEX_STRIPCHARS
bool o_index_stripchars;
#endif
inline bool has_prefix(const string& trm)
{
#ifndef RCL_INDEX_STRIPCHARS
if (o_index_stripchars) {
#endif
return trm.size() && 'A' <= trm[0] && trm[0] <= 'Z';
#ifndef RCL_INDEX_STRIPCHARS
} else {
return trm.size() > 0 && trm[0] == ':';
}
#endif
}
int main(int argc, char **argv)
{
string dbdir = path_cat(path_home(), ".recoll/xapiandb");
@@ -188,10 +209,22 @@ int main(int argc, char **argv)
try {
db = new Xapian::Database(dbdir);
cout << "DB: ndocs " << db->get_doccount() << " lastdocid " <<
db->get_lastdocid() << " avglength " << db->get_avlength() << endl;
#ifndef RCL_INDEX_STRIPCHARS
// If we have terms with a leading ':' it's a new style,
// unstripped index
{
Xapian::TermIterator term = db->allterms_begin(":");
if (term == db->allterms_end())
o_index_stripchars = true;
else
o_index_stripchars = false;
cout<<"DB: terms are "<<(o_index_stripchars?"stripped":"raw")<<endl;
}
#endif
if (op_flags & OPT_T) {
Xapian::TermIterator term;
string printable;
@@ -201,8 +234,7 @@ int main(int argc, char **argv)
for (term = db->termlist_begin(docid);
term != db->termlist_end(docid);term++) {
const string& s = *term;
if ((op_flags&OPT_l) &&
!s.empty() && s[0] >= 'A' && s[0] <= 'Z')
if ((op_flags&OPT_l) && has_prefix(s))
continue;
cout << op << detailstring(s) << cl << endl;
}
@@ -210,8 +242,7 @@ int main(int argc, char **argv)
for (term = db->allterms_begin();
term != db->allterms_end();term++) {
const string& s = *term;
if ((op_flags&OPT_l) &&
!s.empty() && s[0] >= 'A' && s[0] <= 'Z')
if ((op_flags&OPT_l) && has_prefix(s))
continue;
if (op_flags & OPT_f)
cout << db->get_collection_freq(*term) << " "