tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

defined data structure to pass around the search term description used for highlighting and other

Browse Source
This commit is contained in:
"Jean-Francois Dockes ext:(%22)
2012-08-17 10:45:00 +02:00
parent ebdd6faaf5
commit dc7b3420a0
16 changed files with 409 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/query/docseq.h
+29 -43
View File
@@ -19,19 +19,16 @@
#include <string>
#include <list>
#include <vector>
#ifndef NO_NAMESPACES
using std::string;
using std::list;
using std::vector;
#endif
#include "rcldoc.h"
#include "refcntr.h"
#include "hldata.h"
// A result list entry.
struct ResListEntry {
Rcl::Doc doc;
string subHeader;
std::string subHeader;
};
/** Sort specification. */
@@ -40,7 +37,7 @@ class DocSeqSortSpec {
DocSeqSortSpec() : desc(false) {}
bool isNotNull() const {return !field.empty();}
void reset() {field.erase();}
string field;
std::string field;
bool desc;
};
@@ -50,12 +47,12 @@ class DocSeqFiltSpec {
public:
DocSeqFiltSpec() {}
enum Crit {DSFS_MIMETYPE, DSFS_QLANG, DSFS_PASSALL};
void orCrit(Crit crit, const string& value) {
void orCrit(Crit crit, const std::string& value) {
crits.push_back(crit);
values.push_back(value);
}
std::vector<Crit> crits;
std::vector<string> values;
std::vector<std::string> values;
void reset() {crits.clear(); values.clear();}
bool isNotNull() const {return crits.size() != 0;}
};
@@ -73,7 +70,7 @@ class DocSeqFiltSpec {
*/
class DocSequence {
public:
DocSequence(const string &t) : m_title(t) {}
DocSequence(const std::string &t) : m_title(t) {}
virtual ~DocSequence() {}
/** Get document at given rank.
@@ -84,16 +81,17 @@ class DocSequence {
* inside history)
* @return true if ok, false for error or end of data
*/
virtual bool getDoc(int num, Rcl::Doc &doc, string *sh = 0) = 0;
virtual bool getDoc(int num, Rcl::Doc &doc, std::string *sh = 0) = 0;
/** Get next page of documents. This accumulates entries into the result
* list parameter (doesn't reset it). */
virtual int getSeqSlice(int offs, int cnt, vector<ResListEntry>& result);
virtual int getSeqSlice(int offs, int cnt,
std::vector<ResListEntry>& result);
/** Get abstract for document. This is special because it may take time.
* The default is to return the input doc's abstract fields, but some
* sequences can compute a better value (ie: docseqdb) */
virtual bool getAbstract(Rcl::Doc& doc, vector<string>& abs) {
virtual bool getAbstract(Rcl::Doc& doc, std::vector<std::string>& abs) {
abs.push_back(doc.meta[Rcl::Doc::keyabs]);
return true;
}
@@ -103,25 +101,21 @@ class DocSequence {
virtual int getResCnt() = 0;
/** Get title for result list */
virtual string title() {return m_title;}
virtual std::string title() {return m_title;}
/** Get description for underlying query */
virtual string getDescription() = 0;
virtual std::string getDescription() = 0;
/** Get search terms (for highlighting abstracts). Some sequences
* may have no associated search terms. Implement this for them. */
virtual bool getTerms(vector<string>& terms,
vector<vector<string> >& groups,
vector<int>& gslks)
virtual void getTerms(HighlightData& hld)
{
terms.clear(); groups.clear(); gslks.clear(); return true;
hld.clear();
}
/** Get user-input terms (before stemming etc.) */
virtual void getUTerms(vector<string>& terms)
virtual std::list<std::string> expand(Rcl::Doc &)
{
terms.clear();
return std::list<std::string>();
}
virtual list<string> expand(Rcl::Doc &) {return list<string>();}
/** Optional functionality. */
virtual bool canFilter() {return false;}
@@ -130,16 +124,16 @@ class DocSequence {
virtual bool setSortSpec(const DocSeqSortSpec &) {return false;}
virtual RefCntr<DocSequence> getSourceSeq() {return RefCntr<DocSequence>();}
static void set_translations(const string& sort, const string& filt)
static void set_translations(const std::string& sort, const std::string& filt)
{
o_sort_trans = sort;
o_filt_trans = filt;
}
protected:
static string o_sort_trans;
static string o_filt_trans;
static std::string o_sort_trans;
static std::string o_filt_trans;
private:
string m_title;
std::string m_title;
};
/** A modifier has a child sequence which does the real work and does
@@ -152,25 +146,23 @@ public:
{}
virtual ~DocSeqModifier() {}
virtual bool getAbstract(Rcl::Doc& doc, vector<string>& abs)
virtual bool getAbstract(Rcl::Doc& doc, std::vector<std::string>& abs)
{
if (m_seq.isNull())
return false;
return m_seq->getAbstract(doc, abs);
}
virtual string getDescription()
virtual std::string getDescription()
{
if (m_seq.isNull())
return "";
return m_seq->getDescription();
}
virtual bool getTerms(vector<string>& terms,
vector<vector<string> >& groups,
vector<int>& gslks)
virtual void getTerms(HighlightData& hld)
{
if (m_seq.isNull())
return false;
return m_seq->getTerms(terms, groups, gslks);
return;
m_seq->getTerms(hld);
}
virtual bool getEnclosing(Rcl::Doc& doc, Rcl::Doc& pdoc)
{
@@ -178,13 +170,7 @@ public:
return false;
return m_seq->getEnclosing(doc, pdoc);
}
virtual void getUTerms(vector<string>& terms)
{
if (m_seq.isNull())
return;
m_seq->getUTerms(terms);
}
virtual string title() {return m_seq->title();}
virtual std::string title() {return m_seq->title();}
virtual RefCntr<DocSequence> getSourceSeq() {return m_seq;}
protected:
@@ -203,7 +189,7 @@ public:
virtual bool canSort() {return true;}
virtual bool setFiltSpec(const DocSeqFiltSpec &);
virtual bool setSortSpec(const DocSeqSortSpec &);
virtual bool getDoc(int num, Rcl::Doc &doc, string *sh = 0)
virtual bool getDoc(int num, Rcl::Doc &doc, std::string *sh = 0)
{
if (m_seq.isNull())
return false;
@@ -215,7 +201,7 @@ public:
return 0;
return m_seq->getResCnt();
}
virtual string title();
virtual std::string title();
private:
bool buildStack();
void stripStack();
src/query/docseqdb.cpp
+10 -9
View File
@@ -39,16 +39,9 @@ DocSequenceDb::~DocSequenceDb()
{
}
bool DocSequenceDb::getTerms(vector<string>& terms,
vector<vector<string> >& groups,
vector<int>& gslks)
void DocSequenceDb::getTerms(HighlightData& hld)
{
return m_fsdata->getTerms(terms, groups, gslks);
}
void DocSequenceDb::getUTerms(vector<string>& terms)
{
m_sdata->getUTerms(terms);
m_fsdata->getTerms(hld);
}
string DocSequenceDb::getDescription()
@@ -180,5 +173,13 @@ bool DocSequenceDb::setQuery()
return true;
m_rescnt = -1;
m_needSetQuery = !m_q->setQuery(m_fsdata);
if (0) {
HighlightData hld;
m_fsdata->getTerms(hld);
string str;
hld.toString(str);
fprintf(stderr, "DocSequenceDb::setQuery: terms: %s\n", str.c_str());
}
return !m_needSetQuery;
}
src/query/docseqdb.h
+1 -4
View File
@@ -30,10 +30,7 @@ class DocSequenceDb : public DocSequence {
virtual ~DocSequenceDb();
virtual bool getDoc(int num, Rcl::Doc &doc, string * = 0);
virtual int getResCnt();
virtual bool getTerms(vector<string>& terms,
vector<vector<string> >& groups,
vector<int>& gslks);
virtual void getUTerms(vector<string>& terms);
virtual void getTerms(HighlightData& hld);
virtual bool getAbstract(Rcl::Doc &doc, vector<string>&);
virtual bool getEnclosing(Rcl::Doc& doc, Rcl::Doc& pdoc);
virtual string getDescription();
src/query/plaintorich.cpp
+30 -45
View File
@@ -24,12 +24,10 @@
#include <map>
#include <algorithm>
#ifndef NO_NAMESPACES
using std::vector;
using std::list;
using std::pair;
using std::set;
#endif /* NO_NAMESPACES */
#include "rcldb.h"
#include "rclconfig.h"
@@ -51,28 +49,30 @@ static string vecStringToString(const vector<string>& t)
return sterms;
}
// Text splitter callback used to take note of the position of query terms
// inside the result text. This is then used to insert highlight tags.
// Text splitter used to take note of the position of query terms
// inside the result text. This is then used to insert highlight tags.
class TextSplitPTR : public TextSplit {
public:
// Out: begin and end byte positions of query terms/groups in text
vector<pair<int, int> > tboffs;
TextSplitPTR(const vector<string>& its,
const vector<vector<string> >&groups,
const vector<int>& slacks)
: m_wcount(0), m_groups(groups), m_slacks(slacks)
TextSplitPTR(const HighlightData& hdata)
: m_wcount(0), m_hdata(hdata)
{
for (vector<string>::const_iterator it = its.begin();
it != its.end(); it++) {
m_terms.insert(*it);
}
for (vector<vector<string> >::const_iterator vit = m_groups.begin();
vit != m_groups.end(); vit++) {
for (vector<string>::const_iterator it = (*vit).begin();
it != (*vit).end(); it++) {
m_gterms.insert(*it);
// We separate single terms and groups and extract the group
// terms for computing positions list before looking for group
// matches
for (vector<vector<string> >::const_iterator vit = hdata.groups.begin();
vit != hdata.groups.end(); vit++) {
if (vit->size() == 1) {
m_terms.insert(vit->front());
} else if (vit->size() > 1) {
for (vector<string>::const_iterator it = vit->begin();
it != vit->end(); it++) {
m_gterms.insert(*it);
}
}
}
}
@@ -116,15 +116,16 @@ class TextSplitPTR : public TextSplit {
private:
virtual bool matchGroup(const vector<string>& terms, int dist);
// Word count. Used to call checkCancel from time to time.
int m_wcount;
// In: user query terms
set<string> m_terms;
// In: user query groups, for near/phrase searches.
const vector<vector<string> >& m_groups;
const vector<int>& m_slacks;
set<string> m_gterms;
// m_gterms holds all the terms in m_groups, as a set for quick lookup
set<string> m_gterms;
const HighlightData& m_hdata;
// group/near terms word positions.
map<string, vector<int> > m_plists;
@@ -294,10 +295,11 @@ public:
// handle all groups as NEAR (ignore order).
bool TextSplitPTR::matchGroups()
{
vector<vector<string> >::const_iterator vit = m_groups.begin();
vector<int>::const_iterator sit = m_slacks.begin();
for (; vit != m_groups.end() && sit != m_slacks.end(); vit++, sit++) {
matchGroup(*vit, *sit + (*vit).size());
for (unsigned int i = 0; i < m_hdata.groups.size(); i++) {
if (m_hdata.groups[i].size() <= 1)
continue;
matchGroup(m_hdata.groups[i],
m_hdata.groups[i].size() + m_hdata.slacks[i]);
}
// Sort regions by increasing start and decreasing width.
@@ -317,39 +319,22 @@ bool TextSplitPTR::matchGroups()
// the input is html, the body is always a single output chunk.
bool PlainToRich::plaintorich(const string& in,
list<string>& out, // Output chunk list
const HiliteData& hdata,
const HighlightData& hdata,
int chunksize)
{
Chrono chron;
const vector<string>& terms(hdata.terms);
const vector<vector<string> >& groups(hdata.groups);
const vector<int>& slacks(hdata.gslks);
if (0 && DebugLog::getdbl()->getlevel() >= DEBDEB0) {
string sterms = vecStringToString(terms);
LOGDEB0(("plaintorich: terms: %s\n", sterms.c_str()));
sterms.clear();
for (vector<vector<string> >::const_iterator vit = groups.begin();
vit != groups.end(); vit++) {
sterms += "GROUP: ";
sterms += vecStringToString(*vit);
sterms += "\n";
}
LOGDEB0(("plaintorich: groups:\n %s", sterms.c_str()));
LOGDEB2((" TEXT:[%s]\n", in.c_str()));
}
// Compute the positions for the query terms. We use the text
// splitter to break the text into words, and compare the words to
// the search terms,
TextSplitPTR splitter(terms, groups, slacks);
TextSplitPTR splitter(hdata);
// Note: the splitter returns the term locations in byte, not
// character, offsets.
splitter.text_to_words(in);
LOGDEB2(("plaintorich: split done %d mS\n", chron.millis()));
// Compute the positions for NEAR and PHRASE groups.
splitter.matchGroups();
LOGDEB2(("plaintorich: group match done %d mS\n", chron.millis()));
out.clear();
out.push_back("");
src/query/plaintorich.h
+10 -28
View File
@@ -19,26 +19,8 @@
#include <string>
#include <list>
using std::list;
using std::string;
/// Holder for plaintorich() input data: words and groups of words to
/// be highlighted
struct HiliteData {
// Single terms
vector<string> terms;
// NEAR and PHRASE elements
vector<vector<string> > groups;
// Group slacks (number of permitted non-matched words).
// Parallel vector to the above 'groups'
vector<int> gslks;
void reset()
{
terms.clear();
groups.clear();
gslks.clear();
}
};
#include "hldata.h"
/**
* A class for highlighting search results. Overridable methods allow
@@ -83,21 +65,21 @@ public:
* lowercase and unaccented.
* @param chunksize max size of chunks in output list
*/
virtual bool plaintorich(const string &in, list<string> &out,
const HiliteData& hdata,
virtual bool plaintorich(const std::string &in, std::list<std::string> &out,
const HighlightData& hdata,
int chunksize = 50000
);
/* Overridable output methods for headers, highlighting and marking tags */
virtual string header() {return snull;}
virtual string startMatch() {return snull;}
virtual string endMatch() {return snull;}
virtual string startAnchor(int) {return snull;}
virtual string endAnchor() {return snull;}
virtual string startChunk() {return snull;}
virtual std::string header() {return snull;}
virtual std::string startMatch() {return snull;}
virtual std::string endMatch() {return snull;}
virtual std::string startAnchor(int) {return snull;}
virtual std::string endAnchor() {return snull;}
virtual std::string startChunk() {return snull;}
protected:
const string snull;
const std::string snull;
bool m_inputhtml;
// Use <br> to break plain text lines (else caller has used a <pre> tag)
bool m_eolbr;
src/query/reslistpager.cpp
+7 -7
View File
@@ -132,9 +132,8 @@ void ResListPager::resultPageFor(int docnum)
m_respage = npage;
}
void ResListPager::displayDoc(RclConfig *config,
int i, Rcl::Doc& doc, const HiliteData& hdata,
const string& sh)
void ResListPager::displayDoc(RclConfig *config, int i, Rcl::Doc& doc,
const HighlightData& hdata, const string& sh)
{
ostringstream chunk;
int percent;
@@ -309,8 +308,9 @@ void ResListPager::displayPage(RclConfig *config)
if (pageEmpty()) {
chunk << trans("<p><b>No results found</b><br>");
vector<string>uterms;
m_docSource->getUTerms(uterms);
HighlightData hldata;
m_docSource->getTerms(hldata);
vector<string> uterms(hldata.uterms.begin(), hldata.uterms.end());
if (!uterms.empty()) {
map<string, vector<string> > spellings;
suggest(uterms, spellings);
@@ -366,8 +366,8 @@ void ResListPager::displayPage(RclConfig *config)
if (pageEmpty())
return;
HiliteData hdata;
m_docSource->getTerms(hdata.terms, hdata.groups, hdata.gslks);
HighlightData hdata;
m_docSource->getTerms(hdata);
// Emit data for result entry paragraph. Do it in chunks that make sense
// html-wise, else our client may get confused
src/query/reslistpager.h
+6 -6
View File
@@ -19,14 +19,13 @@
#define _reslistpager_h_included_
#include <vector>
using std::vector;
#include "refcntr.h"
#include "docseq.h"
#include "hldata.h"
class RclConfig;
class PlainToRich;
struct HiliteData;
/**
* Manage a paged HTML result list.
@@ -85,7 +84,7 @@ public:
void resultPageFor(int docnum);
void displayPage(RclConfig *);
void displayDoc(RclConfig *, int idx, Rcl::Doc& doc,
const HiliteData& hdata, const string& sh = "");
const HighlightData& hdata, const string& sh = "");
bool pageEmpty() {return m_respage.size() == 0;}
string queryDescription() {return m_docSource.isNull() ? "" :
@@ -112,8 +111,9 @@ public:
virtual string pageTop() {return string();}
virtual string headerContent() {return string();}
virtual string iconUrl(RclConfig *, Rcl::Doc& doc);
virtual void suggest(const vector<string>,
map<string, vector<string> >& sugg) {
virtual void suggest(const std::vector<std::string>,
std::map<std::string, std::vector<std::string> >& sugg)
{
sugg.clear();
}
virtual string absSep() {return "&hellip;";}
@@ -126,7 +126,7 @@ private:
bool m_hasNext;
PlainToRich *m_hiliter;
RefCntr<DocSequence> m_docSource;
vector<ResListEntry> m_respage;
std::vector<ResListEntry> m_respage;
};
#endif /* _reslistpager_h_included_ */