tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Highlighting and snippets extraction: reworked to handle phrases properly. Use a compound position list instead of multiplying the OR groups inside a near clause

Browse Source
This commit is contained in:
Jean-Francois Dockes
2019-07-21 19:09:51 +02:00
parent baa6062de1
commit 35ee3f7a13
7 changed files with 211 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils/hldata.cpp
+146 -77
View File
@@ -31,11 +31,69 @@ using std::pair;
#undef DEBUGGROUPS
#ifdef DEBUGGROUPS
#define LOGRP LOGDEB
#define LOGRP LOGINF
#else
#define LOGRP LOGDEB1
#endif
// Combined position list for or'd terms
struct OrPList {
void addplist(const string& term, const vector<int>* pl) {
terms.push_back(term);
plists.push_back(pl);
indexes.push_back(0);
totalsize += pl->size();
}
// Returns -1 for eof, else the next smallest value in the
// combined lists, according to the current indexes.
int value() {
int minval = INT_MAX;
int minidx = -1;
for (unsigned ii = 0; ii < indexes.size(); ii++) {
const vector<int>& pl(*plists[ii]);
if (indexes[ii] >= pl.size())
continue; // this list done
if (pl[indexes[ii]] < minval) {
minval = pl[indexes[ii]];
minidx = ii;
}
}
if (minidx != -1) {
LOGRP("OrPList::value() -> " << minval << " for " <<
terms[minidx] << "\n");
currentidx = minidx;
return minval;
} else {
LOGRP("OrPList::value(): EOL for " << stringsToString(terms)<<"\n");
return -1;
}
}
int next() {
if (currentidx != -1) {
indexes[currentidx]++;
}
return value();
}
int size() const {
return totalsize;
}
void rewind() {
for (auto& idx : indexes) {
idx = 0;
}
currentidx = -1;
}
vector<const vector<int>*> plists;
vector<unsigned int> indexes;
vector<string> terms;
int currentidx{-1};
int totalsize{0};
};
static inline void setWinMinMax(int pos, int& sta, int& sto)
{
if (pos < sta) {
@@ -65,42 +123,44 @@ static inline void setWinMinMax(int pos, int& sta, int& sto)
* we only look for the next match beyond the current window top.
*/
static bool do_proximity_test(
const int window, vector<const vector<int>*>& plists,
const int window, vector<OrPList>& plists,
unsigned int plist_idx, int min, int max, int *sp, int *ep, int minpos,
bool isphrase)
{
LOGINF("do_prox_test: win " << window << " plist_idx " << plist_idx <<
" min " << min << " max " << max << " minpos " << minpos <<
" isphrase " << isphrase << "\n");
// Overlap interdiction: possibly adjust window start by input minpos
int actualminpos = isphrase ? max + 1 : max + 1 - window;
if (actualminpos < minpos)
actualminpos = minpos;
LOGRP("do_prox_test: win " << window << " plist_idx " << plist_idx <<
" min " << min << " max " << max << " minpos " << minpos <<
" isphrase " << isphrase << " actualminpos " << actualminpos << "\n");
// Find 1st position bigger than window start
auto it = plists[plist_idx]->begin();
while (it != plists[plist_idx]->end() && *it < actualminpos)
it++;
// Find 1st position bigger than window start. A previous call may
// have advanced the index, so we begin by retrieving the current
// value.
int nextpos = plists[plist_idx].value();
while (nextpos != -1 && nextpos < actualminpos)
nextpos = plists[plist_idx].next();
// Look for position inside window. If not found, no match. If
// found: if this is the last list we're done, else recurse on
// next list after adjusting the window
while (it != plists[plist_idx]->end()) {
int pos = *it;
if (pos > min + window - 1)
while (nextpos != -1) {
if (nextpos > min + window - 1) {
return false;
}
if (plist_idx + 1 == plists.size()) {
// Done: set return values
setWinMinMax(pos, *sp, *ep);
// We already checked pos > min, now we also have pos <
// max, and we are the last list: done: set return values.
setWinMinMax(nextpos, *sp, *ep);
return true;
}
setWinMinMax(pos, min, max);
if (do_proximity_test(window,plists, plist_idx + 1,
min, max, sp, ep, minpos)) {
setWinMinMax(nextpos, min, max);
if (do_proximity_test(window, plists, plist_idx + 1,
min, max, sp, ep, minpos, isphrase)) {
return true;
}
it++;
nextpos = plists[plist_idx].next();
}
return false;
}
@@ -111,63 +171,62 @@ bool matchGroup(const HighlightData& hldata,
unsigned int grpidx,
const map<string, vector<int>>& inplists,
const map<int, pair<int,int>>& gpostobytes,
vector<GroupMatchEntry>& tboffs,
bool isphrase
)
vector<GroupMatchEntry>& tboffs)
{
isphrase=true;
const vector<string>& terms = hldata.index_term_groups[grpidx];
int window = int(terms.size() + hldata.slacks[grpidx]);
LOGRP("TextSplitPTR::matchGroup:d " << window << ": " <<
stringsToString(terms) << "\n");
const auto& tg(hldata.index_term_groups[grpidx]);
bool isphrase = tg.kind == HighlightData::TermGroup::TGK_PHRASE;
string allplterms;
for (const auto& entry:inplists) {
allplterms += entry.first + " ";
}
LOGRP("matchGroup: isphrase " << isphrase <<
". Have plists for [" << allplterms << "]\n");
LOGRP("matchGroup: hldata: " << hldata.toString() << std::endl);
int window = int(tg.orgroups.size() + tg.slack);
// The position lists we are going to work with. We extract them from the
// (string->plist) map
vector<const vector<int>*> plists;
// A revert plist->term map. This is so that we can find who is who after
// sorting the plists by length.
map<const vector<int>*, string> plistToTerm;
vector<OrPList> orplists;
// Find the position list for each term in the group. It is
// possible that this particular group was not actually matched by
// the search, so that some terms are not found.
for (const auto& term : terms) {
map<string, vector<int> >::const_iterator pl = inplists.find(term);
if (pl == inplists.end()) {
LOGRP("TextSplitPTR::matchGroup: [" << term <<
"] not found in plists\n");
return false;
// Find the position list for each term in the group and build the
// combined lists for the term or groups (each group is the result
// of the exansion of one user term). It is possible that this
// particular group was not actually matched by the search, so
// that some terms are not found, in which case we bail out.
for (const auto& group : tg.orgroups) {
orplists.push_back(OrPList());
for (const auto& term : group) {
const auto pl = inplists.find(term);
if (pl == inplists.end()) {
LOGRP("TextSplitPTR::matchGroup: term [" << term <<
"] not found in plists\n");
continue;
}
orplists.back().addplist(pl->first, &(pl->second));
}
if (orplists.back().plists.empty()) {
LOGINF("No positions list found for group " <<
stringsToString(group) << std::endl);
orplists.pop_back();
}
plists.push_back(&(pl->second));
plistToTerm[&(pl->second)] = term;
}
// I think this can't actually happen, was useful when we used to
// prune the groups, but doesn't hurt.
if (plists.size() < 2) {
LOGRP("TextSplitPTR::matchGroup: no actual groups found\n");
if (orplists.size() < 2) {
LOGINF("TextSplitPTR::matchGroup: no actual groups found\n");
return false;
}
if (!isphrase) {
// Sort the positions lists so that the shorter is first
std::sort(plists.begin(), plists.end(),
[](const vector<int> *a, const vector<int> *b) -> bool {
return a->size() < b->size();
std::sort(orplists.begin(), orplists.end(),
[](const OrPList& a, const OrPList& b) -> bool {
return a.size() < b.size();
}
);
}
if (0) { // Debug
auto it = plistToTerm.find(plists[0]);
if (it == plistToTerm.end()) {
// SuperWeird
LOGERR("matchGroup: term for first list not found !?!\n");
return false;
}
LOGRP("matchGroup: walking the shortest plist. Term [" <<
it->second << "], len " << plists[0]->size() << "\n");
}
// Minpos is the highest end of a found match. While looking for
// further matches, we don't want the search to extend before
@@ -175,11 +234,12 @@ bool matchGroup(const HighlightData& hldata,
// overlap
int minpos = 0;
// Walk the shortest plist and look for matches
for (int pos : *(plists[0])) {
int pos;
while ((pos = orplists[0].next()) != -1) {
int sta = INT_MAX, sto = 0;
LOGDEB2("MatchGroup: Testing at pos " << pos << "\n");
if (do_proximity_test(
window, plists, 1, pos, pos, &sta, &sto, minpos, isphrase)) {
window, orplists, 1, pos, pos, &sta, &sto, minpos, isphrase)) {
setWinMinMax(pos, sta, sto);
LOGINF("TextSplitPTR::matchGroup: MATCH termpos [" << sta <<
"," << sto << "]\n");
@@ -204,8 +264,9 @@ bool matchGroup(const HighlightData& hldata,
return true;
}
void HighlightData::toString(string& out) const
string HighlightData::toString() const
{
string out;
out.append("\nUser terms (orthograph): ");
for (const auto& term : uterms) {
out.append(" [").append(term).append("]");
@@ -217,29 +278,37 @@ void HighlightData::toString(string& out) const
}
out.append("\nGroups: ");
char cbuf[200];
sprintf(cbuf, "Groups size %d grpsugidx size %d ugroups size %d",
int(index_term_groups.size()), int(grpsugidx.size()),
int(ugroups.size()));
sprintf(cbuf, "index_term_groups size %d ugroups size %d",
int(index_term_groups.size()), int(ugroups.size()));
out.append(cbuf);
size_t ugidx = (size_t) - 1;
for (unsigned int i = 0; i < index_term_groups.size(); i++) {
if (ugidx != grpsugidx[i]) {
ugidx = grpsugidx[i];
for (HighlightData::TermGroup tg : index_term_groups) {
if (ugidx != tg.grpsugidx) {
ugidx = tg.grpsugidx;
out.append("\n(");
for (unsigned int j = 0; j < ugroups[ugidx].size(); j++) {
out.append("[").append(ugroups[ugidx][j]).append("] ");
}
out.append(") ->");
}
out.append(" {");
for (unsigned int j = 0; j < index_term_groups[i].size(); j++) {
out.append("[").append(index_term_groups[i][j]).append("]");
if (tg.kind == HighlightData::TermGroup::TGK_TERM) {
out.append(" <").append(tg.term).append(">");
} else {
out.append(" {");
for (unsigned int j = 0; j < tg.orgroups.size(); j++) {
out.append(" {");
for (unsigned int k = 0; k < tg.orgroups[j].size(); k++) {
out.append("[").append(tg.orgroups[j][k]).append("]");
}
out.append("}");
}
sprintf(cbuf, "%d", tg.slack);
out.append("}").append(cbuf);
}
sprintf(cbuf, "%d", slacks[i]);
out.append("}").append(cbuf);
}
out.append("\n");
return out;
}
void HighlightData::append(const HighlightData& hl)
@@ -249,12 +318,12 @@ void HighlightData::append(const HighlightData& hl)
size_t ugsz0 = ugroups.size();
ugroups.insert(ugroups.end(), hl.ugroups.begin(), hl.ugroups.end());
size_t itgsize = index_term_groups.size();
index_term_groups.insert(index_term_groups.end(),
hl.index_term_groups.begin(),
hl.index_term_groups.end());
slacks.insert(slacks.end(), hl.slacks.begin(), hl.slacks.end());
for (std::vector<size_t>::const_iterator it = hl.grpsugidx.begin();
it != hl.grpsugidx.end(); it++) {
grpsugidx.push_back(*it + ugsz0);
// Adjust the grpsugidx values for the newly inserted entries
for (unsigned int idx = itgsize; idx < index_term_groups.size(); idx++) {
index_term_groups[idx].grpsugidx += ugsz0;
}
}
src/utils/hldata.h
+21 -22
View File
@@ -5,6 +5,7 @@
#include <string>
#include <set>
#include <map>
#include <unordered_map>
/** Store data about user search terms and their expansions. This is used
* mostly for highlighting result text and walking the matches, generating
@@ -22,7 +23,7 @@ struct HighlightData {
* This is used for aggregating term stats when generating snippets (for
* choosing the best terms, allocating slots, etc. )
*/
std::map<std::string, std::string> terms;
std::unordered_map<std::string, std::string> terms;
/** The original user terms-or-groups. This is for display
* purposes: ie when creating a menu to look for a specific
@@ -33,40 +34,39 @@ struct HighlightData {
std::vector<std::vector<std::string> > ugroups;
/** Processed/expanded terms and groups. Used for looking for
* regions to highlight. A group can be a PHRASE or NEAR entry (we
* process everything as NEAR to keep things reasonably
* simple. Terms are just groups with 1 entry. All
* regions to highlight. A group can be a PHRASE or NEAR entry
* Terms are just groups with 1 entry. All
* terms are transformed to be compatible with index content
* (unaccented and lowercased as needed depending on
* configuration), and the list may include values
* expanded from the original terms by stem or wildcard expansion.
* NEAR clauses are expanded to all possible combinations of the
* stem-expanded member terms. Ex:
* "clean floor"p -> (clean floor) (clean floors) (cleaning floor)...
*/
std::vector<std::vector<std::string> > index_term_groups;
/** Group slacks. Parallel to groups */
std::vector<int> slacks;
struct TermGroup {
// We'd use an union but no can do
std::string term;
std::vector<std::vector<std::string> > orgroups;
int slack{0};
/** Index into ugroups for each group. Parallel to groups. As a
* user term or group may generate many processed/expanded terms
* or groups, this is how we relate an expansion to its source
* (used, e.g. for generating anchors for walking search matches
* in the preview window).
*/
std::vector<size_t> grpsugidx;
/* Index into ugroups. As a user term or group may generate
* many processed/expanded terms or groups, this is how we
* relate an expansion to its source (used, e.g. for
* generating anchors for walking search matches in the
* preview window). */
size_t grpsugidx{0};
enum TGK {TGK_TERM, TGK_NEAR, TGK_PHRASE};
TGK kind{TGK_TERM};
};
std::vector<TermGroup> index_term_groups;
void clear() {
uterms.clear();
ugroups.clear();
index_term_groups.clear();
slacks.clear();
grpsugidx.clear();
}
void append(const HighlightData&);
// Print (debug)
void toString(std::string& out) const;
std::string toString() const;
};
/* The following is used by plaintorich.cpp for finding zones to
@@ -109,8 +109,7 @@ extern bool matchGroup(
unsigned int grpidx,
const std::map<std::string, std::vector<int>>& inplists,
const std::map<int, std::pair<int,int>>& gpostobytes,
std::vector<GroupMatchEntry>& tboffs,
bool isphrase = false
std::vector<GroupMatchEntry>& tboffs
);
#endif /* _hldata_h_included_ */