tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

implemented multi-word terms indexing for phrase/prox search on multiword synonyms

Browse Source
This commit is contained in:
Jean-Francois Dockes
2021-01-15 14:13:08 +01:00
parent cb13b8b6df
commit aa2f0bfd73
10 changed files with 296 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/rclconfig.cpp
+2 -2
View File
@@ -1374,9 +1374,9 @@ string RclConfig::getStopfile() const
return getConfdirPath("stoplistfile", "stoplist.txt");
}
string RclConfig::getSynGroupsFile() const
string RclConfig::getIdxSynGroupsFile() const
{
return getConfdirPath("syngroupsfile", "syngroups.txt");
return getConfdirPath("idxsynonyms", "thereisnodefaultidxsynonyms");
}
// The index status file is fast changing, so it's possible to put it outside
src/common/rclconfig.h
+32 -38
View File
@@ -72,8 +72,8 @@ private:
// Hold the description for an external metadata-gathering command
struct MDReaper {
string fieldname;
vector<string> cmdv;
string fieldname;
vector<string> cmdv;
};
// Data associated to a indexed field name:
@@ -90,7 +90,7 @@ struct FieldTraits {
};
class RclConfig {
public:
public:
// Constructor: we normally look for a configuration file, except
// if this was specified on the command line and passed through
@@ -100,7 +100,7 @@ class RclConfig {
RclConfig(const RclConfig &r);
~RclConfig() {
freeAll();
freeAll();
}
// Return a writable clone of the main config. This belongs to the
@@ -133,18 +133,16 @@ class RclConfig {
string getKeyDir() const {return m_keydir;}
/** Get generic configuration parameter according to current keydir */
bool getConfParam(const string &name, string &value,
bool shallow=false) const
{
if (m_conf == 0)
return false;
return m_conf->get(name, value, m_keydir, shallow);
bool getConfParam(const string& name, string& value,
bool shallow=false) const {
if (m_conf == 0)
return false;
return m_conf->get(name, value, m_keydir, shallow);
}
/** Variant with autoconversion to int */
bool getConfParam(const string &name, int *value, bool shallow=false) const;
/** Variant with autoconversion to bool */
bool getConfParam(const string &name, bool *value,
bool shallow=false) const;
bool getConfParam(const string &name, bool *value, bool shallow=false) const;
/** Variant with conversion to vector<string>
* (stringToStrings). Can fail if the string is malformed. */
bool getConfParam(const string &name, vector<string> *value,
@@ -164,18 +162,15 @@ class RclConfig {
* Get list of config names under current sk, with possible
* wildcard filtering
*/
vector<string> getConfNames(const char *pattern = 0) const
{
return m_conf->getNames(m_keydir, pattern);
vector<string> getConfNames(const char *pattern = 0) const {
return m_conf->getNames(m_keydir, pattern);
}
/** Check if name exists anywhere in config */
bool hasNameAnywhere(const string& nm) const
{
bool hasNameAnywhere(const string& nm) const {
return m_conf? m_conf->hasNameAnywhere(nm) : false;
}
/** Get default charset for current keydir (was set during setKeydir)
* filenames are handled differently */
const string &getDefCharset(bool filename = false) const;
@@ -198,7 +193,7 @@ class RclConfig {
/** Get stoplist file name */
string getStopfile() const;
/** Get synonym groups file name */
string getSynGroupsFile() const;
string getIdxSynGroupsFile() const;
/** Get indexing pid file name */
string getPidfile() const;
/** Get indexing status file name */
@@ -207,7 +202,7 @@ class RclConfig {
/** Do path translation according to the ptrans table */
void urlrewrite(const string& dbdir, string& url) const;
ConfSimple *getPTrans() {
return m_ptrans;
return m_ptrans;
}
/** Get Web Queue directory name */
string getWebQueueDir() const;
@@ -215,13 +210,13 @@ class RclConfig {
/** Get list of skipped file names for current keydir */
vector<string>& getSkippedNames();
/** Get list of file name filters for current keydir (only those
names indexed) */
names indexed) */
vector<string>& getOnlyNames();
/** Get list of skipped paths patterns. Doesn't depend on the keydir */
vector<string> getSkippedPaths() const;
/** Get list of skipped paths patterns, daemon version (may add some)
Doesn't depend on the keydir */
Doesn't depend on the keydir */
vector<string> getDaemSkippedPaths() const;
/** Return list of no content suffixes. Used by confgui, indexing uses
@@ -260,7 +255,7 @@ class RclConfig {
* @param whole the raw value. No way to escape a semi-colon in there.
*/
static bool valueSplitAttributes(const string& whole, string& value,
ConfSimple& attrs) ;
ConfSimple& attrs) ;
/** Compute difference between 'base' and 'changed', as elements to be
* added and substracted from base. Input and output strings are in
@@ -288,9 +283,9 @@ class RclConfig {
bool getGuiFilter(const string& filtername, string& frag) const;
/** fields: get field prefix from field name. Use additional query
aliases if isquery is set */
aliases if isquery is set */
bool getFieldTraits(const string& fldname, const FieldTraits **,
bool isquery = false) const;
bool isquery = false) const;
const set<string>& getStoredFields() const {return m_storedFields;}
@@ -311,11 +306,11 @@ class RclConfig {
*/
vector<string> getFieldSectNames(const string &sk, const char* = 0) const;
bool getFieldConfParam(const string &name, const string &sk, string &value)
const;
const;
/** mimeview: get/set external viewer exec string(s) for mimetype(s) */
string getMimeViewerDef(const string &mimetype, const string& apptag,
bool useall) const;
bool useall) const;
set<string> getMimeViewerAllEx() const;
bool setMimeViewerAllEx(const set<string>& allex);
bool getMimeViewerDefs(vector<pair<string, string> >&) const;
@@ -358,26 +353,25 @@ class RclConfig {
string findFilter(const string& cmd) const;
/** Thread config init is not done automatically because not all
programs need it and it uses the debug log so that it's better to
call it after primary init */
programs need it and it uses the debug log so that it's better to
call it after primary init */
void initThrConf();
const string& getOrigCwd()
{
return o_origcwd;
const string& getOrigCwd() {
return o_origcwd;
}
RclConfig& operator=(const RclConfig &r) {
if (this != &r) {
freeAll();
initFrom(r);
}
return *this;
if (this != &r) {
freeAll();
initFrom(r);
}
return *this;
}
friend class ParamStale;
private:
private:
int m_ok;
string m_reason; // Explanation for bad state
string m_confdir; // User directory where the customized files are stored
src/common/syngroups.cpp
+99 -56
View File
@@ -44,8 +44,7 @@ using namespace std;
// groups anyway
class SynGroups::Internal {
public:
Internal() : ok(false) {
}
Internal() {}
void setpath(const string& fn) {
path = path_canon(fn);
stat(path.c_str(), &st);
@@ -61,16 +60,22 @@ public:
}
return st.st_mtime == st1.st_mtime && st.st_size == st1.st_size;
}
bool ok;
bool ok{false};
// Term to group num
std::unordered_map<string, unsigned int> terms;
// Group num to group
vector<vector<string> > groups;
// Aux: set of multiword synonyms used for generating multiword
// terms while indexing
std::set<std::string> multiwords;
size_t multiwords_maxlen{0};
std::string path;
struct stat st;
};
bool SynGroups::ok()
bool SynGroups::ok() const
{
return m && m->ok;
}
@@ -99,7 +104,7 @@ bool SynGroups::setfile(const string& fn)
if (fn.empty()) {
delete m;
m = 0;
return true;
return true;
}
if (m->samefile(fn)) {
@@ -111,8 +116,8 @@ bool SynGroups::setfile(const string& fn)
ifstream input;
input.open(fn.c_str(), ios::in);
if (!input.is_open()) {
LOGSYSERR("SynGroups:setfile", "open", fn);
return false;
LOGSYSERR("SynGroups:setfile", "open", fn);
return false;
}
string cline;
@@ -120,21 +125,24 @@ bool SynGroups::setfile(const string& fn)
string line;
bool eof = false;
int lnum = 0;
m->groups.clear();
m->terms.clear();
m->multiwords.clear();
m->multiwords_maxlen = 0;
for (;;) {
cline.clear();
getline(input, cline);
if (!input.good()) {
if (input.bad()) {
getline(input, cline);
if (!input.good()) {
if (input.bad()) {
LOGERR("Syngroup::setfile(" << fn << "):Parse: input.bad()\n");
return false;
}
// Must be eof ? But maybe we have a partial line which
// must be processed. This happens if the last line before
// eof ends with a backslash, or there is no final \n
return false;
}
// Must be eof ? But maybe we have a partial line which
// must be processed. This happens if the last line before
// eof ends with a backslash, or there is no final \n
eof = true;
}
lnum++;
}
lnum++;
{
string::size_type pos = cline.find_last_not_of("\n\r");
@@ -145,65 +153,85 @@ bool SynGroups::setfile(const string& fn)
}
}
if (appending)
line += cline;
else
line = cline;
if (appending)
line += cline;
else
line = cline;
// Note that we trim whitespace before checking for backslash-eol
// This avoids invisible whitespace problems.
trimstring(line);
if (line.empty() || line.at(0) == '#') {
// Note that we trim whitespace before checking for backslash-eol
// This avoids invisible whitespace problems.
trimstring(line);
if (line.empty() || line.at(0) == '#') {
if (eof)
break;
continue;
}
if (line[line.length() - 1] == '\\') {
line.erase(line.length() - 1);
appending = true;
continue;
}
appending = false;
continue;
}
if (line[line.length() - 1] == '\\') {
line.erase(line.length() - 1);
appending = true;
continue;
}
appending = false;
vector<string> words;
if (!stringToStrings(line, words)) {
LOGERR("SynGroups:setfile: " << fn << ": bad line " << lnum <<
vector<string> words;
if (!stringToStrings(line, words)) {
LOGERR("SynGroups:setfile: " << fn << ": bad line " << lnum <<
": " << line << "\n");
continue;
}
continue;
}
if (words.empty())
continue;
if (words.size() == 1) {
LOGERR("Syngroup::setfile(" << fn << "):single term group at line "
if (words.empty())
continue;
if (words.size() == 1) {
LOGERR("Syngroup::setfile(" << fn << "):single term group at line "
<< lnum << " ??\n");
continue;
}
continue;
}
m->groups.push_back(words);
for (const auto& word : words) {
m->terms[word] = m->groups.size()-1;
}
LOGDEB1("SynGroups::setfile: group: [" <<
m->groups.push_back(words);
for (const auto& word : words) {
m->terms[word] = m->groups.size()-1;
}
LOGDEB1("SynGroups::setfile: group: [" <<
stringsToString(m->groups.back()) << "]\n");
}
LOGDEB("SynGroups::setfile: got " << m->groups.size() <<
" distinct terms." << endl);
for (const auto& group : m->groups) {
for (const auto& term : group) {
std::vector<std::string> words;
stringToTokens(term, words);
if (words.size() > 1) {
std::string multiword;
for (const auto& word : words) {
if (!multiword.empty()) {
multiword += " ";
}
multiword += word;
}
m->multiwords.insert(multiword);
if (m->multiwords_maxlen < words.size()) {
m->multiwords_maxlen = words.size();
}
}
}
}
LOGDEB("SynGroups::setfile: got " << m->groups.size() << " distinct terms. "
"Multiwords: " << stringsToString(m->multiwords) <<"\n");
m->ok = true;
m->setpath(fn);
return true;
}
vector<string> SynGroups::getgroup(const string& term)
vector<string> SynGroups::getgroup(const string& term) const
{
vector<string> ret;
if (!ok())
return ret;
return ret;
const auto it1 = m->terms.find(term);
if (it1 == m->terms.end()) {
LOGDEB0("SynGroups::getgroup: [" << term << "] not found in map\n");
return ret;
LOGDEB0("SynGroups::getgroup: [" << term << "] not found in map\n");
return ret;
}
unsigned int idx = it1->second;
@@ -215,3 +243,18 @@ vector<string> SynGroups::getgroup(const string& term)
<< endl);
return m->groups[idx];
}
const std::set<std::string>& SynGroups::getmultiwords() const
{
return m->multiwords;
}
size_t SynGroups::getmultiwordsmaxlength() const
{
return m->multiwords_maxlen;
}
const std::string& SynGroups::getpath() const
{
return m->path;
}
src/common/syngroups.h
+7 -3
View File
@@ -1,4 +1,4 @@
/* Copyright (C) 2015 J.F.Dockes
/* Copyright (C) 2015-2021 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
@@ -20,6 +20,7 @@
#include <string>
#include <vector>
#include <set>
// Manage synonym groups. This is very different from stemming and
// case/diac expansion because there is no reference form: all terms
@@ -34,8 +35,11 @@ public:
SynGroups& operator=(const SynGroups&&) = delete;
bool setfile(const std::string& fname);
std::vector<std::string> getgroup(const std::string& term);
bool ok();
std::vector<std::string> getgroup(const std::string& term) const;
const std::set<std::string>& getmultiwords() const;
size_t getmultiwordsmaxlength() const;
const std::string& getpath() const;
bool ok() const;
private:
class Internal;
Internal *m;