tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

tabs

Browse Source
This commit is contained in:
Jean-Francois Dockes 2020-01-31 09:28:06 +01:00
parent 4320b3225c
commit da05b762a7
1 changed files with 169 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

338
src/query/recollq.cpp
View File

@ -49,15 +49,15 @@ bool dump_contents(RclConfig *rclconfig, Rcl::Doc& idoc)
Rcl::Doc fdoc; Rcl::Doc fdoc;
string ipath = idoc.ipath; string ipath = idoc.ipath;
if (interner.internfile(fdoc, ipath)) { if (interner.internfile(fdoc, ipath)) {
cout << fdoc.text << endl; cout << fdoc.text << endl;
} else { } else {
cout << "Cant turn to text:" << idoc.url << " | " << idoc.ipath << endl; cout << "Cant turn to text:" << idoc.url << " | " << idoc.ipath << endl;
} }
return true; return true;
} }
void output_fields(vector<string> fields, Rcl::Doc& doc, void output_fields(vector<string> fields, Rcl::Doc& doc,
Rcl::Query& query, Rcl::Db& rcldb, bool printnames) Rcl::Query& query, Rcl::Db& rcldb, bool printnames)
{ {
if (fields.empty()) { if (fields.empty()) {
map<string,string>::const_iterator it; map<string,string>::const_iterator it;
@ -66,19 +66,19 @@ void output_fields(vector<string> fields, Rcl::Doc& doc,
} }
} }
for (vector<string>::const_iterator it = fields.begin(); for (vector<string>::const_iterator it = fields.begin();
it != fields.end(); it++) { it != fields.end(); it++) {
string out; string out;
if (!it->compare("abstract")) { if (!it->compare("abstract")) {
string abstract; string abstract;
query.makeDocAbstract(doc, abstract); query.makeDocAbstract(doc, abstract);
base64_encode(abstract, out); base64_encode(abstract, out);
} else if (!it->compare("xdocid")) { } else if (!it->compare("xdocid")) {
char cdocid[30]; char cdocid[30];
sprintf(cdocid, "%lu", (unsigned long)doc.xdocid); sprintf(cdocid, "%lu", (unsigned long)doc.xdocid);
base64_encode(cdocid, out); base64_encode(cdocid, out);
} else { } else {
base64_encode(doc.meta[*it], out); base64_encode(doc.meta[*it], out);
} }
// Before printnames existed, recollq printed a single blank for empty // Before printnames existed, recollq printed a single blank for empty
// fields. This is a problem when printing names and using strtok, but // fields. This is a problem when printing names and using strtok, but
// have to keep the old behaviour when printnames is not set. // have to keep the old behaviour when printnames is not set.
@ -93,45 +93,45 @@ void output_fields(vector<string> fields, Rcl::Doc& doc,
static char *thisprog; static char *thisprog;
static char usage [] = static char usage [] =
" -P: Show the date span for all the documents present in the index.\n" " -P: Show the date span for all the documents present in the index.\n"
" [-o|-a|-f] [-q] <query string>\n" " [-o|-a|-f] [-q] <query string>\n"
" Runs a recoll query and displays result lines. \n" " Runs a recoll query and displays result lines. \n"
" Default: will interpret the argument(s) as a xesam query string.\n" " Default: will interpret the argument(s) as a xesam query string.\n"
" Query elements: \n" " Query elements: \n"
" * Implicit AND, exclusion, field spec: t1 -t2 title:t3\n" " * Implicit AND, exclusion, field spec: t1 -t2 title:t3\n"
" * OR has priority: t1 OR t2 t3 OR t4 means (t1 OR t2) AND (t3 OR t4)\n" " * OR has priority: t1 OR t2 t3 OR t4 means (t1 OR t2) AND (t3 OR t4)\n"
" * Phrase: \"t1 t2\" (needs additional quoting on cmd line)\n" " * Phrase: \"t1 t2\" (needs additional quoting on cmd line)\n"
" -o Emulate the GUI simple search in ANY TERM mode.\n" " -o Emulate the GUI simple search in ANY TERM mode.\n"
" -a Emulate the GUI simple search in ALL TERMS mode.\n" " -a Emulate the GUI simple search in ALL TERMS mode.\n"
" -f Emulate the GUI simple search in filename mode.\n" " -f Emulate the GUI simple search in filename mode.\n"
" -q is just ignored (compatibility with the recoll GUI command line).\n" " -q is just ignored (compatibility with the recoll GUI command line).\n"
"Common options:\n" "Common options:\n"
" -c <configdir> : specify config directory, overriding $RECOLL_CONFDIR.\n" " -c <configdir> : specify config directory, overriding $RECOLL_CONFDIR.\n"
" -C : collapse duplicates\n" " -C : collapse duplicates\n"
" -d also dump file contents.\n" " -d also dump file contents.\n"
" -n [first-]<cnt> define the result slice. The default value for [first]\n" " -n [first-]<cnt> define the result slice. The default value for [first]\n"
" is 0. Without the option, the default max count is 2000.\n" " is 0. Without the option, the default max count is 2000.\n"
" Use n=0 for no limit.\n" " Use n=0 for no limit.\n"
" -b : basic. Just output urls, no mime types or titles.\n" " -b : basic. Just output urls, no mime types or titles.\n"
" -Q : no result lines, just the processed query and result count.\n" " -Q : no result lines, just the processed query and result count.\n"
" -m : dump the whole document meta[] array for each result.\n" " -m : dump the whole document meta[] array for each result.\n"
" -A : output the document abstracts.\n" " -A : output the document abstracts.\n"
" -S fld : sort by field <fld>.\n" " -S fld : sort by field <fld>.\n"
" -D : sort descending.\n" " -D : sort descending.\n"
" -s stemlang : set stemming language to use (must exist in index...).\n" " -s stemlang : set stemming language to use (must exist in index...).\n"
" Use -s \"\" to turn off stem expansion.\n" " Use -s \"\" to turn off stem expansion.\n"
" -T <synonyms file>: use the parameter (Thesaurus) for word expansion.\n" " -T <synonyms file>: use the parameter (Thesaurus) for word expansion.\n"
" -i <dbdir> : additional index, several can be given.\n" " -i <dbdir> : additional index, several can be given.\n"
" -e use url encoding (%xx) for urls.\n" " -e use url encoding (%xx) for urls.\n"
" -E use exact result count instead of lower bound estimate" " -E use exact result count instead of lower bound estimate"
" -F <field name list> : output exactly these fields for each result.\n" " -F <field name list> : output exactly these fields for each result.\n"
" The field values are encoded in base64, output in one line and \n" " The field values are encoded in base64, output in one line and \n"
" separated by one space character. This is the recommended format \n" " separated by one space character. This is the recommended format \n"
" for use by other programs. Use a normal query with option -m to \n" " for use by other programs. Use a normal query with option -m to \n"
" see the field names. Use -F '' to output all fields, but you probably\n" " see the field names. Use -F '' to output all fields, but you probably\n"
" also want option -N in this case.\n" " also want option -N in this case.\n"
" -N : with -F, print the (plain text) field names before the field values.\n" " -N : with -F, print the (plain text) field names before the field values.\n"
; ;
static void static void
Usage(void) Usage(void)
{ {
@ -200,61 +200,61 @@ int recollq(RclConfig **cfp, int argc, char **argv)
Usage(); Usage();
while (**argv) while (**argv)
switch (*(*argv)++) { switch (*(*argv)++) {
case '-': case '-':
// -- : end of options // -- : end of options
if (*(*argv) != 0) if (*(*argv) != 0)
Usage(); Usage();
goto endopts; goto endopts;
case 'A': op_flags |= OPT_A; break; case 'A': op_flags |= OPT_A; break;
case 'a': op_flags |= OPT_a; break; case 'a': op_flags |= OPT_a; break;
case 'b': op_flags |= OPT_b; break; case 'b': op_flags |= OPT_b; break;
case 'C': op_flags |= OPT_C; break; case 'C': op_flags |= OPT_C; break;
case 'c': op_flags |= OPT_c; if (argc < 2) Usage(); case 'c': op_flags |= OPT_c; if (argc < 2) Usage();
a_config = *(++argv); a_config = *(++argv);
argc--; goto b1; argc--; goto b1;
case 'd': op_flags |= OPT_d; break; case 'd': op_flags |= OPT_d; break;
case 'D': op_flags |= OPT_D; break; case 'D': op_flags |= OPT_D; break;
case 'E': op_flags |= OPT_E; break; case 'E': op_flags |= OPT_E; break;
case 'e': op_flags |= OPT_e; break; case 'e': op_flags |= OPT_e; break;
case 'f': op_flags |= OPT_f; break; case 'f': op_flags |= OPT_f; break;
case 'F': op_flags |= OPT_F; if (argc < 2) Usage(); case 'F': op_flags |= OPT_F; if (argc < 2) Usage();
sf = *(++argv); sf = *(++argv);
argc--; goto b1; argc--; goto b1;
case 'i': op_flags |= OPT_i; if (argc < 2) Usage(); case 'i': op_flags |= OPT_i; if (argc < 2) Usage();
extra_dbs.push_back(*(++argv)); extra_dbs.push_back(*(++argv));
argc--; goto b1; argc--; goto b1;
case 'l': op_flags |= OPT_l; break; case 'l': op_flags |= OPT_l; break;
case 'm': op_flags |= OPT_m; break; case 'm': op_flags |= OPT_m; break;
case 'N': op_flags |= OPT_N; break; case 'N': op_flags |= OPT_N; break;
case 'n': op_flags |= OPT_n; if (argc < 2) Usage(); case 'n': op_flags |= OPT_n; if (argc < 2) Usage();
{ {
string rescnt = *(++argv); string rescnt = *(++argv);
string::size_type dash = rescnt.find("-"); string::size_type dash = rescnt.find("-");
if (dash != string::npos) { if (dash != string::npos) {
firstres = atoi(rescnt.substr(0, dash).c_str()); firstres = atoi(rescnt.substr(0, dash).c_str());
if (dash < rescnt.size()-1) { if (dash < rescnt.size()-1) {
maxcount = atoi(rescnt.substr(dash+1).c_str()); maxcount = atoi(rescnt.substr(dash+1).c_str());
} }
} else { } else {
maxcount = atoi(rescnt.c_str()); maxcount = atoi(rescnt.c_str());
} }
if (maxcount <= 0) maxcount = INT_MAX; if (maxcount <= 0) maxcount = INT_MAX;
} }
argc--; goto b1; argc--; goto b1;
case 'o': op_flags |= OPT_o; break; case 'o': op_flags |= OPT_o; break;
case 'P': op_flags |= OPT_P; break; case 'P': op_flags |= OPT_P; break;
case 'q': op_flags |= OPT_q; break; case 'q': op_flags |= OPT_q; break;
case 'Q': op_flags |= OPT_Q; break; case 'Q': op_flags |= OPT_Q; break;
case 'S': op_flags |= OPT_S; if (argc < 2) Usage(); case 'S': op_flags |= OPT_S; if (argc < 2) Usage();
sortfield = *(++argv); sortfield = *(++argv);
argc--; goto b1; argc--; goto b1;
case 's': op_flags |= OPT_s; if (argc < 2) Usage(); case 's': op_flags |= OPT_s; if (argc < 2) Usage();
stemlang = *(++argv); stemlang = *(++argv);
argc--; goto b1; argc--; goto b1;
case 't': op_flags |= OPT_t; break; case 't': op_flags |= OPT_t; break;
case 'T': op_flags |= OPT_T; if (argc < 2) Usage(); case 'T': op_flags |= OPT_T; if (argc < 2) Usage();
syngroupsfn = *(++argv); syngroupsfn = *(++argv);
argc--; goto b1; argc--; goto b1;
default: Usage(); break; default: Usage(); break;
} }
b1: argc--; argv++; b1: argc--; argv++;
@ -265,17 +265,17 @@ endopts:
*cfp = recollinit(0, 0, 0, reason, &a_config); *cfp = recollinit(0, 0, 0, reason, &a_config);
RclConfig *rclconfig = *cfp; RclConfig *rclconfig = *cfp;
if (!rclconfig || !rclconfig->ok()) { if (!rclconfig || !rclconfig->ok()) {
fprintf(stderr, "Recoll init failed: %s\n", reason.c_str()); fprintf(stderr, "Recoll init failed: %s\n", reason.c_str());
exit(1); exit(1);
} }
if (argc < 1 && !(op_flags & OPT_P)) { if (argc < 1 && !(op_flags & OPT_P)) {
Usage(); Usage();
} }
if (op_flags & OPT_F) { if (op_flags & OPT_F) {
if (op_flags & (OPT_b|OPT_d|OPT_b|OPT_Q|OPT_m|OPT_A)) if (op_flags & (OPT_b|OPT_d|OPT_b|OPT_Q|OPT_m|OPT_A))
Usage(); Usage();
stringToStrings(sf, fields); stringToStrings(sf, fields);
} }
Rcl::Db rcldb(rclconfig); Rcl::Db rcldb(rclconfig);
if (!extra_dbs.empty()) { if (!extra_dbs.empty()) {
@ -295,9 +295,9 @@ endopts:
} }
if (!rcldb.open(Rcl::Db::DbRO)) { if (!rcldb.open(Rcl::Db::DbRO)) {
cerr << "Cant open database in " << rclconfig->getDbDir() << cerr << "Cant open database in " << rclconfig->getDbDir() <<
" reason: " << rcldb.getReason() << endl; " reason: " << rcldb.getReason() << endl;
exit(1); exit(1);
} }
if (op_flags & OPT_P) { if (op_flags & OPT_P) {
@ -312,48 +312,48 @@ endopts:
} }
if (argc < 1) { if (argc < 1) {
Usage(); Usage();
} }
string qs = *argv++;argc--; string qs = *argv++;argc--;
while (argc > 0) { while (argc > 0) {
qs += string(" ") + *argv++;argc--; qs += string(" ") + *argv++;argc--;
} }
{ {
string uq; string uq;
string charset = rclconfig->getDefCharset(true); string charset = rclconfig->getDefCharset(true);
int ercnt; int ercnt;
if (!transcode(qs, uq, charset, "UTF-8", &ercnt)) { if (!transcode(qs, uq, charset, "UTF-8", &ercnt)) {
fprintf(stderr, "Can't convert command line args to utf-8\n"); fprintf(stderr, "Can't convert command line args to utf-8\n");
exit(1); exit(1);
} else if (ercnt) { } else if (ercnt) {
fprintf(stderr, "%d errors while converting arguments from %s " fprintf(stderr, "%d errors while converting arguments from %s "
"to utf-8\n", ercnt, charset.c_str()); "to utf-8\n", ercnt, charset.c_str());
} }
qs = uq; qs = uq;
} }
Rcl::SearchData *sd = 0; Rcl::SearchData *sd = 0;
if (op_flags & (OPT_a|OPT_o|OPT_f)) { if (op_flags & (OPT_a|OPT_o|OPT_f)) {
sd = new Rcl::SearchData(Rcl::SCLT_OR, stemlang); sd = new Rcl::SearchData(Rcl::SCLT_OR, stemlang);
Rcl::SearchDataClause *clp = 0; Rcl::SearchDataClause *clp = 0;
if (op_flags & OPT_f) { if (op_flags & OPT_f) {
clp = new Rcl::SearchDataClauseFilename(qs); clp = new Rcl::SearchDataClauseFilename(qs);
} else { } else {
clp = new Rcl::SearchDataClauseSimple((op_flags & OPT_o)? clp = new Rcl::SearchDataClauseSimple((op_flags & OPT_o)?
Rcl::SCLT_OR : Rcl::SCLT_AND, Rcl::SCLT_OR : Rcl::SCLT_AND,
qs); qs);
} }
if (sd) if (sd)
sd->addClause(clp); sd->addClause(clp);
} else { } else {
sd = wasaStringToRcl(rclconfig, stemlang, qs, reason); sd = wasaStringToRcl(rclconfig, stemlang, qs, reason);
} }
if (!sd) { if (!sd) {
cerr << "Query string interpretation failed: " << reason << endl; cerr << "Query string interpretation failed: " << reason << endl;
return 1; return 1;
} }
std::shared_ptr<Rcl::SearchData> rq(sd); std::shared_ptr<Rcl::SearchData> rq(sd);
@ -362,12 +362,12 @@ endopts:
query.setCollapseDuplicates(true); query.setCollapseDuplicates(true);
} }
if (op_flags & OPT_S) { if (op_flags & OPT_S) {
query.setSortBy(sortfield, (op_flags & OPT_D) ? false : true); query.setSortBy(sortfield, (op_flags & OPT_D) ? false : true);
} }
Chrono chron; Chrono chron;
if (!query.setQuery(rq)) { if (!query.setQuery(rq)) {
cerr << "Query setup failed: " << query.getReason() << endl; cerr << "Query setup failed: " << query.getReason() << endl;
return(1); return(1);
} }
int cnt; int cnt;
if (op_flags & OPT_E) { if (op_flags & OPT_E) {
@ -376,62 +376,62 @@ endopts:
cnt = query.getResCnt(); cnt = query.getResCnt();
} }
if (!(op_flags & OPT_b)) { if (!(op_flags & OPT_b)) {
cout << "Recoll query: " << rq->getDescription() << endl; cout << "Recoll query: " << rq->getDescription() << endl;
if (firstres == 0) { if (firstres == 0) {
if (cnt <= maxcount) if (cnt <= maxcount)
cout << cnt << " results" << endl; cout << cnt << " results" << endl;
else else
cout << cnt << " results (printing " << maxcount << " max):" cout << cnt << " results (printing " << maxcount << " max):"
<< endl; << endl;
} else { } else {
cout << "Printing at most " << cnt - (firstres+maxcount) << cout << "Printing at most " << cnt - (firstres+maxcount) <<
" results from first " << firstres << endl; " results from first " << firstres << endl;
} }
} }
if (op_flags & OPT_Q) if (op_flags & OPT_Q)
cout << "Query setup took " << chron.millis() << " mS" << endl; cout << "Query setup took " << chron.millis() << " mS" << endl;
if (op_flags & OPT_Q) if (op_flags & OPT_Q)
return(0); return(0);
for (int i = firstres; i < firstres + maxcount; i++) { for (int i = firstres; i < firstres + maxcount; i++) {
Rcl::Doc doc; Rcl::Doc doc;
if (!query.getDoc(i, doc)) if (!query.getDoc(i, doc))
break; break;
if (op_flags & OPT_F) { if (op_flags & OPT_F) {
output_fields(fields, doc, query, rcldb, op_flags & OPT_N); output_fields(fields, doc, query, rcldb, op_flags & OPT_N);
continue; continue;
} }
if (op_flags & OPT_e) if (op_flags & OPT_e)
doc.url = url_encode(doc.url); doc.url = url_encode(doc.url);
if (op_flags & OPT_b) { if (op_flags & OPT_b) {
cout << doc.url << endl; cout << doc.url << endl;
} else { } else {
string titleorfn = doc.meta[Rcl::Doc::keytt]; string titleorfn = doc.meta[Rcl::Doc::keytt];
if (titleorfn.empty()) if (titleorfn.empty())
titleorfn = doc.meta[Rcl::Doc::keyfn]; titleorfn = doc.meta[Rcl::Doc::keyfn];
if (titleorfn.empty()) { if (titleorfn.empty()) {
string url; string url;
printableUrl(rclconfig->getDefCharset(), doc.url, url); printableUrl(rclconfig->getDefCharset(), doc.url, url);
titleorfn = path_getsimple(url); titleorfn = path_getsimple(url);
} }
char cpc[20]; char cpc[20];
sprintf(cpc, "%d", doc.pc); sprintf(cpc, "%d", doc.pc);
cout cout
<< doc.mimetype << "\t" << doc.mimetype << "\t"
<< "[" << doc.url << "]" << "\t" << "[" << doc.url << "]" << "\t"
<< "[" << titleorfn << "]" << "\t" << "[" << titleorfn << "]" << "\t"
<< doc.fbytes << "\tbytes" << "\t" << doc.fbytes << "\tbytes" << "\t"
<< endl; << endl;
if (op_flags & OPT_m) { if (op_flags & OPT_m) {
for (const auto ent : doc.meta) { for (const auto ent : doc.meta) {
cout << ent.first << " = " << ent.second << endl; cout << ent.first << " = " << ent.second << endl;
} }
} }
if (op_flags & OPT_A) { if (op_flags & OPT_A) {
string abstract; string abstract;
if (query.makeDocAbstract(doc, abstract)) { if (query.makeDocAbstract(doc, abstract)) {
@ -443,7 +443,7 @@ endopts:
} }
if (op_flags & OPT_d) { if (op_flags & OPT_d) {
dump_contents(rclconfig, doc); dump_contents(rclconfig, doc);
} }
} }
return 0; return 0;