tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

comments and LOG prettifying

Browse Source
This commit is contained in:
Jean-Francois Dockes 2017-07-20 07:52:22 +02:00
parent 9f02bc8119
commit 32e79d301b
5 changed files with 46 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/index/exefetcher.cpp
View File

@ -47,10 +47,12 @@ public:
args.push_back(idoc.ipath); args.push_back(idoc.ipath);
int status = ecmd.doexec1(args, 0, &out); int status = ecmd.doexec1(args, 0, &out);
if (status == 0) { if (status == 0) {
LOGDEB("EXEDocFetcher::Internal: got [" << (out) << "]\n" ); LOGDEB("EXEDocFetcher::Internal: got [" << out << "]\n");
return true; return true;
} else { } else {
LOGERR("EXEDOcFetcher::fetch: " << (bckid) << ": " << (stringsToString(cmd)) << " failed for " << (udi) << " " << (idoc.url) << " " << (idoc.ipath) << "\n" ); LOGERR("EXEDOcFetcher::fetch: " << bckid << ": " <<
stringsToString(cmd) << " failed for " << udi << " " <<
idoc.url << " " << idoc.ipath << "\n");
return false; return false;
} }
} }
@ -59,7 +61,8 @@ public:
EXEDocFetcher::EXEDocFetcher(const EXEDocFetcher::Internal& _m) EXEDocFetcher::EXEDocFetcher(const EXEDocFetcher::Internal& _m)
{ {
m = new Internal(_m); m = new Internal(_m);
LOGDEB("EXEDocFetcher::EXEDocFetcher: fetch is " << (stringsToString(m->sfetch)) << "\n" ); LOGDEB("EXEDocFetcher::EXEDocFetcher: fetch is " <<
stringsToString(m->sfetch) << "\n");
} }
bool EXEDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out) bool EXEDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
@ -82,12 +85,12 @@ EXEDocFetcher *exeDocFetcherMake(RclConfig *config, const string& bckid)
static ConfSimple *bconf; static ConfSimple *bconf;
if (!bconf) { if (!bconf) {
string bconfname = path_cat(config->getConfDir(), "backends"); string bconfname = path_cat(config->getConfDir(), "backends");
LOGDEB("exeDocFetcherMake: using config in " << (bconfname) << "\n" ); LOGDEB("exeDocFetcherMake: using config in " << bconfname << "\n");
bconf = new ConfSimple(bconfname.c_str(), true); bconf = new ConfSimple(bconfname.c_str(), true);
if (!bconf->ok()) { if (!bconf->ok()) {
delete bconf; delete bconf;
bconf = 0; bconf = 0;
LOGDEB("exeDocFetcherMake: bad/no config: " << (bconfname) << "\n" ); LOGDEB("exeDocFetcherMake: bad/no config: " << bconfname << "\n");
return 0; return 0;
} }
} }
@ -97,28 +100,29 @@ EXEDocFetcher *exeDocFetcherMake(RclConfig *config, const string& bckid)
string sfetch; string sfetch;
if (!bconf->get("fetch", sfetch, bckid) || sfetch.empty()) { if (!bconf->get("fetch", sfetch, bckid) || sfetch.empty()) {
LOGERR("exeDocFetcherMake: no 'fetch' for [" << (bckid) << "]\n" ); LOGERR("exeDocFetcherMake: no 'fetch' for [" << bckid << "]\n");
return 0; return 0;
} }
stringToStrings(sfetch, m.sfetch); stringToStrings(sfetch, m.sfetch);
// We look up the command as we do for filters for now // We look up the command as we do for filters for now
m.sfetch[0] = config->findFilter(m.sfetch[0]); m.sfetch[0] = config->findFilter(m.sfetch[0]);
if (!path_isabsolute(m.sfetch[0])) { if (!path_isabsolute(m.sfetch[0])) {
LOGERR("exeDocFetcherMake: " << (m.sfetch[0]) << " not found in exec path or filters dir\n" ); LOGERR("exeDocFetcherMake: " << m.sfetch[0] <<
" not found in exec path or filters dir\n");
return 0; return 0;
} }
string smkid; string smkid;
if (!bconf->get("makesig", smkid, bckid) || smkid.empty()) { if (!bconf->get("makesig", smkid, bckid) || smkid.empty()) {
LOGDEB("exeDocFetcherMake: no 'makesig' for [" << (bckid) << "]\n" ); LOGDEB("exeDocFetcherMake: no 'makesig' for [" << bckid << "]\n");
return 0; return 0;
} }
stringToStrings(smkid, m.smkid); stringToStrings(smkid, m.smkid);
m.smkid[0] = config->findFilter(m.smkid[0]); m.smkid[0] = config->findFilter(m.smkid[0]);
if (!path_isabsolute(m.smkid[0])) { if (!path_isabsolute(m.smkid[0])) {
LOGERR("exeDocFetcherMake: " << (m.smkid[0]) << " not found in exec path or filters dir\n" ); LOGERR("exeDocFetcherMake: " << m.smkid[0] <<
" not found in exec path or filters dir\n");
return 0; return 0;
} }
return new EXEDocFetcher(m); return new EXEDocFetcher(m);
} }

13
src/index/exefetcher.h
View File

@ -23,7 +23,16 @@ class RclConfig;
/** /**
* A fetcher which works by executing external programs, defined in a * A fetcher which works by executing external programs, defined in a
* configuration file: * configuration file.
* At this point this is only used with the sample python mbox indexer,
* to show how recoll can work with completely external data extraction code.
*
* Configuration: The external indexer sets the 'rclbes' recoll field
* (backend definition, can be FS or BGL -web- in standard recoll) to
* a unique value (e.g. MBOX for the python sample). A 'backends' file
* in the configuration directory then links the 'rclbes' value with
* commands to execute for fetching the data, which recoll uses at
* query time for previewing and opening the document.
*/ */
class EXEDocFetcher : public DocFetcher { class EXEDocFetcher : public DocFetcher {
class Internal; class Internal;
@ -32,7 +41,7 @@ class EXEDocFetcher : public DocFetcher {
virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out); virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out);
/** Calls stat to retrieve file signature data */ /** Calls stat to retrieve file signature data */
virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc, std::string& sig); virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc,std::string& sig);
friend EXEDocFetcher *exeDocFetcherMake(RclConfig *, const std::string&); friend EXEDocFetcher *exeDocFetcherMake(RclConfig *, const std::string&);
private: private:
Internal *m; Internal *m;

2
src/index/fetcher.cpp
View File

@ -42,7 +42,7 @@ DocFetcher *docFetcherMake(RclConfig *config, const Rcl::Doc& idoc)
} else { } else {
DocFetcher *f = exeDocFetcherMake(config, backend); DocFetcher *f = exeDocFetcherMake(config, backend);
if (!f) { if (!f) {
LOGERR("DocFetcherFactory: unknown backend [" << (backend) << "]\n" ); LOGERR("DocFetcherFactory: unknown backend [" << backend << "]\n");
} }
return f; return f;
} }

14
src/index/fetcher.h
View File

@ -28,14 +28,19 @@ class RclConfig;
* Generic interface to retrieve the data for a document designated by * Generic interface to retrieve the data for a document designated by
* its index data (udi/ipath/url). This is used to retrieve the data * its index data (udi/ipath/url). This is used to retrieve the data
* for previewing. The actual implementation is specific to the kind * for previewing. The actual implementation is specific to the kind
* of backend (file system, beagle cache, others?...), and the * of backend (file system, web cache, others?...), and the
* implementation may of course may share code with the indexing-time * implementation may of course share code with the indexing-time
* functions from the specific backend. * functions from the specific backend.
* *
* This is used to give access the raw document container (either as a * This normally gives access the raw document container (either as a
* file or as a memory block). The Internfile code will then further * file or as a memory block). The Internfile code will then further
* process it to get to the actual document, especially if * process it to get to the actual document, especially if
* de-embedding is involved. * de-embedding is involved.
*
* The DATADIRECT document kind, which holds final extracted data, is only
* returned when using an external indexer (only the python demo sample at
* this point), in which case the whole extraction is performed by the
* external code.
*/ */
class DocFetcher { class DocFetcher {
public: public:
@ -71,7 +76,8 @@ public:
virtual ~DocFetcher() {} virtual ~DocFetcher() {}
}; };
/** Return an appropriate fetcher object given the backend string identifier */ /** Return an appropriate fetcher object given the backend string
* identifier inside idoc*/
DocFetcher *docFetcherMake(RclConfig *config, const Rcl::Doc& idoc); DocFetcher *docFetcherMake(RclConfig *config, const Rcl::Doc& idoc);
#endif /* _FETCHER_H_INCLUDED_ */ #endif /* _FETCHER_H_INCLUDED_ */

16
src/internfile/extrameta.cpp
View File

@ -33,7 +33,8 @@ static void docfieldfrommeta(RclConfig* cfg, const string& name,
const string &value, Rcl::Doc& doc) const string &value, Rcl::Doc& doc)
{ {
string fieldname = cfg->fieldCanon(name); string fieldname = cfg->fieldCanon(name);
LOGDEB0("Internfile:: setting [" << (fieldname) << "] from cmd/xattr value [" << (value) << "]\n" ); LOGDEB0("Internfile:: setting [" << fieldname <<
"] from cmd/xattr value [" << value << "]\n");
if (fieldname == cstr_dj_keymd) { if (fieldname == cstr_dj_keymd) {
doc.dmtime = value; doc.dmtime = value;
} else { } else {
@ -44,15 +45,17 @@ static void docfieldfrommeta(RclConfig* cfg, const string& name,
void reapXAttrs(const RclConfig* cfg, const string& path, void reapXAttrs(const RclConfig* cfg, const string& path,
map<string, string>& xfields) map<string, string>& xfields)
{ {
LOGDEB2("reapXAttrs: [" << (path) << "]\n" ); LOGDEB2("reapXAttrs: [" << path << "]\n");
#ifndef _WIN32 #ifndef _WIN32
// Retrieve xattrs names from files and mapping table from config // Retrieve xattrs names from files and mapping table from config
vector<string> xnames; vector<string> xnames;
if (!pxattr::list(path, &xnames)) { if (!pxattr::list(path, &xnames)) {
if (errno == ENOTSUP) { if (errno == ENOTSUP) {
LOGDEB("FileInterner::reapXattrs: pxattr::list: errno " << (errno) << "\n" ); LOGDEB("FileInterner::reapXattrs: pxattr::list: errno " <<
errno << "\n");
} else { } else {
LOGERR("FileInterner::reapXattrs: pxattr::list: errno " << (errno) << "\n" ); LOGERR("FileInterner::reapXattrs: pxattr::list: errno " <<
errno << "\n");
} }
return; return;
} }
@ -74,12 +77,13 @@ void reapXAttrs(const RclConfig* cfg, const string& path,
} }
string value; string value;
if (!pxattr::get(path, *it, &value, pxattr::PXATTR_NOFOLLOW)) { if (!pxattr::get(path, *it, &value, pxattr::PXATTR_NOFOLLOW)) {
LOGERR("FileInterner::reapXattrs: pxattr::get failedfor " << ((*it)) << ", errno " << (errno) << "\n" ); LOGERR("FileInterner::reapXattrs: pxattr::get failed for " << *it
<< ", errno " << errno << "\n");
continue; continue;
} }
// Encode should we ? // Encode should we ?
xfields[key] = value; xfields[key] = value;
LOGDEB2("reapXAttrs: [" << (key) << "] -> [" << (value) << "]\n" ); LOGDEB2("reapXAttrs: [" << key << "] -> [" << value << "]\n");
} }
#endif #endif
} }