diff --git a/src/common/rclconfig.cpp b/src/common/rclconfig.cpp index 046fd929..8211bd99 100644 --- a/src/common/rclconfig.cpp +++ b/src/common/rclconfig.cpp @@ -67,7 +67,7 @@ bool ParamStale::needrecompute() { LOGDEB2(("ParamStale:: needrecompute. parent gen %d mine %d\n", parent->m_keydirgen, savedkeydirgen)); - if (parent->m_keydirgen != savedkeydirgen) { + if (active && parent->m_keydirgen != savedkeydirgen) { LOGDEB2(("ParamState:: needrecompute. conffile %p\n", conffile)); savedkeydirgen = parent->m_keydirgen; @@ -90,6 +90,9 @@ void ParamStale::init(RclConfig *rconf, ConfNull *cnf, const string& nm) parent = rconf; conffile = cnf; paramname = nm; + active = false; + if (conffile) + active = conffile->hasNameAnywhere(nm); savedkeydirgen = -1; } @@ -107,6 +110,7 @@ void RclConfig::zeroMe() { m_stpsuffstate.init(this, 0, "recoll_noindex"); m_skpnstate.init(this, 0, "skippedNames"); m_rmtstate.init(this, 0, "indexedmimetypes"); + m_mdrstate.init(this, 0, "metadatacmds"); } bool RclConfig::isDefaultConfig() const @@ -246,6 +250,7 @@ RclConfig::RclConfig(const string *argcnf) m_stpsuffstate.init(this, mimemap, "recoll_noindex"); m_skpnstate.init(this, m_conf, "skippedNames"); m_rmtstate.init(this, m_conf, "indexedmimetypes"); + m_mdrstate.init(this, m_conf, "metadatacmds"); return; } @@ -262,13 +267,14 @@ bool RclConfig::updateMainConfig() m_ok = false; m_skpnstate.init(this, 0, "skippedNames"); m_rmtstate.init(this, 0, "indexedmimetypes"); + m_mdrstate.init(this, 0, "metadatacmds"); return false; } delete m_conf; m_conf = newconf; m_skpnstate.init(this, m_conf, "skippedNames"); m_rmtstate.init(this, m_conf, "indexedmimetypes"); - + m_mdrstate.init(this, m_conf, "metadatacmds"); setKeyDir(cstr_null); bool nocjk = false; @@ -666,6 +672,32 @@ string RclConfig::getMimeHandlerDef(const string &mtype, bool filtertypes) return hs; } +const vector& RclConfig::getMDReapers() +{ + string hs; + if (m_mdrstate.needrecompute()) { + m_mdreapers.clear(); + // New value now stored in m_mdrstate.savedvalue + string& sreapers = m_mdrstate.savedvalue; + if (sreapers.empty()) + return m_mdreapers; + string value; + ConfSimple attrs; + valueSplitAttributes(sreapers, value, attrs); + vector nmlst = attrs.getNames(cstr_null); + for (vector::const_iterator it = nmlst.begin(); + it != nmlst.end(); it++) { + MDReaper reaper; + reaper.fieldname = fieldCanon(*it); + string s; + attrs.get(*it, s); + stringToStrings(s, reaper.cmdv); + m_mdreapers.push_back(reaper); + } + } + return m_mdreapers; +} + bool RclConfig::getGuiFilterNames(vector& cats) const { if (!mimeconf) @@ -1346,6 +1378,7 @@ void RclConfig::initFrom(const RclConfig& r) m_stpsuffstate.init(this, mimemap, r.m_stpsuffstate.paramname); m_skpnstate.init(this, m_conf, r.m_skpnstate.paramname); m_rmtstate.init(this, m_conf, r.m_rmtstate.paramname); + m_mdrstate.init(this, m_conf, r.m_mdrstate.paramname); m_thrConf = r.m_thrConf; } diff --git a/src/common/rclconfig.h b/src/common/rclconfig.h index 11abe361..e158286d 100644 --- a/src/common/rclconfig.h +++ b/src/common/rclconfig.h @@ -44,6 +44,7 @@ public: RclConfig *parent; ConfNull *conffile; string paramname; + bool active; // Check at init if config defines name at all int savedkeydirgen; string savedvalue; @@ -51,6 +52,12 @@ public: bool needrecompute(); }; +// Hold the description for an external metadata-gathering command +struct MDReaper { + string fieldname; + vector cmdv; +}; + // Data associated to a indexed field name: struct FieldTraits { string pfx; // indexing prefix, @@ -244,6 +251,9 @@ class RclConfig { * exceptions are found in the nouncompforviewmts mimeview list */ bool mimeViewerNeedsUncomp(const string &mimetype) const; + /** Retrieve extra metadata-gathering commands */ + const vector& getMDReapers(); + /** Store/retrieve missing helpers description string */ bool getMissingHelperDesc(string&) const; void storeMissingHelperDesc(const string &s); @@ -319,6 +329,11 @@ class RclConfig { set m_restrictMTypes; vector > m_thrConf; + // Same idea with the metadata-gathering external commands, + // (e.g. used to reap tagging info: "tmsu tags %f") + ParamStale m_mdrstate; + vector m_mdreapers; + /** Create initial user configuration */ bool initUserConfig(); /** Copy from other */ diff --git a/src/index/fsindexer.cpp b/src/index/fsindexer.cpp index 55466641..109ab060 100644 --- a/src/index/fsindexer.cpp +++ b/src/index/fsindexer.cpp @@ -75,13 +75,12 @@ extern void *FsIndexerDbUpdWorker(void*); class InternfileTask { public: InternfileTask(const std::string &f, const struct stat *i_stp, - map lfields, vector reapers) - : fn(f), statbuf(*i_stp), localfields(lfields), mdreapers(reapers) + map lfields) + : fn(f), statbuf(*i_stp), localfields(lfields) {} string fn; struct stat statbuf; map localfields; - vector mdreapers; }; extern void *FsIndexerInternfileWorker(void*); #endif // IDX_THREADS @@ -113,7 +112,6 @@ FsIndexer::FsIndexer(RclConfig *cnf, Rcl::Db *db, DbIxStatusUpdater *updfunc) { LOGDEB1(("FsIndexer::FsIndexer\n")); m_havelocalfields = m_config->hasNameAnywhere("localfields"); - m_havemdreapers = m_config->hasNameAnywhere("metadatacmds"); #ifdef IDX_THREADS m_stableconfig = new RclConfig(*m_config); @@ -326,8 +324,6 @@ bool FsIndexer::indexFiles(list& files, ConfIndexer::IxFlag flag) m_config->setKeyDir(path_getfather(*it)); if (m_havelocalfields) localfieldsfromconf(); - if (m_havemdreapers) - mdreapersfromconf(); bool follow = false; m_config->getConfParam("followLinks", &follow); @@ -465,58 +461,6 @@ void FsIndexer::setlocalfields(const map& fields, Rcl::Doc& doc) } } -// Metadata gathering commands -void FsIndexer::mdreapersfromconf() -{ - LOGDEB1(("FsIndexer::mdreapersfromconf\n")); - - string sreapers; - m_config->getConfParam("metadatacmds", sreapers); - if (!sreapers.compare(m_smdreapers)) - return; - - m_smdreapers = sreapers; - m_mdreapers.clear(); - if (sreapers.empty()) - return; - - string value; - ConfSimple attrs; - m_config->valueSplitAttributes(sreapers, value, attrs); - vector nmlst = attrs.getNames(cstr_null); - for (vector::const_iterator it = nmlst.begin(); - it != nmlst.end(); it++) { - MDReaper reaper; - reaper.fieldname = m_config->fieldCanon(*it); - string s; - attrs.get(*it, s); - stringToStrings(s, reaper.cmdv); - m_mdreapers.push_back(reaper); - } -} - -void FsIndexer::reapmetadata(const vector& reapers, const string& fn, - Rcl::Doc& doc) -{ - map smap = create_map('f', fn); - for (vector::const_iterator rp = reapers.begin(); - rp != reapers.end(); rp++) { - vector cmd; - for (vector::const_iterator it = rp->cmdv.begin(); - it != rp->cmdv.end(); it++) { - string s; - pcSubst(*it, s, smap); - cmd.push_back(s); - } - string output; - if (ExecCmd::backtick(cmd, output)) { - // addmeta() creates or appends. fieldname is already - // canonic (see above) - doc.addmeta(rp->fieldname, output); - } - } -} - void FsIndexer::makesig(const struct stat *stp, string& out) { char cbuf[100]; @@ -569,7 +513,7 @@ void *FsIndexerInternfileWorker(void * fsp) } LOGDEB0(("FsIndexerInternfileWorker: task fn %s\n", tsk->fn.c_str())); if (fip->processonefile(&myconf, tsk->fn, &tsk->statbuf, - tsk->localfields, tsk->mdreapers) != + tsk->localfields) != FsTreeWalker::FtwOk) { LOGERR(("FsIndexerInternfileWorker: processone failed\n")); tqp->workerExit(); @@ -617,8 +561,6 @@ FsIndexer::processone(const std::string &fn, const struct stat *stp, // Adjust local fields from config for this subtree if (m_havelocalfields) localfieldsfromconf(); - if (m_havemdreapers) - mdreapersfromconf(); if (flg == FsTreeWalker::FtwDirReturn) return FsTreeWalker::FtwOk; @@ -626,17 +568,16 @@ FsIndexer::processone(const std::string &fn, const struct stat *stp, #ifdef IDX_THREADS if (m_haveInternQ) { - InternfileTask *tp = new InternfileTask(fn, stp, m_localfields, - m_mdreapers); - if (m_iwqueue.put(tp)) { - return FsTreeWalker::FtwOk; - } else { + InternfileTask *tp = new InternfileTask(fn, stp, m_localfields); + if (m_iwqueue.put(tp)) { + return FsTreeWalker::FtwOk; + } else { return FsTreeWalker::FtwError; - } + } } #endif - return processonefile(m_config, fn, stp, m_localfields, m_mdreapers); + return processonefile(m_config, fn, stp, m_localfields); } // File name transcoded to utf8 for indexing. If this fails, the file @@ -664,8 +605,7 @@ static string compute_utf8fn(RclConfig *config, const string& fn) FsTreeWalker::Status FsIndexer::processonefile(RclConfig *config, const std::string &fn, const struct stat *stp, - const map& localfields, - const vector& mdreapers) + const map& localfields) { //////////////////// // Check db up to date ? Doing this before file type @@ -742,8 +682,6 @@ FsIndexer::processonefile(RclConfig *config, // for the main file. if (doc.ipath.empty()) { hadNullIpath = true; - if (m_havemdreapers) - reapmetadata(mdreapers, fn, doc); if (hadNonNullIpath) { // Note that only the filters can reliably compute // this. What we do is dependant of the doc order (if @@ -843,8 +781,6 @@ FsIndexer::processonefile(RclConfig *config, fileDoc.url = cstr_fileu + fn; if (m_havelocalfields) setlocalfields(localfields, fileDoc); - if (m_havemdreapers) - reapmetadata(mdreapers, fn, fileDoc); char cbuf[100]; sprintf(cbuf, OFFTPC, stp->st_size); fileDoc.pcbytes = cbuf; diff --git a/src/index/fsindexer.h b/src/index/fsindexer.h index 0269152e..4f3a176c 100644 --- a/src/index/fsindexer.h +++ b/src/index/fsindexer.h @@ -76,12 +76,6 @@ class FsIndexer : public FsTreeWalkerCB { /** Make signature for file up to date checks */ static void makesig(const struct stat *stp, string& out); - /* Hold the description for an external metadata-gathering command */ - struct MDReaper { - string fieldname; - vector cmdv; - }; - private: class PurgeCandidateRecorder { @@ -131,16 +125,13 @@ class FsIndexer : public FsTreeWalkerCB { // all files in a file system area. Ie: set "rclaptg = thunderbird" // inside ~/.thunderbird. The boolean is set at init to avoid // further wasteful processing if no local fields are set. + // This should probably moved to internfile so that the + // localfields get exactly the same processing as those generated by the + // filters (as was done for metadatacmds fields) bool m_havelocalfields; string m_slocalfields; map m_localfields; - // Same idea with the metadata-gathering external commands, - // (e.g. used to reap tagging info: "tmsu tags %f") - bool m_havemdreapers; - string m_smdreapers; - vector m_mdreapers; - #ifdef IDX_THREADS friend void *FsIndexerDbUpdWorker(void*); friend void *FsIndexerInternfileWorker(void*); @@ -154,15 +145,11 @@ class FsIndexer : public FsTreeWalkerCB { bool init(); void localfieldsfromconf(); - void mdreapersfromconf(); void setlocalfields(const map& flds, Rcl::Doc& doc); - void reapmetadata(const vector& reapers, const string &fn, - Rcl::Doc& doc); string getDbDir() {return m_config->getDbDir();} FsTreeWalker::Status processonefile(RclConfig *config, const string &fn, - const struct stat *, const map& localfields, - const vector& mdreapers); + const struct stat *, const map& localfields); }; #endif /* _fsindexer_h_included_ */ diff --git a/src/internfile/internfile.cpp b/src/internfile/internfile.cpp index 9efb9b81..a82380b4 100644 --- a/src/internfile/internfile.cpp +++ b/src/internfile/internfile.cpp @@ -118,6 +118,28 @@ void FileInterner::reapXAttrs(const string& path) } #endif // RCL_USE_XATTR +void FileInterner::reapCmdMetadata(const string& fn) +{ + const vector& reapers = m_cfg->getMDReapers(); + if (reapers.empty()) + return; + map smap = create_map('f', fn); + for (vector::const_iterator rp = reapers.begin(); + rp != reapers.end(); rp++) { + vector cmd; + for (vector::const_iterator it = rp->cmdv.begin(); + it != rp->cmdv.end(); it++) { + string s; + pcSubst(*it, s, smap); + cmd.push_back(s); + } + string output; + if (ExecCmd::backtick(cmd, output)) { + m_cmdFields[rp->fieldname] = output; + } + } +} + // This is used when the user wants to retrieve a search result doc's parent // (ie message having a given attachment) bool FileInterner::getEnclosingUDI(const Rcl::Doc &doc, string& udi) @@ -265,12 +287,12 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf, if (!df or df->is_unknown()) { // No real handler for this type, for now :( LOGDEB(("FileInterner:: unprocessed mime: [%s] [%s]\n", - l_mime.c_str(), f.c_str())); + l_mime.c_str(), f.c_str())); if (!df) return; } df->set_property(Dijon::Filter::OPERATING_MODE, - m_forPreview ? "view" : "index"); + m_forPreview ? "view" : "index"); df->set_property(Dijon::Filter::DJF_UDI, udi); #ifdef RCL_USE_XATTR @@ -279,6 +301,7 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf, // file reapXAttrs(f); #endif //RCL_USE_XATTR + reapCmdMetadata(f); df->set_docsize(docsize); if (!df->set_document_file(l_mime, m_fn)) { @@ -625,6 +648,21 @@ void FileInterner::collectIpathAndMT(Rcl::Doc& doc) const } #endif //RCL_USE_XATTR + // Set fields from external commands + // These override those from xattrs and can be later augmented by + // values from inside the file + for (map::const_iterator it = m_cmdFields.begin(); + it != m_cmdFields.end(); it++) { + string fieldname = m_cfg->fieldCanon(it->first); + LOGDEB0(("Internfile:: setting [%s] from cmd value [%s]\n", + fieldname.c_str(), it->second.c_str())); + if (fieldname == cstr_dj_keymd) { + doc.dmtime = it->second; + } else { + doc.meta[fieldname] = it->second; + } + } + // If there is no ipath stack, the mimetype is the one from the file doc.mimetype = m_mimetype; diff --git a/src/internfile/internfile.h b/src/internfile/internfile.h index b7c9489c..07b0dcb0 100644 --- a/src/internfile/internfile.h +++ b/src/internfile/internfile.h @@ -259,6 +259,8 @@ class FileInterner { // processing the internal doc hierarchy. map m_XAttrsFields; #endif // RCL_USE_XATTR + // Fields gathered by executing configured external commands + map m_cmdFields; // Filter stack, path to the current document from which we're // fetching subdocs @@ -289,6 +291,7 @@ class FileInterner { #ifdef RCL_USE_XATTR void reapXAttrs(const string& fn); #endif + void reapCmdMetadata(const string& fn); }; diff --git a/src/lib/Makefile b/src/lib/Makefile index cd685045..9d030a4d 100644 --- a/src/lib/Makefile +++ b/src/lib/Makefile @@ -4,7 +4,7 @@ include $(depth)/mk/sysconf LIBRECOLL=librecoll.so.$(RCLLIBVERSION) LIBS = librecoll.a -#LIBS = librecoll.a $(LIBRECOLL) +LIBS = librecoll.a $(LIBRECOLL) all: $(LIBS)