move the execution of external metadata-gathering commands from fsindexer to internfile for consistency of handling with filter-generated metadata

src/index/fsindexer.cpp

@@ -75,13 +75,12 @@ extern void *FsIndexerDbUpdWorker(void*);
 class InternfileTask {
 public:
     InternfileTask(const std::string &f, const struct stat *i_stp,
-		   map<string,string> lfields, vector<FsIndexer::MDReaper> reapers)
-	: fn(f), statbuf(*i_stp), localfields(lfields), mdreapers(reapers)
+		   map<string,string> lfields)
+	: fn(f), statbuf(*i_stp), localfields(lfields)
     {}
     string fn;
     struct stat statbuf;
     map<string,string> localfields;
-    vector<FsIndexer::MDReaper> mdreapers;
 };
 extern void *FsIndexerInternfileWorker(void*);
 #endif // IDX_THREADS
@@ -113,7 +112,6 @@ FsIndexer::FsIndexer(RclConfig *cnf, Rcl::Db *db, DbIxStatusUpdater *updfunc)
 {
     LOGDEB1(("FsIndexer::FsIndexer\n"));
     m_havelocalfields = m_config->hasNameAnywhere("localfields");
-    m_havemdreapers = m_config->hasNameAnywhere("metadatacmds");
 
 #ifdef IDX_THREADS
     m_stableconfig = new RclConfig(*m_config);
@@ -326,8 +324,6 @@ bool FsIndexer::indexFiles(list<string>& files, ConfIndexer::IxFlag flag)
         m_config->setKeyDir(path_getfather(*it));
 	if (m_havelocalfields)
 	    localfieldsfromconf();
-	if (m_havemdreapers)
-	    mdreapersfromconf();
 
 	bool follow = false;
 	m_config->getConfParam("followLinks", &follow);
@@ -465,58 +461,6 @@ void FsIndexer::setlocalfields(const map<string, string>& fields, Rcl::Doc& doc)
     }
 }
 
-// Metadata gathering commands
-void FsIndexer::mdreapersfromconf()
-{
-    LOGDEB1(("FsIndexer::mdreapersfromconf\n"));
-
-    string sreapers;
-    m_config->getConfParam("metadatacmds", sreapers);
-    if (!sreapers.compare(m_smdreapers)) 
-	return;
-
-    m_smdreapers = sreapers;
-    m_mdreapers.clear();
-    if (sreapers.empty())
-	return;
-
-    string value;
-    ConfSimple attrs;
-    m_config->valueSplitAttributes(sreapers, value, attrs);
-    vector<string> nmlst = attrs.getNames(cstr_null);
-    for (vector<string>::const_iterator it = nmlst.begin();
-         it != nmlst.end(); it++) {
-	MDReaper reaper;
-	reaper.fieldname = m_config->fieldCanon(*it);
-	string s;
-	attrs.get(*it, s);
-	stringToStrings(s, reaper.cmdv);
-	m_mdreapers.push_back(reaper);
-    }
-}
-
-void FsIndexer::reapmetadata(const vector<MDReaper>& reapers, const string& fn,
-			     Rcl::Doc& doc)
-{
-    map<char,string> smap = create_map<char, string>('f', fn);
-    for (vector<MDReaper>::const_iterator rp = reapers.begin();
-	 rp != reapers.end(); rp++) {
-	vector<string> cmd;
-	for (vector<string>::const_iterator it = rp->cmdv.begin();
-	     it != rp->cmdv.end(); it++) {
-	    string s;
-	    pcSubst(*it, s, smap);
-	    cmd.push_back(s);
-	}
-	string output;
-	if (ExecCmd::backtick(cmd, output)) {
-	    // addmeta() creates or appends. fieldname is already
-	    // canonic (see above)
-	    doc.addmeta(rp->fieldname, output);
-	}
-    }
-}
-
 void FsIndexer::makesig(const struct stat *stp, string& out)
 {
     char cbuf[100]; 
@@ -569,7 +513,7 @@ void *FsIndexerInternfileWorker(void * fsp)
 	}
 	LOGDEB0(("FsIndexerInternfileWorker: task fn %s\n", tsk->fn.c_str()));
 	if (fip->processonefile(&myconf, tsk->fn, &tsk->statbuf,
-				tsk->localfields, tsk->mdreapers) !=
+				tsk->localfields) !=
 	    FsTreeWalker::FtwOk) {
 	    LOGERR(("FsIndexerInternfileWorker: processone failed\n"));
 	    tqp->workerExit();
@@ -617,8 +561,6 @@ FsIndexer::processone(const std::string &fn, const struct stat *stp,
         // Adjust local fields from config for this subtree
 	if (m_havelocalfields)
 	    localfieldsfromconf();
-	if (m_havemdreapers)
-	    mdreapersfromconf();
 
 	if (flg == FsTreeWalker::FtwDirReturn)
 	    return FsTreeWalker::FtwOk;
@@ -626,17 +568,16 @@ FsIndexer::processone(const std::string &fn, const struct stat *stp,
 
 #ifdef IDX_THREADS
     if (m_haveInternQ) {
-	InternfileTask *tp = new InternfileTask(fn, stp, m_localfields, 
-	    m_mdreapers);
-	if (m_iwqueue.put(tp)) {
-	    return FsTreeWalker::FtwOk;
-	} else {
+        InternfileTask *tp = new InternfileTask(fn, stp, m_localfields);
+        if (m_iwqueue.put(tp)) {
+            return FsTreeWalker::FtwOk;
+        } else {
 	    return FsTreeWalker::FtwError;
-	}
+        }
     }
 #endif
 
-    return processonefile(m_config, fn, stp, m_localfields, m_mdreapers);
+    return processonefile(m_config, fn, stp, m_localfields);
 }
 
 // File name transcoded to utf8 for indexing.  If this fails, the file
@@ -664,8 +605,7 @@ static string compute_utf8fn(RclConfig *config, const string& fn)
 FsTreeWalker::Status 
 FsIndexer::processonefile(RclConfig *config, 
 			  const std::string &fn, const struct stat *stp,
-			  const map<string, string>& localfields,
-			  const vector<MDReaper>& mdreapers)
+			  const map<string, string>& localfields)
 {
     ////////////////////
     // Check db up to date ? Doing this before file type
@@ -742,8 +682,6 @@ FsIndexer::processonefile(RclConfig *config,
 	// for the main file.
 	if (doc.ipath.empty()) {
 	    hadNullIpath = true;
-	    if (m_havemdreapers)
-		reapmetadata(mdreapers, fn, doc);
 	    if (hadNonNullIpath) {
 		// Note that only the filters can reliably compute
 		// this. What we do is dependant of the doc order (if
@@ -843,8 +781,6 @@ FsIndexer::processonefile(RclConfig *config,
 	fileDoc.url = cstr_fileu + fn;
         if (m_havelocalfields) 
             setlocalfields(localfields, fileDoc);
-	if (m_havemdreapers)
-	    reapmetadata(mdreapers, fn, fileDoc);
 	char cbuf[100]; 
 	sprintf(cbuf, OFFTPC, stp->st_size);
 	fileDoc.pcbytes = cbuf;

src/index/fsindexer.h

@@ -76,12 +76,6 @@ class FsIndexer : public FsTreeWalkerCB {
     /** Make signature for file up to date checks */
     static void makesig(const struct stat *stp, string& out);
 
-    /* Hold the description for an external metadata-gathering command */
-    struct MDReaper {
-	string fieldname;
-	vector<string> cmdv;
-    };
-
  private:
 
     class PurgeCandidateRecorder {
@@ -131,16 +125,13 @@ class FsIndexer : public FsTreeWalkerCB {
     // all files in a file system area. Ie: set "rclaptg = thunderbird"
     // inside ~/.thunderbird. The boolean is set at init to avoid
     // further wasteful processing if no local fields are set.
+    // This should probably moved to internfile so that the
+    // localfields get exactly the same processing as those generated by the
+    // filters (as was done for metadatacmds fields)
     bool         m_havelocalfields;
     string       m_slocalfields;
     map<string, string>  m_localfields;
 
-    // Same idea with the metadata-gathering external commands,
-    // (e.g. used to reap tagging info: "tmsu tags %f")
-    bool m_havemdreapers;
-    string m_smdreapers;
-    vector<MDReaper> m_mdreapers;
-
 #ifdef IDX_THREADS
     friend void *FsIndexerDbUpdWorker(void*);
     friend void *FsIndexerInternfileWorker(void*);
@@ -154,15 +145,11 @@ class FsIndexer : public FsTreeWalkerCB {
 
     bool init();
     void localfieldsfromconf();
-    void mdreapersfromconf();
     void setlocalfields(const map<string, string>& flds, Rcl::Doc& doc);
-    void reapmetadata(const vector<MDReaper>& reapers, const string &fn,
-		      Rcl::Doc& doc);
     string getDbDir() {return m_config->getDbDir();}
     FsTreeWalker::Status 
     processonefile(RclConfig *config, const string &fn, 
-		   const struct stat *, const map<string,string>& localfields,
-		   const vector<MDReaper>& mdreapers);
+		   const struct stat *, const map<string,string>& localfields);
 };
 
 #endif /* _fsindexer_h_included_ */