Add recollindex option to write file not indexed reasons to diagnostics file

src/internfile/internfile.cpp

@@ -240,17 +240,15 @@ void FileInterner::init(const string &f, const struct PathStat *stp,
     m_mimetype = l_mime;
 
     // Look for appropriate handler (might still return empty)
-    RecollFilter *df = getMimeHandler(l_mime, m_cfg, !m_forPreview);
+    RecollFilter *df = getMimeHandler(l_mime, m_cfg, !m_forPreview, f);
 
     if (!df || df->is_unknown()) {
         // No real handler for this type, for now :( 
-        LOGDEB("FileInterner:: unprocessed mime: [" << l_mime << "] [" << f <<
-               "]\n");
+        LOGDEB("FileInterner:: unprocessed mime: [" << l_mime << "] [" << f << "]\n");
         if (!df)
             return;
     }
-    df->set_property(Dijon::Filter::OPERATING_MODE, 
-                     m_forPreview ? "view" : "index");
+    df->set_property(Dijon::Filter::OPERATING_MODE, m_forPreview ? "view" : "index");
     df->set_property(Dijon::Filter::DJF_UDI, udi);
 
     df->set_docsize(docsize);
@@ -271,8 +269,7 @@ FileInterner::FileInterner(const string &data, RclConfig *cnf,
     init(data, cnf, flags, imime);
 }
 
-void FileInterner::init(const string &data, RclConfig *,
-                        int, const string& imime)
+void FileInterner::init(const string &data, RclConfig *, int, const string& imime)
 {
     if (imime.empty()) {
         LOGERR("FileInterner: inmemory constructor needs input mime type\n");
@@ -281,7 +278,7 @@ void FileInterner::init(const string &data, RclConfig *,
     m_mimetype = imime;
 
     // Look for appropriate handler (might still return empty)
-    RecollFilter *df = getMimeHandler(m_mimetype, m_cfg, !m_forPreview);
+    RecollFilter *df = getMimeHandler(m_mimetype, m_cfg, !m_forPreview, m_fn);
 
     if (!df) {
         // No handler for this type, for now :( if indexallfilenames
@@ -289,8 +286,7 @@ void FileInterner::init(const string &data, RclConfig *,
         LOGDEB("FileInterner:: unprocessed mime [" << m_mimetype << "]\n");
         return;
     }
-    df->set_property(Dijon::Filter::OPERATING_MODE, 
-                     m_forPreview ? "view" : "index");
+    df->set_property(Dijon::Filter::OPERATING_MODE, m_forPreview ? "view" : "index");
 
     df->set_docsize(data.length());
     if (df->is_data_input_ok(Dijon::Filter::DOCUMENT_STRING)) {
@@ -741,12 +737,11 @@ int FileInterner::addHandler()
     getKeyValue(docdata, cstr_dj_keyipath, ipathel);
     bool dofilter = !m_forPreview &&
         (mimetype.compare(cstr_texthtml) || !ipathel.empty());
-    RecollFilter *newflt = getMimeHandler(mimetype, m_cfg, dofilter);
+    RecollFilter *newflt = getMimeHandler(mimetype, m_cfg, dofilter, m_fn);
     if (!newflt) {
         // If we can't find a handler, this doc can't be handled
         // but there can be other ones so we go on
-        LOGINFO("FileInterner::addHandler: no filter for [" << mimetype <<
-                "]\n");
+        LOGINFO("FileInterner::addHandler: no filter for [" << mimetype << "]\n");
         return ADD_CONTINUE;
     }
     newflt->set_property(Dijon::Filter::OPERATING_MODE, 

src/internfile/mh_exec.cpp

@@ -29,6 +29,7 @@
 #include "smallut.h"
 #include "md5ut.h"
 #include "rclconfig.h"
+#include "idxdiags.h"
 
 using namespace std;
 
@@ -186,6 +187,7 @@ bool MimeHandlerExec::next_document()
             missingHelper = true;
             m_reason = string("RECFILTERROR HELPERNOTFOUND ") + cmd;
             whatHelper = m_reason;
+            IdxDiags::theDiags().record(IdxDiags::MissingHelper, m_fn);
         } else if (output.find("RECFILTERROR") == 0) {
             // If the output string begins with RECFILTERROR, then it's 
             // interpretable error information out from a recoll script
@@ -193,6 +195,7 @@ bool MimeHandlerExec::next_document()
             std::string::size_type pos;
             if ((pos = output.find("RECFILTERROR ")) == 0) {
                 if (output.find("HELPERNOTFOUND") != string::npos) {
+                    IdxDiags::theDiags().record(IdxDiags::MissingHelper, m_fn);
                     missingHelper = true;
                     whatHelper = output.substr(pos);
                 }

src/internfile/mh_execm.cpp

@@ -33,6 +33,7 @@
 #include "mimetype.h"
 #include "idfile.h"
 #include "rclutil.h"
+#include "idxdiags.h"
 
 using namespace std;
 
@@ -72,6 +73,7 @@ bool MimeHandlerExecMultiple::startCmd()
     vector<string>myparams(params.begin() + 1, params.end());
 
     if (m_cmd.startExec(cmd, myparams, 1, 1) < 0) {
+        IdxDiags::theDiags().record(IdxDiags::MissingHelper, m_fn);
         m_reason = string("RECFILTERROR HELPERNOTFOUND ") + cmd;
         missingHelper = true;
         whatHelper = cmd;
@@ -113,6 +115,7 @@ bool MimeHandlerExecMultiple::readDataElement(string& name, string &data)
     if ((pos = ibuf.find("RECFILTERROR ")) == 0) {
         m_reason = ibuf;
         if (ibuf.find("HELPERNOTFOUND") != string::npos) {
+            IdxDiags::theDiags().record(IdxDiags::MissingHelper, m_fn);
             missingHelper = true;
             whatHelper = ibuf.substr(pos);
         }

src/internfile/mimehandler.cpp

@@ -256,8 +256,8 @@ MimeHandlerExec *mhExecFactory(RclConfig *cfg, const string& mtype, string& hs,
 }
 
 /* Get handler/filter object for given mime type: */
-RecollFilter *getMimeHandler(const string &mtype, RclConfig *cfg, 
-                             bool filtertypes)
+RecollFilter *getMimeHandler(const string &mtype, RclConfig *cfg,
+                             bool filtertypes, const std::string& fn)
 {
     LOGDEB("getMimeHandler: mtype [" << mtype << "] filtertypes " <<
            filtertypes << "\n");
@@ -270,7 +270,7 @@ RecollFilter *getMimeHandler(const string &mtype, RclConfig *cfg,
     // indexedmimetypes but an html handler could still be in the
     // cache because it was needed by some other interning stack).
     string hs;
-    hs = cfg->getMimeHandlerDef(mtype, filtertypes);
+    hs = cfg->getMimeHandlerDef(mtype, filtertypes, fn);
     string id;
 
     if (!hs.empty()) { 

src/internfile/mimehandler.h

@@ -169,7 +169,7 @@ protected:
  *     indexedmimetypes (if this is set at all).
  */
 extern RecollFilter *getMimeHandler(const std::string &mtyp, RclConfig *cfg,
-                                    bool filtertypes);
+                                    bool filtertypes, const std::string& fn = std::string());
 
 /// Free up filter for reuse (you can also delete it)
 extern void returnMimeHandler(RecollFilter *);