textfilemaxmbs

src/internfile/mh_text.cpp

@@ -17,6 +17,9 @@ static char rcsid[] = "@(#$Id: mh_text.cpp,v 1.6 2006-12-15 12:40:02 dockes Exp
  *   Free Software Foundation, Inc.,
  *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  */
+#include <unistd.h>
+#include <sys/stat.h>
+#include <errno.h>
 
 #include <iostream>
 #include <string>
@@ -30,16 +33,35 @@ using namespace std;
 #include "readfile.h"
 #include "transcode.h"
 #include "md5.h"
+#include "rclconfig.h"
+
+const int MB = 1024*1024;
 
 // Process a plain text file
 bool MimeHandlerText::set_document_file(const string &fn)
 {
     RecollFilter::set_document_file(fn);
+
+    // file size
+    struct stat st;
+    if (stat(fn.c_str(), &st) < 0) {
+        LOGERR(("MimeHandlerText::set_document_file: stat(%s) errno %d\n",
+                fn.c_str(), errno));
+        return false;
+    }
+
+    // Handle max file size parameter. If it's too big, we just don't index
+    // the text at all (should we index the first maxmbs instead ?)
+    int maxmbs = -1;
+    RclConfig::getMainConfig()->getConfParam("textfilemaxmbs", &maxmbs);
+
     string otext;
-    string reason;
+    if (st.st_size / MB <= maxmbs) {
-    if (!file_to_string(fn, otext, &reason)) {
+        string reason;
-	LOGERR(("MimeHandlerText: can't read file: %s\n", reason.c_str()));
+        if (!file_to_string(fn, otext, &reason)) {
-	return false;
+            LOGERR(("MimeHandlerText: can't read file: %s\n", reason.c_str()));
+            return false;
+        }
     }
     return set_document_string(otext);
 }

src/qtgui/confgui/confguiindex.cpp

@@ -365,6 +365,18 @@ ConfSubPanelW::ConfSubPanelW(QWidget *parent, ConfNull *config)
 		      -1, 1000000);
     m_widgets.push_back(ezfmaxkbs);
 
+    ConfLink lnktxtmaxmbs(new ConfLinkRclRep(config, 
+					    "textfilemaxmbs"));
+    ConfParamIntW *etxtmaxmbs = new 
+	ConfParamIntW(m_groupbox, lnktxtmaxmbs, 
+		      tr("Max. text file size (MB)"),
+		      tr("This value sets a threshold beyond which text "
+			 "files will not be processed. Set to -1 for no "
+			 "limit. This is for excluding monster "
+                         "log files from the index."),
+		      -1, 1000000);
+    m_widgets.push_back(etxtmaxmbs);
+
     vboxLayout->addWidget(m_groupbox);
     subDirChanged();
 }

src/sampleconf/recoll.conf.in

@@ -104,6 +104,9 @@ indexallfilenames = 1
 # processing of any compressed file
 # compressedfilemaxkbs = -1
 
+# Size limit for text files. This is for skipping monster logs
+textfilemaxmbs = -1
+
 # Length of abstracts we store while indexing. Longer will make for a
 # bigger db
 # idxabsmlen = 250