Add GUI dialog to start indexing with special options. Closes issue #264

src/doc/man/recollindex.1

@@ -67,8 +67,18 @@ recollindex \- indexing command for the Recoll full text search system
 .B \-Z
 ]
 [
+.B \-K
+]
+[
+.B \-e
+]
+[
 .B \-f
 ]
+[
+.B \-p 
+pattern
+]
 <dirpath>
 .br
 .B recollindex 
@@ -208,6 +218,14 @@ control system).
 will erase data for individual files from the database. The stem expansion
 databases will not be updated.
 .PP
+Options
+.B
+\-i
+and
+.B
+\-e
+can be combined. This will first perform the purge, then the indexing.
+.PP
 With options 
 .B \-i 
 or 
@@ -215,24 +233,30 @@ or
 , if no file names are given on the command line, they
 will be read from stdin, so that you could for example run:
 .PP
-find /path/to/dir \-print | recollindex \-e 
+find /path/to/dir \-print | recollindex \-e \-i
-.PP
-followed by 
-.PP
-find /path/to/dir \-print | recollindex \-i
 .PP
 to force the reindexing of a directory tree (which has to exist inside the
 file system area defined by
 .I topdirs 
 in recoll.conf). You could mostly accomplish the same thing with
 .PP
-.B find /path/to/dir \-print | recollindex \-f \-Z 
+find /path/to/dir \-print | recollindex \-Z \-i
+.PP
+The latter will perform a less thorough job of purging stale sub-documents
+though.
 .PP
 .B recollindex \-r
-mostly works like \-i, but the parameter is a single directory, which will
+mostly works like 
+.B \-i
+, but the parameter is a single directory, which will
 be recursively updated. This mostly does nothing more than 
 .B find topdir | recollindex \-i
-but it may be more convenient to use when started from another program.
+but it may be more convenient to use when started from another
+program. This retries failed files by default, use option
+.B \-K
+to change. One or multiple 
+.B \-p
+options can be used to set shell-type selection patterns (e.g.: *.pdf).
 .PP
 .B recollindex \-l 
 will list the names of available language stemmers.

src/index/recollindex.cpp

@@ -19,6 +19,7 @@
 #include <stdio.h>
 #include <signal.h>
 #include <errno.h>
+#include <fnmatch.h>
 #ifndef _WIN32
 #include <sys/time.h>
 #include <sys/resource.h>
@@ -78,7 +79,7 @@ static int     op_flags;
 #define OPT_r     0x40000
 #define OPT_k     0x80000
 #define OPT_E     0x100000
-
+#define OPT_K     0x200000
 ReExec *o_reexec;
 
 // Globals for atexit cleanup
@@ -195,27 +196,46 @@ void rclIxIonice(const RclConfig *config)
 
 class MakeListWalkerCB : public FsTreeWalkerCB {
 public:
-    MakeListWalkerCB(list<string>& files)
+    MakeListWalkerCB(list<string>& files, const vector<string>& selpats)
-	: m_files(files)
+	: m_files(files), m_pats(selpats)
     {
     }
     virtual FsTreeWalker::Status 
     processone(const string& fn, const struct stat *, FsTreeWalker::CbFlag flg) 
     {
-	if (flg == FsTreeWalker::FtwDirEnter || flg == FsTreeWalker::FtwRegular)
+	if (flg== FsTreeWalker::FtwDirEnter || flg == FsTreeWalker::FtwRegular){
-	    m_files.push_back(fn);
+            if (m_pats.empty()) {
+                cerr << "Selecting " << fn << endl;
+                m_files.push_back(fn);
+            } else {
+                for (vector<string>::const_iterator it = m_pats.begin();
+                     it != m_pats.end(); it++) {
+                    if (fnmatch(it->c_str(), fn.c_str(), 0) == 0) {
+                        m_files.push_back(fn);
+                        break;
+                    }
+                }
+            }
+        }
 	return FsTreeWalker::FtwOk;
     }
     list<string>& m_files;
+    const vector<string>& m_pats;
 };
 
-// Build a list of things to index and call indexfiles.
+// Build a list of things to index, then call purgefiles and/or
-bool recursive_index(RclConfig *config, const string& top)
+// indexfiles.  This is basically the same as find xxx | recollindex
+// -i [-e] without the find (so, simpler but less powerfull)
+bool recursive_index(RclConfig *config, const string& top, 
+                     const vector<string>& selpats)
 {
     list<string> files;
-    MakeListWalkerCB cb(files);
+    MakeListWalkerCB cb(files, selpats);
     FsTreeWalker walker;
     walker.walk(top, cb);
+    if (op_flags & OPT_e) {
+        purgefiles(config, files);
+    }
     return indexfiles(config, files);
 }
 
@@ -232,9 +252,13 @@ bool indexfiles(RclConfig *config, list<string> &filenames)
     if (filenames.empty())
 	return true;
     makeIndexerOrExit(config, (op_flags & OPT_Z) != 0);
-    return confindexer->indexFiles(filenames, (op_flags&OPT_f) ? 
+    // The default is to retry failed files
-				   ConfIndexer::IxFIgnoreSkip : 
+    int indexerFlags = ConfIndexer::IxFNone; 
-				   ConfIndexer::IxFNone);
+    if (op_flags & OPT_K) 
+        indexerFlags |= ConfIndexer::IxFNoRetryFailed; 
+    if (op_flags & OPT_f)
+        indexerFlags |= ConfIndexer::IxFIgnoreSkip;
+    return confindexer->indexFiles(filenames, indexerFlags);
 }
 
 // Delete a list of files. Same comments about call contexts as indexfiles.
@@ -348,8 +372,11 @@ static const char usage [] =
 "recollindex -i [-f] [-Z] <filename [filename ...]>\n"
 "    Index individual files. No database purge or stem database updates\n"
 "    -f : ignore skippedPaths and skippedNames while doing this\n"
-"recollindex -r [-f] [-Z] <top> \n"
+"recollindex -r [-K] [-f] [-Z] [-p pattern] <top> \n"
-"   Recursive partial reindex\n"
+"   Recursive partial reindex. \n"
+"     -p : filter file names, multiple instances are allowed, e.g.: \n"
+"        -p *.odt -p *.pdf\n"
+"     -K : skip previously failed files (they are retried by default)\n"
 "recollindex -l\n"
 "    List available stemming languages\n"
 "recollindex -s <lang>\n"
@@ -398,6 +425,7 @@ int main(int argc, char **argv)
 {
     string a_config;
     int sleepsecs = 60;
+    vector<string> selpatterns;
 
     // The reexec struct is used by the daemon to shed memory after
     // the initial indexing pass and to restart when the configuration
@@ -430,9 +458,13 @@ int main(int argc, char **argv)
 	    case 'h': op_flags |= OPT_h; break;
 	    case 'i': op_flags |= OPT_i; break;
 	    case 'k': op_flags |= OPT_k; break;
+	    case 'K': op_flags |= OPT_K; break;
 	    case 'l': op_flags |= OPT_l; break;
 	    case 'm': op_flags |= OPT_m; break;
 	    case 'n': op_flags |= OPT_n; break;
+	    case 'p':	if (argc < 2)  Usage();
+		selpatterns.push_back(*(++argv));
+		argc--; goto b1;
 	    case 'r': op_flags |= OPT_r; break;
 	    case 's': op_flags |= OPT_s; break;
 #ifdef RCL_USE_ASPELL
@@ -545,7 +577,15 @@ int main(int argc, char **argv)
     rclIxIonice(config);
 #endif
 
-    if (op_flags & (OPT_i|OPT_e)) {
+    if (op_flags & OPT_r) {
+	if (argc != 1) 
+	    Usage();
+	string top = *argv++; argc--;
+	bool status = recursive_index(config, top, selpatterns);
+        if (confindexer && !confindexer->getReason().empty())
+            cerr << confindexer->getReason() << endl;
+        exit(status ? 0 : 1);
+    } else if (op_flags & (OPT_i|OPT_e)) {
 	lockorexit(&pidfile);
 
 	list<string> filenames;
@@ -577,14 +617,6 @@ int main(int argc, char **argv)
         if (confindexer && !confindexer->getReason().empty())
             cerr << confindexer->getReason() << endl;
         exit(status ? 0 : 1);
-    } else if (op_flags & OPT_r) {
-	if (argc != 1) 
-	    Usage();
-	string top = *argv++; argc--;
-	bool status = recursive_index(config, top);
-        if (confindexer && !confindexer->getReason().empty())
-            cerr << confindexer->getReason() << endl;
-        exit(status ? 0 : 1);
     } else if (op_flags & OPT_l) {
 	if (argc != 0) 
 	    Usage();

src/qtgui/rclm_idx.cpp

@@ -26,6 +26,7 @@
 #include "transcode.h"
 #include "indexer.h"
 #include "rclmain_w.h"
+#include "specialindex.h"
 
 using namespace std;
 
@@ -116,25 +117,25 @@ void RclMain::periodic100()
 	m_indexerState = IXST_RUNNINGMINE;
 	fileToggleIndexingAction->setText(tr("Stop &Indexing"));
 	fileToggleIndexingAction->setEnabled(true);
-        fileRetryFailedAction->setEnabled(false);
 	fileRebuildIndexAction->setEnabled(false);
+        actionSpecial_Indexing->setEnabled(false);
 	periodictimer->setInterval(200);
     } else {
 	Pidfile pidfile(theconfig->getPidfile());
 	if (pidfile.open() == 0) {
 	    m_indexerState = IXST_NOTRUNNING;
 	    fileToggleIndexingAction->setText(tr("Update &Index"));
-            fileRetryFailedAction->setEnabled(true);
 	    fileToggleIndexingAction->setEnabled(true);
 	    fileRebuildIndexAction->setEnabled(true);
+            actionSpecial_Indexing->setEnabled(true);
 	    periodictimer->setInterval(1000);
 	} else {
 	    // Real time or externally started batch indexer running
 	    m_indexerState = IXST_RUNNINGNOTMINE;
 	    fileToggleIndexingAction->setText(tr("Stop &Indexing"));
 	    fileToggleIndexingAction->setEnabled(true);
-            fileRetryFailedAction->setEnabled(false);
 	    fileRebuildIndexAction->setEnabled(false);
+            actionSpecial_Indexing->setEnabled(false);
 	    periodictimer->setInterval(200);
 	}	    
     }
@@ -229,8 +230,6 @@ void RclMain::toggleIndexing()
         args.clear();
 	args.push_back("-c");
 	args.push_back(theconfig->getConfDir());
-        if (fileRetryFailedAction->isChecked())
-            args.push_back("-k");
 	m_idxproc = new ExecCmd;
 	m_idxproc->startExec("recollindex", args, false, false);
     }
@@ -249,6 +248,10 @@ void RclMain::rebuildIndex()
 	return; //?? Should not have been called
     case IXST_NOTRUNNING:
     {
+        if (m_idxproc) {
+            LOGERR(("RclMain::rebuildIndex: current indexer exec not null\n"));
+            return;
+        }
 	int rep = 
 	    QMessageBox::warning(0, tr("Erasing index"), 
 				     tr("Reset the index and start "
@@ -274,6 +277,126 @@ void RclMain::rebuildIndex()
     }
 }
 
+void SpecIdxW::onBrowsePB_clicked()
+{
+    QString dir = myGetFileName(true, tr("Top indexed entity"), true);
+    targLE->setText(dir);
+}
+
+bool SpecIdxW::noRetryFailed()
+{
+    return noRetryFailedCB->isChecked();
+}
+
+bool SpecIdxW::eraseFirst()
+{
+    return eraseBeforeCB->isChecked();
+}
+
+std::vector<std::string> SpecIdxW::selpatterns()
+{
+    vector<string> pats;
+    string text = qs2utf8s(selPatsLE->text());
+    if (!text.empty()) {
+        stringToStrings(text, pats);
+    }
+    return pats;
+}
+
+std::string SpecIdxW::toptarg()
+{
+    return qs2utf8s(targLE->text());
+}
+
+void SpecIdxW::onTargLE_textChanged(const QString& text)
+{
+    if (text.isEmpty())
+        selPatsLE->setEnabled(false);
+    else
+        selPatsLE->setEnabled(true);
+}
+
+static string execToString(const string& cmd, const vector<string>& args)
+{
+    string command = cmd + " ";
+    for (vector<string>::const_iterator it = args.begin();
+         it != args.end(); it++) {
+        command += "{" + *it + "} ";
+    }
+    return command;
+}
+
+void RclMain::specialIndex()
+{
+    LOGDEB(("RclMain::specialIndex\n"));
+    switch (m_indexerState) {
+    case IXST_UNKNOWN:
+    case IXST_RUNNINGMINE:
+    case IXST_RUNNINGNOTMINE:
+	return; //?? Should not have been called
+    case IXST_NOTRUNNING:
+    default:
+        break;
+    }
+    if (m_idxproc) {
+        LOGERR(("RclMain::rebuildIndex: current indexer exec not null\n"));
+        return;
+    }
+    if (!specidx) // ??
+        return;
+
+    vector<string> args;
+    args.push_back("-c");
+    args.push_back(theconfig->getConfDir());
+
+    string top = specidx->toptarg();
+    if (!top.empty()) {
+        args.push_back("-r");
+    }
+
+    if (specidx->eraseFirst()) {
+        if (top.empty()) {
+            args.push_back("-Z");
+        } else {
+            args.push_back("-e");
+        }
+    } 
+
+    // The default for retrying differ depending if -r is set
+    if (top.empty()) {
+        if (!specidx->noRetryFailed()) {
+            args.push_back("-k");
+        }
+    } else {
+        if (specidx->noRetryFailed()) {
+            args.push_back("-K");
+        }
+    }
+        
+    vector<string> selpats = specidx->selpatterns();
+    if (!selpats.empty() && top.empty()) {
+        QMessageBox::warning(0, tr("Selection patterns need topdir"), 
+                             tr("Selection patterns can only be used with a "
+                                "start directory"),
+                                     QMessageBox::Ok,
+                                     QMessageBox::NoButton);
+        return;
+    }
+
+    for (vector<string>::const_iterator it = selpats.begin();
+         it != selpats.end(); it++) {
+        args.push_back("-p");
+        args.push_back(*it);
+    }
+    if (!top.empty()) {
+        args.push_back(top);
+    }
+    m_idxproc = new ExecCmd;
+    LOGINFO(("specialIndex: exec: %s\n", 
+            execToString("recollindex", args).c_str()));
+    m_idxproc->startExec("recollindex", args, false, false);
+}
+
 void RclMain::updateIdxForDocs(vector<Rcl::Doc>& docs)
 {
     if (m_idxproc) {
@@ -297,6 +420,6 @@ void RclMain::updateIdxForDocs(vector<Rcl::Doc>& docs)
 	fileToggleIndexingAction->setText(tr("Stop &Indexing"));
     }
     fileToggleIndexingAction->setEnabled(false);
-    fileRetryFailedAction->setEnabled(false);
+    actionSpecial_Indexing->setEnabled(false);
 }
 

src/qtgui/rclm_wins.cpp

@@ -28,6 +28,7 @@
 #include "rtitool.h"
 #include "snippets_w.h"
 #include "fragbuts.h"
+#include "specialindex.h"
 #include "rclmain_w.h"
 
 using namespace std;
@@ -94,6 +95,19 @@ void RclMain::showFragButs()
     }
 }
 
+void RclMain::showSpecIdx()
+{
+    if (specidx == 0) {
+	specidx = new SpecIdxW(0);
+        connect(specidx, SIGNAL(accepted()), this, SLOT(specialIndex()));
+        specidx->show();
+    } else {
+	// Close and reopen, in hope that makes us visible...
+	specidx->close();
+        specidx->show();
+    }
+}
+
 void RclMain::showIndexConfig()
 {
     showIndexConfig(false);

src/qtgui/rclmain.ui

@@ -73,8 +73,8 @@
      <string>&amp;File</string>
     </property>
     <addaction name="fileToggleIndexingAction"/>
-    <addaction name="fileRetryFailedAction"/>
     <addaction name="fileRebuildIndexAction"/>
+    <addaction name="actionSpecial_Indexing"/>
     <addaction name="separator"/>
     <addaction name="actionSave_last_query"/>
     <addaction name="actionLoad_saved_query"/>
@@ -497,6 +497,14 @@
     <string>Load saved query</string>
    </property>
   </action>
+  <action name="actionSpecial_Indexing">
+   <property name="text">
+    <string>Special Indexing</string>
+   </property>
+   <property name="toolTip">
+    <string>Indexing with special options</string>
+   </property>
+  </action>
  </widget>
  <layoutdefault spacing="2" margin="2"/>
  <customwidgets>

src/qtgui/rclmain_w.cpp

@@ -327,6 +327,8 @@ void RclMain::init()
 	    this, SLOT(showSpellDialog()));
     connect(actionQuery_Fragments, SIGNAL(triggered()), 
 	    this, SLOT(showFragButs()));
+    connect(actionSpecial_Indexing, SIGNAL(triggered()), 
+	    this, SLOT(showSpecIdx()));
     connect(indexConfigAction, SIGNAL(triggered()), 
 	    this, SLOT(showIndexConfig()));
     connect(indexScheduleAction, SIGNAL(triggered()), 

src/qtgui/rclmain_w.h

@@ -40,6 +40,7 @@ class ResTable;
 class CronToolW;
 class RTIToolW;
 class FragButs;
+class SpecIdxW;
 
 #include "ui_rclmain.h"
 
@@ -69,6 +70,7 @@ public:
 	  rtiTool(0),
 	  spellform(0),
           fragbuts(0),
+          specidx(0),
 	  periodictimer(0),
 	  restable(0),
 	  displayingTable(0),
@@ -114,11 +116,13 @@ public slots:
     virtual void periodic100();
     virtual void toggleIndexing();
     virtual void rebuildIndex();
+    virtual void specialIndex();
     virtual void startSearch(STD_SHARED_PTR<Rcl::SearchData> sdata, bool issimple);
     virtual void previewClosed(Preview *w);
     virtual void showAdvSearchDialog();
     virtual void showSpellDialog();
     virtual void showFragButs();
+    virtual void showSpecIdx();
     virtual void showAboutDialog();
     virtual void showMissingHelpers();
     virtual void showActiveTypes();
@@ -191,6 +195,7 @@ private:
     RTIToolW       *rtiTool;
     SpellW         *spellform;
     FragButs       *fragbuts;
+    SpecIdxW       *specidx;
     QTimer         *periodictimer;
     ResTable       *restable;
     bool            displayingTable;

src/qtgui/specialindex.h

@@ -0,0 +1,53 @@
+/*
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ */
+#ifndef _SPECIDX_W_H_INCLUDED_
+#define _SPECIDX_W_H_INCLUDED_
+
+#include <vector>
+#include <string>
+
+#include "ui_specialindex.h"
+
+class QPushButton;
+
+class SpecIdxW : public QDialog, public Ui::SpecIdxW {
+    Q_OBJECT
+
+public:
+
+    SpecIdxW(QWidget * parent = 0) 
+	: QDialog(parent)
+    {
+	setupUi(this);
+        selPatsLE->setEnabled(false);
+        connect(browsePB, SIGNAL(clicked()), this, SLOT(onBrowsePB_clicked()));
+        connect(targLE, SIGNAL(textChanged(const QString&)), 
+                this, SLOT(onTargLE_textChanged(const QString&)));
+    }
+    bool noRetryFailed();
+    bool eraseFirst();
+    std::vector<std::string> selpatterns();
+    std::string toptarg();
+
+public slots:
+
+    void onTargLE_textChanged(const QString&);
+    void onBrowsePB_clicked();
+};
+
+
+#endif /* _SPECIDX_W_H_INCLUDED_ */

src/qtgui/specialindex.ui

@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>SpecIdxW</class>
+ <widget class="QDialog" name="SpecIdxW">
+  <property name="windowModality">
+   <enum>Qt::WindowModal</enum>
+  </property>
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>413</width>
+    <height>191</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Special Indexing</string>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout_2">
+   <item>
+    <layout class="QVBoxLayout" name="verticalLayout">
+     <item>
+      <widget class="QCheckBox" name="noRetryFailedCB">
+       <property name="text">
+        <string>Do not retry previously failed files.</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QCheckBox" name="eraseBeforeCB">
+       <property name="toolTip">
+        <string>Else only modified or failed files will be processed.</string>
+       </property>
+       <property name="text">
+        <string>Erase selected files data before indexing.</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <layout class="QGridLayout">
+       <item row="1" column="0">
+        <widget class="QLineEdit" name="targLE">
+         <property name="sizePolicy">
+          <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+           <horstretch>8</horstretch>
+           <verstretch>0</verstretch>
+          </sizepolicy>
+         </property>
+         <property name="minimumSize">
+          <size>
+           <width>300</width>
+           <height>0</height>
+          </size>
+         </property>
+         <property name="toolTip">
+          <string>Directory to recursively index</string>
+         </property>
+        </widget>
+       </item>
+       <item row="1" column="1">
+        <widget class="QPushButton" name="browsePB">
+         <property name="text">
+          <string>Browse</string>
+         </property>
+         <property name="autoDefault">
+          <bool>false</bool>
+         </property>
+        </widget>
+       </item>
+       <item row="0" column="0">
+        <widget class="QLabel" name="textLabel1">
+         <property name="text">
+          <string>Start directory (else use regular topdirs):</string>
+         </property>
+         <property name="wordWrap">
+          <bool>false</bool>
+         </property>
+        </widget>
+       </item>
+      </layout>
+     </item>
+     <item>
+      <layout class="QHBoxLayout" name="horizontalLayout">
+       <item>
+        <widget class="QLabel" name="patLBL">
+         <property name="toolTip">
+          <string>Leave empty to select all files. You can use multiple space-separated shell-type patterns.&lt;br&gt;Patterns with embedded spaces should be quoted with double quotes.&lt;br&gt;Can only be used if the start target is set.</string>
+         </property>
+         <property name="text">
+          <string>Selection patterns:</string>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <widget class="QLineEdit" name="selPatsLE"/>
+       </item>
+      </layout>
+     </item>
+    </layout>
+   </item>
+   <item>
+    <widget class="QDialogButtonBox" name="buttonBox">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons">
+      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>SpecIdxW</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>SpecIdxW</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>