Merge branch 'RECOLL_1_23_MAINT'

packaging/debian/buildppa.sh

@@ -6,9 +6,9 @@
 
 PPA_KEYID=D38B9201
 
-RCLVERS=1.23.5
+RCLVERS=1.23.6
 SCOPEVERS=1.20.2.4
-PPAVERS=4
+PPAVERS=1
 
 # 
 RCLSRC=/y/home/dockes/projets/fulltext/recoll/src
@@ -20,7 +20,7 @@ case $RCLVERS in
     1.14*) PPANAME=recoll-ppa;;
     *)     PPANAME=recoll15-ppa;;
 esac
- PPANAME=recollexp-ppa
+# PPANAME=recollexp-ppa
 echo "PPA: $PPANAME. Type CR if Ok, else ^C"
 read rep
 
@@ -47,7 +47,7 @@ debdir=debian
 # No new releases for trusty either because of risk of kio compat (kio
 # wont build)
 series="xenial zesty artful bionic"
-series=trusty
+series="zesty artful bionic"
 
 if test "X$series" != X ; then
     check_recoll_orig
@@ -79,7 +79,7 @@ done
 ### KIO. Does not build on trusty from recoll 1.23 because of the need
 ### for c++11
 series="xenial zesty artful bionic"
-series=
+series="zesty artful bionic"
 
 debdir=debiankio
 topdir=kio-recoll-${RCLVERS}

packaging/debian/debian/changelog

@@ -1,3 +1,10 @@
+recoll (1.23.6-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
+                
+  * Make sure new indexes are created in Chert format, even with Xapian 1.4
+    This works around the Glass snippets generation issue (UI freezes).
+    
+ -- Jean-Francois Dockes <jf@dockes.org>  Fri, 08 Dec 2017 15:18:00 +0100
+
 recoll (1.23.5-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
                 
   * Support the new browser plugin based on WebExtensions.

packaging/debian/debiankio/changelog

@@ -1,3 +1,10 @@
+recoll (1.23.6-1~ppaPPAVERS~SERIES1) SERIES; urgency=low
+                
+  * Make sure new indexes are created in Chert format, even with Xapian 1.4
+    This works around the Glass snippets generation issue (UI freezes).
+    
+ -- Jean-Francois Dockes <jf@dockes.org>  Fri, 08 Dec 2017 15:18:00 +0100
+
 kio-recoll (1.23.5-0~ppaPPAVERS~SERIES1) SERIES; urgency=low
                 
   * Follow main package

src/Makefile.am

@@ -357,8 +357,6 @@ qtgui/confgui/confgui.cpp \
 qtgui/confgui/confgui.h \
 qtgui/confgui/confguiindex.cpp \
 qtgui/confgui/confguiindex.h \
-qtgui/confgui/conflinkrcl.h \
-qtgui/confgui/main.cpp \
 qtgui/crontool.cpp \
 qtgui/crontool.h \
 qtgui/crontool.ui \

src/VERSION

@@ -1 +1 @@
-1.23.5
+1.23.6

src/index/recollindex.cpp

@@ -240,10 +240,16 @@ bool recursive_index(RclConfig *config, const string& top,
     MakeListWalkerCB cb(files, selpats);
     FsTreeWalker walker;
     walker.walk(top, cb);
+    bool ret = false;
     if (op_flags & OPT_e) {
-        purgefiles(config, files);
+        if (!(ret = purgefiles(config, files))) {
+            return ret;
         }
-    return indexfiles(config, files);
+    }
+    if (!(op_flags & OPT_e) || ((op_flags & OPT_e) &&(op_flags & OPT_i))) {
+        ret = indexfiles(config, files);
+    }
+    return ret;
 }
 
 // Index a list of files. We just call the top indexer method, which

src/qtgui/confgui/confguiindex.cpp

@@ -31,6 +31,7 @@
 #include <list>
 #include <set>
 #include <string>
+#include <functional>
 using std::list;
 using std::set;
 using std::string;

src/qtgui/reslist.cpp

@@ -113,7 +113,7 @@ void logdata(const char *data)
 // /// QtGuiResListPager methods:
 bool QtGuiResListPager::append(const string& data)
 {
-    LOGDEB2("QtGuiReslistPager::appendString   : "  << (data) << "\n" );
+    LOGDEB2("QtGuiReslistPager::appendString   : " << data << "\n");
     logdata(data.c_str());
     m_reslist->append(QString::fromUtf8(data.c_str()));
     return true;
@@ -122,7 +122,8 @@ bool QtGuiResListPager::append(const string& data)
 bool QtGuiResListPager::append(const string& data, int docnum, 
 			       const Rcl::Doc&)
 {
-    LOGDEB2("QtGuiReslistPager::appendDoc: blockCount "  << (m_reslist->document()->blockCount()) << ", "  << (data) << "\n" );
+    LOGDEB2("QtGuiReslistPager::appendDoc: blockCount " <<
+            m_reslist->document()->blockCount() << ", " << data << "\n");
     logdata(data.c_str());
 #ifdef RESLIST_TEXTBROWSER
     int blkcnt0 = m_reslist->document()->blockCount();
@@ -288,7 +289,7 @@ ResList::ResList(QWidget* parent, const char* name)
     else 
 	setObjectName(name);
 #ifdef RESLIST_TEXTBROWSER
-    LOGDEB("Reslist: using QTextBrowser\n" );
+    LOGDEB("Reslist: using QTextBrowser\n");
     setReadOnly(true);
     setUndoRedoEnabled(false);
     setOpenLinks(false);
@@ -297,7 +298,7 @@ ResList::ResList(QWidget* parent, const char* name)
     connect(this, SIGNAL(anchorClicked(const QUrl &)), 
 	    this, SLOT(linkWasClicked(const QUrl &)));
 #else
-    LOGDEB("Reslist: using QWebView\n" );
+    LOGDEB("Reslist: using QWebView\n");
     // signals and slots connections
     connect(this, SIGNAL(linkClicked(const QUrl &)), 
 	    this, SLOT(linkWasClicked(const QUrl &)));
@@ -404,7 +405,7 @@ extern "C" int XFlush(void *);
 
 void ResList::setDocSource(std::shared_ptr<DocSequence> nsource)
 {
-    LOGDEB("ResList::setDocSource()\n" );
+    LOGDEB("ResList::setDocSource()\n");
     m_source = std::shared_ptr<DocSequence>(new DocSource(theconfig, nsource));
 }
 
@@ -412,7 +413,7 @@ void ResList::setDocSource(std::shared_ptr<DocSequence> nsource)
 // re-read the results.
 void ResList::readDocSource()
 {
-    LOGDEB("ResList::readDocSource()\n" );
+    LOGDEB("ResList::readDocSource()\n");
     resetView();
     if (!m_source)
 	return;
@@ -427,7 +428,7 @@ void ResList::readDocSource()
 
 void ResList::resetList() 
 {
-    LOGDEB("ResList::resetList()\n" );
+    LOGDEB("ResList::resetList()\n");
     setDocSource(std::shared_ptr<DocSequence>());
     resetView();
 }
@@ -491,14 +492,15 @@ int ResList::docnumfromparnum(int block)
 // Get range of paragraph numbers which make up the result for document number
 pair<int,int> ResList::parnumfromdocnum(int docnum)
 {
-    LOGDEB("parnumfromdocnum: docnum "  << (docnum) << "\n" );
+    LOGDEB("parnumfromdocnum: docnum " << docnum << "\n");
     if (m_pager->pageNumber() < 0) {
-	LOGDEB("parnumfromdocnum: no page return -1,-1\n" );
+	LOGDEB("parnumfromdocnum: no page return -1,-1\n");
 	return pair<int,int>(-1,-1);
     }
     int winfirst = pageFirstDocNum();
     if (docnum - winfirst < 0) {
-	LOGDEB("parnumfromdocnum: docnum "  << (docnum) << " < winfirst "  << (winfirst) << " return -1,-1\n" );
+	LOGDEB("parnumfromdocnum: docnum " << docnum << " < winfirst " <<
+               winfirst << " return -1,-1\n");
 	return pair<int,int>(-1,-1);
     }
     docnum -= winfirst;
@@ -512,11 +514,11 @@ pair<int,int> ResList::parnumfromdocnum(int docnum)
 		   m_pageParaToReldocnums.end() && it1->second == docnum) {
 		last++;
 	    }
-	    LOGDEB("parnumfromdocnum: return "  << (first) << ","  << (last) << "\n" );
+	    LOGDEB("parnumfromdocnum: return " << first << "," << last << "\n");
 	    return pair<int,int>(first, last);
 	}
     }
-    LOGDEB("parnumfromdocnum: not found return -1,-1\n" );
+    LOGDEB("parnumfromdocnum: not found return -1,-1\n");
     return pair<int,int>(-1,-1);
 }
 #endif // TEXTBROWSER
@@ -527,7 +529,8 @@ pair<int,int> ResList::parnumfromdocnum(int docnum)
 // result in a one-page change.
 bool ResList::getDoc(int docnum, Rcl::Doc &doc)
 {
-    LOGDEB("ResList::getDoc: docnum "  << (docnum) << " winfirst "  << (pageFirstDocNum()) << "\n" );
+    LOGDEB("ResList::getDoc: docnum " << docnum << " winfirst " <<
+           pageFirstDocNum() << "\n");
     int winfirst = pageFirstDocNum();
     int winlast = m_pager->pageLastDocNum();
     if (docnum < 0 ||  winfirst < 0 || winlast < 0)
@@ -617,7 +620,8 @@ void ResList::resPageDownOrNext()
 #ifdef RESLIST_TEXTBROWSER
     int vpos = verticalScrollBar()->value();
     verticalScrollBar()->triggerAction(QAbstractSlider::SliderPageStepAdd);
-    LOGDEB("ResList::resPageDownOrNext: vpos before "  << (vpos) << ", after "  << (verticalScrollBar()->value()) << "\n" );
+    LOGDEB("ResList::resPageDownOrNext: vpos before " << vpos << ", after "
+           << verticalScrollBar()->value() << "\n");
     if (vpos == verticalScrollBar()->value()) 
 	resultPageNext();
 #else
@@ -650,9 +654,9 @@ bool ResList::scrollIsAtBottom()
 	int max = frame->scrollBarMaximum(Qt::Vertical);
 	int cur = frame->scrollBarValue(Qt::Vertical); 
 	ret = (max != 0) && (cur == max);
-	LOGDEB2("Scrollatbottom: cur "  << (cur) << " max "  << (max) << "\n" );
+	LOGDEB2("Scrollatbottom: cur " << cur << " max " << max << "\n");
     }
-    LOGDEB2("scrollIsAtBottom: returning "  << (ret) << "\n" );
+    LOGDEB2("scrollIsAtBottom: returning " << ret << "\n");
     return ret;
 #endif
 }
@@ -669,10 +673,10 @@ bool ResList::scrollIsAtTop()
     } else {
 	int cur = frame->scrollBarValue(Qt::Vertical);
 	int min = frame->scrollBarMinimum(Qt::Vertical);
-	LOGDEB("Scrollattop: cur "  << (cur) << " min "  << (min) << "\n" );
+	LOGDEB("Scrollattop: cur " << cur << " min " << min << "\n");
 	ret = (cur == min);
     }
-    LOGDEB2("scrollIsAtTop: returning "  << (ret) << "\n" );
+    LOGDEB2("scrollIsAtTop: returning " << ret << "\n");
     return ret;
 #endif
 }
@@ -743,7 +747,7 @@ void ResList::displayPage()
 // Color paragraph (if any) of currently visible preview
 void ResList::previewExposed(int docnum)
 {
-    LOGDEB("ResList::previewExposed: doc "  << (docnum) << "\n" );
+    LOGDEB("ResList::previewExposed: doc " << docnum << "\n");
 
     // Possibly erase old one to white
     if (m_curPvDoc != -1) {
@@ -762,13 +766,13 @@ void ResList::previewExposed(int docnum)
 #else
 	QString sel = 
 	   QString("div[rcldocnum=\"%1\"]").arg(m_curPvDoc - pageFirstDocNum());
-	LOGDEB2("Searching for element, selector: ["  << (qs2utf8s(sel)) << "]\n" );
+	LOGDEB2("Searching for element, selector: [" << qs2utf8s(sel) << "]\n");
 	QWebElement elt = page()->mainFrame()->findFirstElement(sel);
 	if (!elt.isNull()) {
-	    LOGDEB2("Found\n" );
+	    LOGDEB2("Found\n");
 	    elt.removeAttribute("style");
 	} else {
-	    LOGDEB2("Not Found\n" );
+	    LOGDEB2("Not Found\n");
 	}
 #endif
 	m_curPvDoc = -1;
@@ -798,13 +802,13 @@ void ResList::previewExposed(int docnum)
 #else
     QString sel = 
 	QString("div[rcldocnum=\"%1\"]").arg(docnum - pageFirstDocNum());
-    LOGDEB2("Searching for element, selector: ["  << (qs2utf8s(sel)) << "]\n" );
+    LOGDEB2("Searching for element, selector: [" << qs2utf8s(sel) << "]\n");
     QWebElement elt = page()->mainFrame()->findFirstElement(sel);
     if (!elt.isNull()) {
-	LOGDEB2("Found\n" );
+	LOGDEB2("Found\n");
 	elt.setAttribute("style", "background: LightBlue;}");
     } else {
-	LOGDEB2("Not Found\n" );
+	LOGDEB2("Not Found\n");
     }
 #endif
 }
@@ -835,10 +839,14 @@ void ResList::showQueryDetails()
 
 void ResList::linkWasClicked(const QUrl &url)
 {
-    string ascurl = qs2utf8s(url.toString());
+    // qt5: url.toString() does not accept FullyDecoded, but that's what we
-    LOGDEB("ResList::linkWasClicked: ["  << (ascurl) << "]\n" );
+    // want. e.g. Suggestions links are like Sterm|spelling which we
+    // receive as Sterm%7CSpelling
+    string strurl = url_decode(qs2utf8s(url.toString()));
     
-    int what = ascurl[0];
+    LOGDEB("ResList::linkWasClicked: [" << strurl << "]\n");
+
+    int what = strurl[0];
     switch (what) {
 
     // Open abstract/snippets window
@@ -846,10 +854,10 @@ void ResList::linkWasClicked(const QUrl &url)
     {
 	if (!m_source) 
 	    return;
-	int i = atoi(ascurl.c_str()+1) - 1;
+	int i = atoi(strurl.c_str()+1) - 1;
 	Rcl::Doc doc;
 	if (!getDoc(i, doc)) {
-	    LOGERR("ResList::linkWasClicked: can't get doc for "  << (i) << "\n" );
+	    LOGERR("ResList::linkWasClicked: can't get doc for " << i << "\n");
 	    return;
 	}
 	emit(showSnippets(doc));
@@ -861,10 +869,10 @@ void ResList::linkWasClicked(const QUrl &url)
     {
 	if (!m_source) 
 	    return;
-	int i = atoi(ascurl.c_str()+1) - 1;
+	int i = atoi(strurl.c_str()+1) - 1;
 	Rcl::Doc doc;
 	if (!getDoc(i, doc)) {
-	    LOGERR("ResList::linkWasClicked: can't get doc for "  << (i) << "\n" );
+	    LOGERR("ResList::linkWasClicked: can't get doc for " << i << "\n");
 	    return;
 	}
 	vector<Rcl::Doc> dups;
@@ -877,10 +885,10 @@ void ResList::linkWasClicked(const QUrl &url)
     // Open parent folder
     case 'F':
     {
-	int i = atoi(ascurl.c_str()+1) - 1;
+	int i = atoi(strurl.c_str()+1) - 1;
 	Rcl::Doc doc;
 	if (!getDoc(i, doc)) {
-	    LOGERR("ResList::linkWasClicked: can't get doc for "  << (i) << "\n" );
+	    LOGERR("ResList::linkWasClicked: can't get doc for " << i << "\n");
 	    return;
 	}
         emit editRequested(ResultPopup::getParent(std::shared_ptr<DocSequence>(),
@@ -899,10 +907,10 @@ void ResList::linkWasClicked(const QUrl &url)
     case 'P': 
     case 'E': 
     {
-	int i = atoi(ascurl.c_str()+1) - 1;
+	int i = atoi(strurl.c_str()+1) - 1;
 	Rcl::Doc doc;
 	if (!getDoc(i, doc)) {
-	    LOGERR("ResList::linkWasClicked: can't get doc for "  << (i) << "\n" );
+	    LOGERR("ResList::linkWasClicked: can't get doc for " << i << "\n");
 	    return;
 	}
 	if (what == 'P') {
@@ -928,7 +936,7 @@ void ResList::linkWasClicked(const QUrl &url)
 	// Run script. Link format Rnn|Script Name
     case 'R':
     {
-	int i = atoi(ascurl.c_str() + 1) - 1;
+	int i = atoi(strurl.c_str() + 1) - 1;
 	QString s = url.toString();
 	int bar = s.indexOf("|");
 	if (bar == -1 || bar >= s.size()-1)
@@ -949,31 +957,32 @@ void ResList::linkWasClicked(const QUrl &url)
 	// Spelling: replacement suggestion clicked
     case 'S':
     {
-	QString s = url.toString();
+        string s;
-	if (!s.isEmpty())
+	if (!strurl.empty())
-	    s = s.right(s.size()-1);
+	    s = strurl.substr(1);
-	int bar = s.indexOf("|");
+        string::size_type bar = s.find_first_of("|");
-	if (bar != -1 && bar < s.size()-1) {
+	if (bar != string::npos && bar < s.size() - 1) {
-	    QString o = s.left(bar);
+	    string o = s.substr(0, bar);
-	    QString n = s.right(s.size() - (bar+1));
+	    string n = s.substr(bar+1);
-	    emit wordReplace(o, n);
+            LOGDEB2("Emitting wordreplace " << o << " -> " << n << std::endl);
+	    emit wordReplace(u8s2qs(o), u8s2qs(n));
 	}
     }
     break;
 
     default: 
-	LOGERR("ResList::linkWasClicked: bad link ["  << (ascurl) << "]\n" );
+	LOGERR("ResList::linkWasClicked: bad link [" << strurl << "]\n");
 	break;// ?? 
     }
 }
 
 void ResList::createPopupMenu(const QPoint& pos)
 {
-    LOGDEB("ResList::createPopupMenu("  << (pos.x()) << ", "  << (pos.y()) << ")\n" );
+    LOGDEB("ResList::createPopupMenu(" << pos.x() << ", " << pos.y() << ")\n");
 #ifdef RESLIST_TEXTBROWSER
     QTextCursor cursor = cursorForPosition(pos);
     int blocknum = cursor.blockNumber();
-    LOGDEB("ResList::createPopupMenu(): block "  << (blocknum) << "\n" );
+    LOGDEB("ResList::createPopupMenu(): block " << blocknum << "\n");
     m_popDoc = docnumfromparnum(blocknum);
 #else
     QWebHitTestResult htr = page()->mainFrame()->hitTestContent(pos);

src/query/reslistpager.cpp

@@ -397,15 +397,13 @@ void ResListPager::displayPage(RclConfig *config)
 		    
 		    }
 
-		    for (map<string, vector<string> >::const_iterator it0 =
+		    for (const auto& entry: spellings) {
-			     spellings.begin(); it0 != spellings.end(); it0++) {
+			chunk << "<b>" << entry.first << "</b> : ";
-			chunk << "<b>" << it0->first << "</b> : ";
+                        for (const auto& spelling : entry.second) {
-			for (vector<string>::const_iterator it = 
+			    chunk << spelling << " ";
-				 it0->second.begin();
-			     it != it0->second.end(); it++) {
-			    chunk << *it << " ";
 			}
 			chunk << "<br />";
+                        std::cerr << chunk.str() << endl;
 		    }
 		    chunk << "</blockquote></p>";
 		}

src/rcldb/rcldb.cpp

@@ -18,6 +18,7 @@
 
 #include <stdio.h>
 #include <cstring>
+#include <exception>
 #include "safeunistd.h"
 #include <math.h>
 #include <time.h>
@@ -792,7 +793,20 @@ bool Db::open(OpenMode mode, OpenError *error)
 	    {
 		int action = (mode == DbUpd) ? Xapian::DB_CREATE_OR_OPEN :
 		    Xapian::DB_CREATE_OR_OVERWRITE;
+                if (::access(dir.c_str(), 0) != 0) {
+                    // New index. use a stub to force using Chert
+                    string stub = path_cat(m_config->getConfDir(),
+                                           "xapian.stub");
+                    FILE *fp = fopen(stub.c_str(), "w");
+                    if (nullptr == fp) {
+                        throw(string("Can't create ") + stub);
+                    }
+                    fprintf(fp, "chert %s\n", dir.c_str());
+                    fclose(fp);
+                    m_ndb->xwdb = Xapian::WritableDatabase(stub, action);
+                } else {
                     m_ndb->xwdb = Xapian::WritableDatabase(dir, action);
+                }
                 // If db is empty, write the data format version at once
                 // to avoid stupid error messages:
                 if (m_ndb->xwdb.get_doccount() == 0)