From 643f4d56bb9596794a84b418f6be592e615c2864 Mon Sep 17 00:00:00 2001 From: Jean-Francois Dockes Date: Tue, 5 Jun 2012 07:16:11 +0200 Subject: [PATCH] internals: virtualized the doc fetcher interface --- src/index/bglfetcher.cpp | 70 ++++++++++++++++++++ src/index/bglfetcher.h | 30 +++++++++ src/index/fetcher.cpp | 43 ++++++++++++ src/index/fetcher.h | 71 ++++++++++++++++++++ src/index/fsfetcher.cpp | 70 ++++++++++++++++++++ src/index/fsfetcher.h | 34 ++++++++++ src/internfile/internfile.cpp | 120 ++++++++-------------------------- src/internfile/internfile.h | 2 +- src/lib/Makefile | 46 +++++++++---- src/lib/mkMake | 7 +- src/qtgui/rclmain_w.cpp | 2 +- website/index.html.en | 6 +- 12 files changed, 389 insertions(+), 112 deletions(-) create mode 100644 src/index/bglfetcher.cpp create mode 100644 src/index/bglfetcher.h create mode 100644 src/index/fetcher.cpp create mode 100644 src/index/fetcher.h create mode 100644 src/index/fsfetcher.cpp create mode 100644 src/index/fsfetcher.h diff --git a/src/index/bglfetcher.cpp b/src/index/bglfetcher.cpp new file mode 100644 index 00000000..5ded6e91 --- /dev/null +++ b/src/index/bglfetcher.cpp @@ -0,0 +1,70 @@ +/* Copyright (C) 2012 J.F.Dockes + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#ifdef HAVE_CONFIG_H +#include "autoconfig.h" +#endif + +#include +#include + +#include "debuglog.h" +#include "rcldoc.h" + +#include "fetcher.h" +#include "bglfetcher.h" +#include "debuglog.h" +#include "ptmutex.h" +#include "beaglequeuecache.h" + +// We use a single beagle cache object to access beagle data. We protect it +// against multiple thread access. +static PTMutexInit o_beagler_mutex; + +bool BGLDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out) +{ + string udi; + if (!idoc.getmeta(Rcl::Doc::keyudi, &udi) || udi.empty()) { + LOGERR(("BGLDocFetcher:: no udi in idoc\n")); + return false; + } + string data; + Rcl::Doc dotdoc; + { + PTMutexLocker locker(o_beagler_mutex); + // Retrieve from our webcache (beagle data). The beagler + // object is created at the first call of this routine and + // deleted when the program exits. + static BeagleQueueCache o_beagler(cnf); + if (!o_beagler.getFromCache(udi, dotdoc, data)) { + LOGINFO(("BGLDocFetcher::fetch: failed for [%s]\n", udi.c_str())); + return false; + } + } + if (dotdoc.mimetype.compare(idoc.mimetype)) { + LOGINFO(("BGLDocFetcher:: udi [%s], mimetp mismatch: in: [%s], bgl " + "[%s]\n", idoc.mimetype.c_str(), dotdoc.mimetype.c_str())); + } + return true; +} + +bool BGLDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig) +{ + // Bgl sigs are empty + sig.clear(); + return true; +} + diff --git a/src/index/bglfetcher.h b/src/index/bglfetcher.h new file mode 100644 index 00000000..0690ef66 --- /dev/null +++ b/src/index/bglfetcher.h @@ -0,0 +1,30 @@ +/* Copyright (C) 2012 J.F.Dockes + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#ifndef _BGLFETCHER_H_INCLUDED_ +#define _BGLFETCHER_H_INCLUDED_ +#include "fetcher.h" + +/** + * The Beagle cache fetcher: + */ +class BGLDocFetcher : public DocFetcher{ + virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out); + virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig); + virtual ~BGLDocFetcher() {} +}; + +#endif /* _BGLFETCHER_H_INCLUDED_ */ diff --git a/src/index/fetcher.cpp b/src/index/fetcher.cpp new file mode 100644 index 00000000..3f3a0723 --- /dev/null +++ b/src/index/fetcher.cpp @@ -0,0 +1,43 @@ +/* Copyright (C) 2012 J.F.Dockes + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#ifdef HAVE_CONFIG_H +#include "autoconfig.h" +#endif + +#include "debuglog.h" + +#include "fetcher.h" +#include "fsfetcher.h" +#include "bglfetcher.h" + +DocFetcher *docFetcherMake(const Rcl::Doc& idoc) +{ + if (idoc.url.empty()) { + LOGERR(("docFetcherMakeg:: no url in doc!\n")); + return false; + } + string backend; + idoc.getmeta(Rcl::Doc::keybcknd, &backend); + if (backend.empty() || !backend.compare("FS")) { + return new FSDocFetcher; + } else if (!backend.compare("BGL")) { + return new BGLDocFetcher; + } else { + LOGERR(("DocFetcherFactory: unknown backend [%s]\n", backend.c_str())); + return 0; + } +} diff --git a/src/index/fetcher.h b/src/index/fetcher.h new file mode 100644 index 00000000..69e158f1 --- /dev/null +++ b/src/index/fetcher.h @@ -0,0 +1,71 @@ +/* Copyright (C) 2012 J.F.Dockes + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#ifndef _FETCHER_H_INCLUDED_ +#define _FETCHER_H_INCLUDED_ + +#include +#include + +#include "rcldoc.h" + +class RclConfig; + +/** + * Definition for a generic method to retrieve the data + * for a document designated by its index data (udi/ipath/url). + * This is used to retrieve the data for previewing. The + * actual implementation is specific to the kind of backend (file + * system, beagle cache, others in the future ?), and of course may + * share code with the indexing-time functions from the specific backend. + */ +class DocFetcher { +public: + /** A RawDoc is the data for a source document either as a memory + block, or pointed to by a file name */ + struct RawDoc { + enum RawDocKind {RDK_FILENAME, RDK_DATA}; + RawDocKind kind; + std::string data; // Doc data or file name + struct stat st; // Only used if RDK_FILENAME + }; + + /** + * Return the data for the requested document, either as a + * file-system file or as a memory object (maybe stream too in the + * future?) + * @param cnf the global config + * @param idoc the data gathered from the index for this doc (udi/ipath) + * @param out we may return either a file name or the document data. + */ + virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out) = 0; + + /** + * Return the signature for the requested document. This is used for + * up-to-date tests performed out of indexing (e.g.: verifying that a + * document is not stale before previewing it). + * @param cnf the global config + * @param idoc the data gathered from the index for this doc (udi/ipath) + * @param sig output. + */ + virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig) = 0; + virtual ~DocFetcher() {} +}; + +/** Returns an appropriate fetcher object given the backend string identifier */ +DocFetcher *docFetcherMake(const Rcl::Doc& idoc); + +#endif /* _FETCHER_H_INCLUDED_ */ diff --git a/src/index/fsfetcher.cpp b/src/index/fsfetcher.cpp new file mode 100644 index 00000000..cc809d86 --- /dev/null +++ b/src/index/fsfetcher.cpp @@ -0,0 +1,70 @@ +/* Copyright (C) 2012 J.F.Dockes + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#ifdef HAVE_CONFIG_H +#include "autoconfig.h" +#endif + +#include +#include + +#include "debuglog.h" +#include "cstr.h" + +#include "fetcher.h" +#include "fsfetcher.h" +#include "fsindexer.h" +#include "debuglog.h" + +using std::string; + +static bool urltopath(const Rcl::Doc& idoc, string& fn, struct stat& st) +{ + // The url has to be like file:// + if (idoc.url.find(cstr_fileu) != 0) { + LOGERR(("FSDocFetcher::fetch/sig: non fs url: [%s]\n", + idoc.url.c_str())); + return false; + } + fn = idoc.url.substr(7, string::npos); + if (stat(fn.c_str(), &st) < 0) { + LOGERR(("FSDocFetcher::fetch: stat errno %d for [%s]\n", + errno, fn.c_str())); + return false; + } + return true; +} + +bool FSDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out) +{ + string fn; + if (!urltopath(idoc, fn, out.st)) + return false; + out.kind = RawDoc::RDK_FILENAME; + out.data = fn; + return true; +} + +bool FSDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig) +{ + string fn; + struct stat st; + if (!urltopath(idoc, fn, st)) + return false; + FsIndexer::makesig(&st, sig); + return true; +} + diff --git a/src/index/fsfetcher.h b/src/index/fsfetcher.h new file mode 100644 index 00000000..8a6ed7e6 --- /dev/null +++ b/src/index/fsfetcher.h @@ -0,0 +1,34 @@ +/* Copyright (C) 2012 J.F.Dockes + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#ifndef _FSFETCHER_H_INCLUDED_ +#define _FSFETCHER_H_INCLUDED_ + +#include "fetcher.h" + +/** + * The file-system fetcher: + */ +class FSDocFetcher : public DocFetcher{ + /** FSDocFetcher::fetch always returns a file name */ + virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out); + + /** Calls stat to retrieve file signature data */ + virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig); + virtual ~FSDocFetcher() {} +}; + +#endif /* _FSFETCHER_H_INCLUDED_ */ diff --git a/src/internfile/internfile.cpp b/src/internfile/internfile.cpp index 5ec50022..5571c18c 100644 --- a/src/internfile/internfile.cpp +++ b/src/internfile/internfile.cpp @@ -43,10 +43,9 @@ using namespace std; #include "rclconfig.h" #include "mh_html.h" #include "fileudi.h" -#include "beaglequeuecache.h" #include "cancelcheck.h" #include "copyfile.h" -#include "ptmutex.h" +#include "fetcher.h" #ifdef RCL_USE_XATTR #include "pxattr.h" @@ -381,10 +380,6 @@ void FileInterner::initcommon(RclConfig *cnf, int flags) m_targetMType = cstr_textplain; } -// We use a single beagle cache object to access beagle data. We protect it -// against multiple thread access. -static PTMutexInit o_beagler_mutex; - FileInterner::FileInterner(const Rcl::Doc& idoc, RclConfig *cnf, TempDir& td, int flags) : m_tdir(td), m_ok(false), m_missingdatap(0) @@ -392,101 +387,40 @@ FileInterner::FileInterner(const Rcl::Doc& idoc, RclConfig *cnf, LOGDEB(("FileInterner::FileInterner(idoc)\n")); initcommon(cnf, flags); - // We do insist on having an url... - if (idoc.url.empty()) { - LOGERR(("FileInterner::FileInterner:: no url!\n")); + DocFetcher *fetcher = docFetcherMake(idoc); + if (fetcher == 0) { + LOGERR(("FileInterner:: no backend\n")); return; } - - // This stuff will be moved to some kind of generic function: - // get(idoc, ofn, odata, ometa) - // and use some kind of backstore object factory next time we add a - // backend (if ever). - string backend; - idoc.getmeta(Rcl::Doc::keybcknd, &backend); - - if (backend.empty() || !backend.compare("FS")) { - // Filesystem document. Intern from file. - // The url has to be like file:// - if (idoc.url.find(cstr_fileu) != 0) { - LOGERR(("FileInterner: FS backend and non fs url: [%s]\n", - idoc.url.c_str())); - return; - } - string fn = idoc.url.substr(7, string::npos); - struct stat st; - if (stat(fn.c_str(), &st) < 0) { - LOGERR(("FileInterner:: cannot access document file: [%s]\n", - fn.c_str())); - return; - } - init(fn, &st, cnf, flags, &idoc.mimetype); - } else if (!backend.compare("BGL")) { - string udi; - if (!idoc.getmeta(Rcl::Doc::keyudi, &udi) || udi.empty()) { - LOGERR(("FileInterner:: no udi in idoc\n")); - return; - } - - string data; - Rcl::Doc dotdoc; - { - PTMutexLocker locker(o_beagler_mutex); - // Retrieve from our webcache (beagle data). The beagler - // object is created at the first call of this routine and - // deleted when the program exits. - static BeagleQueueCache o_beagler(cnf); - if (!o_beagler.getFromCache(udi, dotdoc, data)) { - LOGINFO(("FileInterner:: failed fetch from Beagle cache for [%s]\n", - udi.c_str())); - return; - } - } - if (dotdoc.mimetype.compare(idoc.mimetype)) { - LOGINFO(("FileInterner:: udi [%s], mimetp mismatch: in: [%s], bgl " - "[%s]\n", idoc.mimetype.c_str(), dotdoc.mimetype.c_str())); - } - init(data, cnf, flags, dotdoc.mimetype); - } else { - LOGERR(("FileInterner:: unknown backend: [%s]\n", backend.c_str())); - return; + DocFetcher::RawDoc rawdoc; + string data; + if (!fetcher->fetch(cnf, idoc, rawdoc)) { + LOGERR(("FileInterner:: fetcher failed\n")); + return; } + switch (rawdoc.kind) { + case DocFetcher::RawDoc::RDK_FILENAME: + init(rawdoc.data, &rawdoc.st, cnf, flags, &idoc.mimetype); + break; + case DocFetcher::RawDoc::RDK_DATA: + init(data, cnf, flags, idoc.mimetype); + break; + } + return; } -#include "fsindexer.h" -bool FileInterner::makesig(const Rcl::Doc& idoc, string& sig) +bool FileInterner::makesig(RclConfig *cnf, const Rcl::Doc& idoc, string& sig) { - if (idoc.url.empty()) { - LOGERR(("FileInterner::makesig:: no url!\n")); + DocFetcher *fetcher = docFetcherMake(idoc); + if (fetcher == 0) { + LOGERR(("FileInterner::makesig no backend for doc\n")); return false; } - string backend; - idoc.getmeta(Rcl::Doc::keybcknd, &backend); - - if (backend.empty() || !backend.compare("FS")) { - if (idoc.url.find(cstr_fileu) != 0) { - LOGERR(("FileInterner: FS backend and non fs url: [%s]\n", - idoc.url.c_str())); - return false; - } - string fn = idoc.url.substr(7, string::npos); - struct stat st; - if (stat(fn.c_str(), &st) < 0) { - LOGERR(("FileInterner:: cannot access document file: [%s]\n", - fn.c_str())); - return false; - } - FsIndexer::makesig(&st, sig); - return true; - } else if (!backend.compare("BGL")) { - // Bgl sigs are empty - sig.clear(); - return true; - } else { - LOGERR(("FileInterner:: unknown backend: [%s]\n", backend.c_str())); - return false; - } - return false; + + bool ret = fetcher->makesig(cnf, idoc, sig); + + delete fetcher; + return ret; } FileInterner::~FileInterner() diff --git a/src/internfile/internfile.h b/src/internfile/internfile.h index 1eabb931..b97c2a63 100644 --- a/src/internfile/internfile.h +++ b/src/internfile/internfile.h @@ -124,7 +124,7 @@ class FileInterner { /** * Build sig for doc coming from rcldb. This is here because we know how * to query the right backend */ - static bool makesig(const Rcl::Doc& idoc, string& sig); + static bool makesig(RclConfig *cnf, const Rcl::Doc& idoc, string& sig); ~FileInterner(); diff --git a/src/lib/Makefile b/src/lib/Makefile index 25f3f1e3..67126dae 100644 --- a/src/lib/Makefile +++ b/src/lib/Makefile @@ -6,8 +6,8 @@ LIBS = librcl.a all: $(LIBS) -OBJS = rclaspell.o beaglequeuecache.o cstr.o rclconfig.o rclinit.o textsplit.o unacpp.o beaglequeue.o fsindexer.o indexer.o mimetype.o subtreelist.o htmlparse.o myhtmlparse.o mimehandler.o internfile.o mh_exec.o mh_execm.o mh_html.o mh_mail.o mh_mbox.o mh_text.o txtdcode.o docseq.o docseqdb.o docseqhist.o filtseq.o dynconf.o plaintorich.o recollq.o reslistpager.o sortseq.o wasastringtoquery.o wasatorcl.o rcldb.o rcldoc.o rclquery.o searchdata.o stemdb.o stoplist.o unac.o base64.o circache.o closefrom.o conftree.o copyfile.o debuglog.o ecrontab.o execmd.o fstreewalk.o idfile.o fileudi.o md5.o mimeparse.o netcon.o pathut.o pxattr.o rclionice.o readfile.o smallut.o transcode.o wipedir.o x11mon.o mime-parsefull.o mime-parseonlyheader.o mime-printbody.o mime.o convert.o iodevice.o iofactory.o -DEPS = rclaspell.dep.stamp beaglequeuecache.dep.stamp cstr.dep.stamp rclconfig.dep.stamp rclinit.dep.stamp textsplit.dep.stamp unacpp.dep.stamp beaglequeue.dep.stamp fsindexer.dep.stamp indexer.dep.stamp mimetype.dep.stamp subtreelist.dep.stamp htmlparse.dep.stamp myhtmlparse.dep.stamp mimehandler.dep.stamp internfile.dep.stamp mh_exec.dep.stamp mh_execm.dep.stamp mh_html.dep.stamp mh_mail.dep.stamp mh_mbox.dep.stamp mh_text.dep.stamp txtdcode.dep.stamp docseq.dep.stamp docseqdb.dep.stamp docseqhist.dep.stamp filtseq.dep.stamp dynconf.dep.stamp plaintorich.dep.stamp recollq.dep.stamp reslistpager.dep.stamp sortseq.dep.stamp wasastringtoquery.dep.stamp wasatorcl.dep.stamp rcldb.dep.stamp rcldoc.dep.stamp rclquery.dep.stamp searchdata.dep.stamp stemdb.dep.stamp stoplist.dep.stamp unac.dep.stamp base64.dep.stamp circache.dep.stamp closefrom.dep.stamp conftree.dep.stamp copyfile.dep.stamp debuglog.dep.stamp ecrontab.dep.stamp execmd.dep.stamp fstreewalk.dep.stamp idfile.dep.stamp fileudi.dep.stamp md5.dep.stamp mimeparse.dep.stamp netcon.dep.stamp pathut.dep.stamp pxattr.dep.stamp rclionice.dep.stamp readfile.dep.stamp smallut.dep.stamp transcode.dep.stamp wipedir.dep.stamp x11mon.dep.stamp mime-parsefull.dep.stamp mime-parseonlyheader.dep.stamp mime-printbody.dep.stamp mime.dep.stamp convert.dep.stamp iodevice.dep.stamp iofactory.dep.stamp +OBJS = rclaspell.o beaglequeuecache.o cstr.o rclconfig.o rclinit.o textsplit.o unacpp.o beaglequeue.o bglfetcher.o fetcher.o fsfetcher.o fsindexer.o indexer.o mimetype.o subtreelist.o htmlparse.o internfile.o mh_exec.o mh_execm.o mh_html.o mh_mail.o mh_mbox.o mh_text.o mimehandler.o myhtmlparse.o txtdcode.o docseq.o docseqdb.o docseqhist.o filtseq.o dynconf.o plaintorich.o recollq.o reslistpager.o sortseq.o wasastringtoquery.o wasatorcl.o rcldb.o rcldoc.o rclquery.o searchdata.o stemdb.o stoplist.o unac.o base64.o circache.o closefrom.o conftree.o copyfile.o debuglog.o ecrontab.o execmd.o fstreewalk.o idfile.o fileudi.o md5.o mimeparse.o netcon.o pathut.o pxattr.o rclionice.o readfile.o smallut.o transcode.o wipedir.o x11mon.o mime-parsefull.o mime-parseonlyheader.o mime-printbody.o mime.o convert.o iodevice.o iofactory.o +DEPS = rclaspell.dep.stamp beaglequeuecache.dep.stamp cstr.dep.stamp rclconfig.dep.stamp rclinit.dep.stamp textsplit.dep.stamp unacpp.dep.stamp beaglequeue.dep.stamp bglfetcher.dep.stamp fetcher.dep.stamp fsfetcher.dep.stamp fsindexer.dep.stamp indexer.dep.stamp mimetype.dep.stamp subtreelist.dep.stamp htmlparse.dep.stamp internfile.dep.stamp mh_exec.dep.stamp mh_execm.dep.stamp mh_html.dep.stamp mh_mail.dep.stamp mh_mbox.dep.stamp mh_text.dep.stamp mimehandler.dep.stamp myhtmlparse.dep.stamp txtdcode.dep.stamp docseq.dep.stamp docseqdb.dep.stamp docseqhist.dep.stamp filtseq.dep.stamp dynconf.dep.stamp plaintorich.dep.stamp recollq.dep.stamp reslistpager.dep.stamp sortseq.dep.stamp wasastringtoquery.dep.stamp wasatorcl.dep.stamp rcldb.dep.stamp rcldoc.dep.stamp rclquery.dep.stamp searchdata.dep.stamp stemdb.dep.stamp stoplist.dep.stamp unac.dep.stamp base64.dep.stamp circache.dep.stamp closefrom.dep.stamp conftree.dep.stamp copyfile.dep.stamp debuglog.dep.stamp ecrontab.dep.stamp execmd.dep.stamp fstreewalk.dep.stamp idfile.dep.stamp fileudi.dep.stamp md5.dep.stamp mimeparse.dep.stamp netcon.dep.stamp pathut.dep.stamp pxattr.dep.stamp rclionice.dep.stamp readfile.dep.stamp smallut.dep.stamp transcode.dep.stamp wipedir.dep.stamp x11mon.dep.stamp mime-parsefull.dep.stamp mime-parseonlyheader.dep.stamp mime-printbody.dep.stamp mime.dep.stamp convert.dep.stamp iodevice.dep.stamp iofactory.dep.stamp librcl.a : $(DEPS) $(OBJS) ar ru librcl.a $(OBJS) @@ -29,6 +29,12 @@ unacpp.o : ../common/unacpp.cpp $(depth)/mk/localdefs $(CXX) $(ALL_CXXFLAGS) -c ../common/unacpp.cpp beaglequeue.o : ../index/beaglequeue.cpp $(depth)/mk/localdefs $(CXX) $(ALL_CXXFLAGS) -c ../index/beaglequeue.cpp +bglfetcher.o : ../index/bglfetcher.cpp $(depth)/mk/localdefs + $(CXX) $(ALL_CXXFLAGS) -c ../index/bglfetcher.cpp +fetcher.o : ../index/fetcher.cpp $(depth)/mk/localdefs + $(CXX) $(ALL_CXXFLAGS) -c ../index/fetcher.cpp +fsfetcher.o : ../index/fsfetcher.cpp $(depth)/mk/localdefs + $(CXX) $(ALL_CXXFLAGS) -c ../index/fsfetcher.cpp fsindexer.o : ../index/fsindexer.cpp $(depth)/mk/localdefs $(CXX) $(ALL_CXXFLAGS) -c ../index/fsindexer.cpp indexer.o : ../index/indexer.cpp $(depth)/mk/localdefs @@ -39,10 +45,6 @@ subtreelist.o : ../index/subtreelist.cpp $(depth)/mk/localdefs $(CXX) $(ALL_CXXFLAGS) -c ../index/subtreelist.cpp htmlparse.o : ../internfile/htmlparse.cpp $(depth)/mk/localdefs $(CXX) $(ALL_CXXFLAGS) -c ../internfile/htmlparse.cpp -myhtmlparse.o : ../internfile/myhtmlparse.cpp $(depth)/mk/localdefs - $(CXX) $(ALL_CXXFLAGS) -c ../internfile/myhtmlparse.cpp -mimehandler.o : ../internfile/mimehandler.cpp $(depth)/mk/localdefs - $(CXX) $(ALL_CXXFLAGS) -c ../internfile/mimehandler.cpp internfile.o : ../internfile/internfile.cpp $(depth)/mk/localdefs $(CXX) $(ALL_CXXFLAGS) -c ../internfile/internfile.cpp mh_exec.o : ../internfile/mh_exec.cpp $(depth)/mk/localdefs @@ -57,6 +59,10 @@ mh_mbox.o : ../internfile/mh_mbox.cpp $(depth)/mk/localdefs $(CXX) $(ALL_CXXFLAGS) -c ../internfile/mh_mbox.cpp mh_text.o : ../internfile/mh_text.cpp $(depth)/mk/localdefs $(CXX) $(ALL_CXXFLAGS) -c ../internfile/mh_text.cpp +mimehandler.o : ../internfile/mimehandler.cpp $(depth)/mk/localdefs + $(CXX) $(ALL_CXXFLAGS) -c ../internfile/mimehandler.cpp +myhtmlparse.o : ../internfile/myhtmlparse.cpp $(depth)/mk/localdefs + $(CXX) $(ALL_CXXFLAGS) -c ../internfile/myhtmlparse.cpp txtdcode.o : ../internfile/txtdcode.cpp $(depth)/mk/localdefs $(CXX) $(ALL_CXXFLAGS) -c ../internfile/txtdcode.cpp docseq.o : ../query/docseq.cpp $(depth)/mk/localdefs @@ -183,6 +189,15 @@ unacpp.dep.stamp : ../common/unacpp.cpp $(depth)/mk/localdefs beaglequeue.dep.stamp : ../index/beaglequeue.cpp $(depth)/mk/localdefs $(CXX) -M $(ALL_CXXFLAGS) ../index/beaglequeue.cpp > beaglequeue.dep touch beaglequeue.dep.stamp +bglfetcher.dep.stamp : ../index/bglfetcher.cpp $(depth)/mk/localdefs + $(CXX) -M $(ALL_CXXFLAGS) ../index/bglfetcher.cpp > bglfetcher.dep + touch bglfetcher.dep.stamp +fetcher.dep.stamp : ../index/fetcher.cpp $(depth)/mk/localdefs + $(CXX) -M $(ALL_CXXFLAGS) ../index/fetcher.cpp > fetcher.dep + touch fetcher.dep.stamp +fsfetcher.dep.stamp : ../index/fsfetcher.cpp $(depth)/mk/localdefs + $(CXX) -M $(ALL_CXXFLAGS) ../index/fsfetcher.cpp > fsfetcher.dep + touch fsfetcher.dep.stamp fsindexer.dep.stamp : ../index/fsindexer.cpp $(depth)/mk/localdefs $(CXX) -M $(ALL_CXXFLAGS) ../index/fsindexer.cpp > fsindexer.dep touch fsindexer.dep.stamp @@ -198,12 +213,6 @@ subtreelist.dep.stamp : ../index/subtreelist.cpp $(depth)/mk/localdefs htmlparse.dep.stamp : ../internfile/htmlparse.cpp $(depth)/mk/localdefs $(CXX) -M $(ALL_CXXFLAGS) ../internfile/htmlparse.cpp > htmlparse.dep touch htmlparse.dep.stamp -myhtmlparse.dep.stamp : ../internfile/myhtmlparse.cpp $(depth)/mk/localdefs - $(CXX) -M $(ALL_CXXFLAGS) ../internfile/myhtmlparse.cpp > myhtmlparse.dep - touch myhtmlparse.dep.stamp -mimehandler.dep.stamp : ../internfile/mimehandler.cpp $(depth)/mk/localdefs - $(CXX) -M $(ALL_CXXFLAGS) ../internfile/mimehandler.cpp > mimehandler.dep - touch mimehandler.dep.stamp internfile.dep.stamp : ../internfile/internfile.cpp $(depth)/mk/localdefs $(CXX) -M $(ALL_CXXFLAGS) ../internfile/internfile.cpp > internfile.dep touch internfile.dep.stamp @@ -225,6 +234,12 @@ mh_mbox.dep.stamp : ../internfile/mh_mbox.cpp $(depth)/mk/localdefs mh_text.dep.stamp : ../internfile/mh_text.cpp $(depth)/mk/localdefs $(CXX) -M $(ALL_CXXFLAGS) ../internfile/mh_text.cpp > mh_text.dep touch mh_text.dep.stamp +mimehandler.dep.stamp : ../internfile/mimehandler.cpp $(depth)/mk/localdefs + $(CXX) -M $(ALL_CXXFLAGS) ../internfile/mimehandler.cpp > mimehandler.dep + touch mimehandler.dep.stamp +myhtmlparse.dep.stamp : ../internfile/myhtmlparse.cpp $(depth)/mk/localdefs + $(CXX) -M $(ALL_CXXFLAGS) ../internfile/myhtmlparse.cpp > myhtmlparse.dep + touch myhtmlparse.dep.stamp txtdcode.dep.stamp : ../internfile/txtdcode.cpp $(depth)/mk/localdefs $(CXX) -M $(ALL_CXXFLAGS) ../internfile/txtdcode.cpp > txtdcode.dep touch txtdcode.dep.stamp @@ -356,13 +371,14 @@ include rclinit.dep include textsplit.dep include unacpp.dep include beaglequeue.dep +include bglfetcher.dep +include fetcher.dep +include fsfetcher.dep include fsindexer.dep include indexer.dep include mimetype.dep include subtreelist.dep include htmlparse.dep -include myhtmlparse.dep -include mimehandler.dep include internfile.dep include mh_exec.dep include mh_execm.dep @@ -370,6 +386,8 @@ include mh_html.dep include mh_mail.dep include mh_mbox.dep include mh_text.dep +include mimehandler.dep +include myhtmlparse.dep include txtdcode.dep include docseq.dep include docseqdb.dep diff --git a/src/lib/mkMake b/src/lib/mkMake index 7724be2e..f3d0a5e9 100755 --- a/src/lib/mkMake +++ b/src/lib/mkMake @@ -12,13 +12,14 @@ ${depth}/common/rclinit.cpp \ ${depth}/common/textsplit.cpp \ ${depth}/common/unacpp.cpp \ ${depth}/index/beaglequeue.cpp \ +${depth}/index/bglfetcher.cpp \ +${depth}/index/fetcher.cpp \ +${depth}/index/fsfetcher.cpp \ ${depth}/index/fsindexer.cpp \ ${depth}/index/indexer.cpp \ ${depth}/index/mimetype.cpp \ ${depth}/index/subtreelist.cpp \ ${depth}/internfile/htmlparse.cpp \ -${depth}/internfile/myhtmlparse.cpp \ -${depth}/internfile/mimehandler.cpp \ ${depth}/internfile/internfile.cpp \ ${depth}/internfile/mh_exec.cpp \ ${depth}/internfile/mh_execm.cpp \ @@ -26,6 +27,8 @@ ${depth}/internfile/mh_html.cpp \ ${depth}/internfile/mh_mail.cpp \ ${depth}/internfile/mh_mbox.cpp \ ${depth}/internfile/mh_text.cpp \ +${depth}/internfile/mimehandler.cpp \ +${depth}/internfile/myhtmlparse.cpp \ ${depth}/internfile/txtdcode.cpp \ ${depth}/query/docseq.cpp \ ${depth}/query/docseqdb.cpp \ diff --git a/src/qtgui/rclmain_w.cpp b/src/qtgui/rclmain_w.cpp index 390e023b..e3b85f25 100644 --- a/src/qtgui/rclmain_w.cpp +++ b/src/qtgui/rclmain_w.cpp @@ -1189,7 +1189,7 @@ void RclMain::startPreview(int docnum, Rcl::Doc doc, int mod) doc.getmeta(Rcl::Doc::keyudi, &udi); if (!udi.empty()) { string sig; - FileInterner::makesig(doc, sig); + FileInterner::makesig(theconfig, doc, sig); if (rcldb->needUpdate(udi, sig)) { int rep = QMessageBox::warning(0, tr("Warning"), diff --git a/website/index.html.en b/website/index.html.en index 85dff8c6..22dcf584 100644 --- a/website/index.html.en +++ b/website/index.html.en @@ -11,7 +11,6 @@ - @@ -87,6 +86,11 @@

News

    +
  • 2012-06-01: an + updated filter for the + OpenDocument format will properly handle exported Google Docs + files.
    • +
  • 2012-05-25: a new filter for indexing tar archives.