Removed most ancient references to Beagle in the code, replaced by WebQueue or such to clarify things. No real changes

This commit is contained in:
Jean-Francois Dockes 2018-05-31 15:51:58 +02:00
parent be3ac2e180
commit 9f94b9ed20
12 changed files with 118 additions and 114 deletions

View File

@ -58,8 +58,8 @@ bincimapmime/mime-printbody.cc \
bincimapmime/mime-utils.h \ bincimapmime/mime-utils.h \
bincimapmime/mime.cc \ bincimapmime/mime.cc \
bincimapmime/mime.h \ bincimapmime/mime.h \
common/beaglequeuecache.cpp \ common/webstore.cpp \
common/beaglequeuecache.h \ common/webstore.h \
common/conf_post.h \ common/conf_post.h \
common/cstr.cpp \ common/cstr.cpp \
common/cstr.h \ common/cstr.h \
@ -76,10 +76,10 @@ common/unacpp.h \
common/uproplist.h \ common/uproplist.h \
common/utf8fn.cpp \ common/utf8fn.cpp \
common/utf8fn.h \ common/utf8fn.h \
index/beaglequeue.cpp \ index/webqueue.cpp \
index/beaglequeue.h \ index/webqueue.h \
index/bglfetcher.cpp \ index/webqueuefetcher.cpp \
index/bglfetcher.h \ index/webqueuefetcher.h \
index/checkretryfailed.cpp \ index/checkretryfailed.cpp \
index/checkretryfailed.h \ index/checkretryfailed.h \
index/exefetcher.cpp \ index/exefetcher.cpp \

View File

@ -17,10 +17,11 @@
#include "autoconfig.h" #include "autoconfig.h"
#include "webstore.h"
#include <stdint.h> #include <stdint.h>
#include "cstr.h" #include "cstr.h"
#include "beaglequeuecache.h"
#include "circache.h" #include "circache.h"
#include "log.h" #include "log.h"
#include "rclconfig.h" #include "rclconfig.h"
@ -29,42 +30,43 @@
const string cstr_bgc_mimetype("mimetype"); const string cstr_bgc_mimetype("mimetype");
BeagleQueueCache::BeagleQueueCache(RclConfig *cnf) WebStore::WebStore(RclConfig *cnf)
{ {
string ccdir = cnf->getWebcacheDir(); string ccdir = cnf->getWebcacheDir();
int maxmbs = 40; int maxmbs = 40;
cnf->getConfParam("webcachemaxmbs", &maxmbs); cnf->getConfParam("webcachemaxmbs", &maxmbs);
if ((m_cache = new CirCache(ccdir)) == 0) { if ((m_cache = new CirCache(ccdir)) == 0) {
LOGERR("BeagleQueueCache: cant create CirCache object\n" ); LOGERR("WebStore: cant create CirCache object\n" );
return; return;
} }
if (!m_cache->create(int64_t(maxmbs)*1000*1024, CirCache::CC_CRUNIQUE)) { if (!m_cache->create(int64_t(maxmbs)*1000*1024, CirCache::CC_CRUNIQUE)) {
LOGERR("BeagleQueueCache: cache file creation failed: " << (m_cache->getReason()) << "\n" ); LOGERR("WebStore: cache file creation failed: " <<
m_cache->getReason() << "\n");
delete m_cache; delete m_cache;
m_cache = 0; m_cache = 0;
return; return;
} }
} }
BeagleQueueCache::~BeagleQueueCache() WebStore::~WebStore()
{ {
delete m_cache; delete m_cache;
} }
// Read document from cache. Return the metadata as an Rcl::Doc // Read document from cache. Return the metadata as an Rcl::Doc
// @param htt Beagle Hit Type // @param htt Web Hit Type
bool BeagleQueueCache::getFromCache(const string& udi, Rcl::Doc &dotdoc, bool WebStore::getFromCache(const string& udi, Rcl::Doc &dotdoc,
string& data, string *htt) string& data, string *htt)
{ {
string dict; string dict;
if (m_cache == 0) { if (m_cache == 0) {
LOGERR("BeagleQueueCache::getFromCache: cache is null\n" ); LOGERR("WebStore::getFromCache: cache is null\n");
return false; return false;
} }
if (!m_cache->get(udi, dict, &data)) { if (!m_cache->get(udi, dict, &data)) {
LOGDEB("BeagleQueueCache::getFromCache: get failed\n" ); LOGDEB("WebStore::getFromCache: get failed\n");
return false; return false;
} }

View File

@ -14,11 +14,10 @@
* Free Software Foundation, Inc., * Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/ */
#ifndef _beaglequeuecache_h_included_ #ifndef _webstore_h_included_
#define _beaglequeuecache_h_included_ #define _webstore_h_included_
#include <string> #include <string>
using std::string;
class RclConfig; class RclConfig;
namespace Rcl { namespace Rcl {
@ -28,23 +27,24 @@ namespace Rcl {
class CirCache; class CirCache;
/** /**
* Manage the CirCache for the Beagle Queue indexer. Separated from the main * Manage the CirCache for the Web Queue indexer. Separated from the main
* indexer code because it's also used for querying (getting the data for a * indexer code because it's also used for querying (getting the data for a
* preview * preview
*/ */
class BeagleQueueCache { class WebStore {
public: public:
BeagleQueueCache(RclConfig *config); WebStore(RclConfig *config);
~BeagleQueueCache(); ~WebStore();
bool getFromCache(const string& udi, Rcl::Doc &doc, string& data, bool getFromCache(const std::string& udi, Rcl::Doc &doc, std::string& data,
string *hittype = 0); std::string *hittype = 0);
// We could write proxies for all the circache ops, but why bother? // We could write proxies for all the circache ops, but why bother?
CirCache *cc() {return m_cache;} CirCache *cc() {return m_cache;}
private: private:
CirCache *m_cache; CirCache *m_cache;
}; };
extern const string cstr_bgc_mimetype;
#endif /* _beaglequeuecache_h_included_ */ extern const std::string cstr_bgc_mimetype;
#endif /* _webstore_h_included_ */

View File

@ -16,13 +16,11 @@
*/ */
#include "autoconfig.h" #include "autoconfig.h"
#include "log.h" #include "log.h"
#include "rclconfig.h" #include "rclconfig.h"
#include "fetcher.h" #include "fetcher.h"
#include "fsfetcher.h" #include "fsfetcher.h"
#include "bglfetcher.h" #include "webqueuefetcher.h"
#include "exefetcher.h" #include "exefetcher.h"
DocFetcher *docFetcherMake(RclConfig *config, const Rcl::Doc& idoc) DocFetcher *docFetcherMake(RclConfig *config, const Rcl::Doc& idoc)
@ -37,7 +35,7 @@ DocFetcher *docFetcherMake(RclConfig *config, const Rcl::Doc& idoc)
return new FSDocFetcher; return new FSDocFetcher;
#ifndef DISABLE_WEB_INDEXER #ifndef DISABLE_WEB_INDEXER
} else if (!backend.compare("BGL")) { } else if (!backend.compare("BGL")) {
return new BGLDocFetcher; return new WQDocFetcher;
#endif #endif
} else { } else {
DocFetcher *f = exeDocFetcherMake(config, backend); DocFetcher *f = exeDocFetcherMake(config, backend);

View File

@ -27,7 +27,7 @@
#include "indexer.h" #include "indexer.h"
#include "fsindexer.h" #include "fsindexer.h"
#ifndef DISABLE_WEB_INDEXER #ifndef DISABLE_WEB_INDEXER
#include "beaglequeue.h" #include "webqueue.h"
#endif #endif
#include "mimehandler.h" #include "mimehandler.h"
#include "pathut.h" #include "pathut.h"
@ -132,7 +132,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
if (m_doweb && (typestorun & IxTWebQueue)) { if (m_doweb && (typestorun & IxTWebQueue)) {
runWebFilesMoverScript(m_config); runWebFilesMoverScript(m_config);
deleteZ(m_webindexer); deleteZ(m_webindexer);
m_webindexer = new BeagleQueueIndexer(m_config, &m_db, m_updater); m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater);
if (!m_webindexer || !m_webindexer->index()) { if (!m_webindexer || !m_webindexer->index()) {
m_db.close(); m_db.close();
addIdxReason("indexer", "Web index creation failed. See log"); addIdxReason("indexer", "Web index creation failed. See log");
@ -208,7 +208,7 @@ bool ConfIndexer::indexFiles(list<string>& ifiles, int flag)
if (m_doweb && !myfiles.empty() && !(flag & IxFNoWeb)) { if (m_doweb && !myfiles.empty() && !(flag & IxFNoWeb)) {
if (!m_webindexer) if (!m_webindexer)
m_webindexer = new BeagleQueueIndexer(m_config, &m_db, m_updater); m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater);
if (m_webindexer) { if (m_webindexer) {
ret = ret && m_webindexer->indexFiles(myfiles); ret = ret && m_webindexer->indexFiles(myfiles);
} else { } else {
@ -267,7 +267,7 @@ bool ConfIndexer::purgeFiles(list<string> &files, int flag)
#ifndef DISABLE_WEB_INDEXER #ifndef DISABLE_WEB_INDEXER
if (m_doweb && !myfiles.empty() && !(flag & IxFNoWeb)) { if (m_doweb && !myfiles.empty() && !(flag & IxFNoWeb)) {
if (!m_webindexer) if (!m_webindexer)
m_webindexer = new BeagleQueueIndexer(m_config, &m_db, m_updater); m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater);
if (m_webindexer) { if (m_webindexer) {
ret = ret && m_webindexer->purgeFiles(myfiles); ret = ret && m_webindexer->purgeFiles(myfiles);
} else { } else {

View File

@ -29,7 +29,7 @@
#include "idxstatus.h" #include "idxstatus.h"
class FsIndexer; class FsIndexer;
class BeagleQueueIndexer; class WebQueueIndexer;
/** Callback to say what we're doing. If the update func returns false, we /** Callback to say what we're doing. If the update func returns false, we
* stop as soon as possible without corrupting state */ * stop as soon as possible without corrupting state */
@ -118,7 +118,7 @@ class ConfIndexer {
Rcl::Db m_db; Rcl::Db m_db;
FsIndexer *m_fsindexer; FsIndexer *m_fsindexer;
bool m_doweb; bool m_doweb;
BeagleQueueIndexer *m_webindexer; WebQueueIndexer *m_webindexer;
DbIxStatusUpdater *m_updater; DbIxStatusUpdater *m_updater;
string m_reason; string m_reason;

View File

@ -49,7 +49,7 @@ using namespace std;
#include "cancelcheck.h" #include "cancelcheck.h"
#include "rcldb.h" #include "rcldb.h"
#ifndef DISABLE_WEB_INDEXER #ifndef DISABLE_WEB_INDEXER
#include "beaglequeue.h" #include "webqueue.h"
#endif #endif
#include "recollindex.h" #include "recollindex.h"
#include "fsindexer.h" #include "fsindexer.h"

View File

@ -16,6 +16,8 @@
*/ */
#include "autoconfig.h" #include "autoconfig.h"
#include "webqueue.h"
#include <string.h> #include <string.h>
#include <errno.h> #include <errno.h>
#include "safesysstat.h" #include "safesysstat.h"
@ -26,8 +28,7 @@
#include "rclutil.h" #include "rclutil.h"
#include "log.h" #include "log.h"
#include "fstreewalk.h" #include "fstreewalk.h"
#include "beaglequeue.h" #include "webstore.h"
#include "beaglequeuecache.h"
#include "circache.h" #include "circache.h"
#include "smallut.h" #include "smallut.h"
#include "fileudi.h" #include "fileudi.h"
@ -44,12 +45,13 @@
using namespace std; using namespace std;
// Beagle creates a file named .xxx (where xxx is the name for the main file // The browser plugin creates a file named .xxx (where xxx is the name
// in the queue), to hold external metadata (http or created by Beagle). // for the main file in the queue), to hold external metadata (http or
// This class reads the .xxx, dotfile, and turns it into an Rcl::Doc holder // created by the plugin). This class reads the .xxx, dotfile, and turns
class BeagleDotFile { // it into an Rcl::Doc holder
class WebQueueDotFile {
public: public:
BeagleDotFile(RclConfig *conf, const string& fn) WebQueueDotFile(RclConfig *conf, const string& fn)
: m_conf(conf), m_fn(fn) : m_conf(conf), m_fn(fn)
{} {}
@ -62,7 +64,7 @@ public:
m_input.getline(cline, LL-1); m_input.getline(cline, LL-1);
if (!m_input.good()) { if (!m_input.good()) {
if (m_input.bad()) { if (m_input.bad()) {
LOGERR("beagleDotFileRead: input.bad()\n" ); LOGERR("WebQueueDotFileRead: input.bad()\n" );
} }
return false; return false;
} }
@ -72,18 +74,18 @@ public:
ll--; ll--;
} }
line.assign(cline, ll); line.assign(cline, ll);
LOGDEB2("BeagleDotFile:readLine: [" << (line) << "]\n" ); LOGDEB2("WebQueueDotFile:readLine: [" << (line) << "]\n" );
return true; return true;
} }
// Process a beagle dot file and set interesting stuff in the doc // Process a Web queue dot file and set interesting stuff in the doc
bool toDoc(Rcl::Doc& doc) bool toDoc(Rcl::Doc& doc)
{ {
string line; string line;
m_input.open(m_fn.c_str(), ios::in); m_input.open(m_fn.c_str(), ios::in);
if (!m_input.good()) { if (!m_input.good()) {
LOGERR("BeagleDotFile: open failed for [" << (m_fn) << "]\n" ); LOGERR("WebQueueDotFile: open failed for [" << (m_fn) << "]\n" );
return false; return false;
} }
@ -173,24 +175,24 @@ public:
// Initialize. Compute paths and create a temporary directory that will be // Initialize. Compute paths and create a temporary directory that will be
// used by internfile() // used by internfile()
BeagleQueueIndexer::BeagleQueueIndexer(RclConfig *cnf, Rcl::Db *db, WebQueueIndexer::WebQueueIndexer(RclConfig *cnf, Rcl::Db *db,
DbIxStatusUpdater *updfunc) DbIxStatusUpdater *updfunc)
: m_config(cnf), m_db(db), m_cache(0), m_updater(updfunc), : m_config(cnf), m_db(db), m_cache(0), m_updater(updfunc),
m_nocacheindex(false) m_nocacheindex(false)
{ {
m_queuedir = m_config->getWebQueueDir(); m_queuedir = m_config->getWebQueueDir();
path_catslash(m_queuedir); path_catslash(m_queuedir);
m_cache = new BeagleQueueCache(cnf); m_cache = new WebStore(cnf);
} }
BeagleQueueIndexer::~BeagleQueueIndexer() WebQueueIndexer::~WebQueueIndexer()
{ {
LOGDEB("BeagleQueueIndexer::~\n" ); LOGDEB("WebQueueIndexer::~\n" );
deleteZ(m_cache); deleteZ(m_cache);
} }
// Index document stored in the cache. // Index document stored in the cache.
bool BeagleQueueIndexer::indexFromCache(const string& udi) bool WebQueueIndexer::indexFromCache(const string& udi)
{ {
if (!m_db) if (!m_db)
return false; return false;
@ -202,12 +204,12 @@ bool BeagleQueueIndexer::indexFromCache(const string& udi)
string hittype; string hittype;
if (!m_cache || !m_cache->getFromCache(udi, dotdoc, data, &hittype)) { if (!m_cache || !m_cache->getFromCache(udi, dotdoc, data, &hittype)) {
LOGERR("BeagleQueueIndexer::indexFromCache: cache failed\n" ); LOGERR("WebQueueIndexer::indexFromCache: cache failed\n" );
return false; return false;
} }
if (hittype.empty()) { if (hittype.empty()) {
LOGERR("BeagleIndexer::index: cc entry has no hit type\n" ); LOGERR("WebQueueIndexer::index: cc entry has no hit type\n" );
return false; return false;
} }
@ -224,11 +226,11 @@ bool BeagleQueueIndexer::indexFromCache(const string& udi)
try { try {
fis = interner.internfile(doc); fis = interner.internfile(doc);
} catch (CancelExcept) { } catch (CancelExcept) {
LOGERR("BeagleQueueIndexer: interrupted\n" ); LOGERR("WebQueueIndexer: interrupted\n" );
return false; return false;
} }
if (fis != FileInterner::FIDone) { if (fis != FileInterner::FIDone) {
LOGERR("BeagleQueueIndexer: bad status from internfile\n" ); LOGERR("WebQueueIndexer: bad status from internfile\n" );
return false; return false;
} }
@ -242,7 +244,7 @@ bool BeagleQueueIndexer::indexFromCache(const string& udi)
} }
} }
void BeagleQueueIndexer::updstatus(const string& udi) void WebQueueIndexer::updstatus(const string& udi)
{ {
if (m_updater) { if (m_updater) {
++(m_updater->status.docsdone); ++(m_updater->status.docsdone);
@ -253,18 +255,18 @@ void BeagleQueueIndexer::updstatus(const string& udi)
} }
} }
bool BeagleQueueIndexer::index() bool WebQueueIndexer::index()
{ {
if (!m_db) if (!m_db)
return false; return false;
LOGDEB("BeagleQueueIndexer::processqueue: [" << (m_queuedir) << "]\n" ); LOGDEB("WebQueueIndexer::processqueue: [" << (m_queuedir) << "]\n" );
m_config->setKeyDir(m_queuedir); m_config->setKeyDir(m_queuedir);
if (!path_makepath(m_queuedir, 0700)) { if (!path_makepath(m_queuedir, 0700)) {
LOGERR("BeagleQueueIndexer:: can't create queuedir [" << (m_queuedir) << "] errno " << (errno) << "\n" ); LOGERR("WebQueueIndexer:: can't create queuedir [" << (m_queuedir) << "] errno " << (errno) << "\n" );
return false; return false;
} }
if (!m_cache || !m_cache->cc()) { if (!m_cache || !m_cache->cc()) {
LOGERR("BeagleQueueIndexer: cache initialization failed\n" ); LOGERR("WebQueueIndexer: cache initialization failed\n" );
return false; return false;
} }
CirCache *cc = m_cache->cc(); CirCache *cc = m_cache->cc();
@ -282,7 +284,7 @@ bool BeagleQueueIndexer::index()
do { do {
string udi; string udi;
if (!cc->getCurrentUdi(udi)) { if (!cc->getCurrentUdi(udi)) {
LOGERR("BeagleQueueIndexer:: cache file damaged\n" ); LOGERR("WebQueueIndexer:: cache file damaged\n" );
break; break;
} }
if (udi.empty()) if (udi.empty())
@ -295,7 +297,7 @@ bool BeagleQueueIndexer::index()
indexFromCache(udi); indexFromCache(udi);
updstatus(udi); updstatus(udi);
} catch (CancelExcept) { } catch (CancelExcept) {
LOGERR("BeagleQueueIndexer: interrupted\n" ); LOGERR("WebQueueIndexer: interrupted\n" );
return false; return false;
} }
} }
@ -307,17 +309,17 @@ bool BeagleQueueIndexer::index()
FsTreeWalker walker(FsTreeWalker::FtwNoRecurse); FsTreeWalker walker(FsTreeWalker::FtwNoRecurse);
walker.addSkippedName(".*"); walker.addSkippedName(".*");
FsTreeWalker::Status status = walker.walk(m_queuedir, *this); FsTreeWalker::Status status = walker.walk(m_queuedir, *this);
LOGDEB("BeagleQueueIndexer::processqueue: done: status " << (status) << "\n" ); LOGDEB("WebQueueIndexer::processqueue: done: status " << (status) << "\n" );
return true; return true;
} }
// Index a list of files (sent by the real time monitor) // Index a list of files (sent by the real time monitor)
bool BeagleQueueIndexer::indexFiles(list<string>& files) bool WebQueueIndexer::indexFiles(list<string>& files)
{ {
LOGDEB("BeagleQueueIndexer::indexFiles\n" ); LOGDEB("WebQueueIndexer::indexFiles\n" );
if (!m_db) { if (!m_db) {
LOGERR("BeagleQueueIndexer::indexfiles no db??\n" ); LOGERR("WebQueueIndexer::indexfiles no db??\n" );
return false; return false;
} }
for (list<string>::iterator it = files.begin(); it != files.end();) { for (list<string>::iterator it = files.begin(); it != files.end();) {
@ -326,7 +328,7 @@ bool BeagleQueueIndexer::indexFiles(list<string>& files)
} }
string father = path_getfather(*it); string father = path_getfather(*it);
if (father.compare(m_queuedir)) { if (father.compare(m_queuedir)) {
LOGDEB("BeagleQueueIndexer::indexfiles: skipping [" << *it << "] (nq)\n" ); LOGDEB("WebQueueIndexer::indexfiles: skipping [" << *it << "] (nq)\n" );
it++; continue; it++; continue;
} }
// Pb: we are often called with the dot file, before the // Pb: we are often called with the dot file, before the
@ -342,11 +344,11 @@ bool BeagleQueueIndexer::indexFiles(list<string>& files)
} }
struct stat st; struct stat st;
if (path_fileprops(*it, &st) != 0) { if (path_fileprops(*it, &st) != 0) {
LOGERR("BeagleQueueIndexer::indexfiles: cant stat [" << *it << "]\n" ); LOGERR("WebQueueIndexer::indexfiles: cant stat [" << *it << "]\n" );
it++; continue; it++; continue;
} }
if (!S_ISREG(st.st_mode)) { if (!S_ISREG(st.st_mode)) {
LOGDEB("BeagleQueueIndexer::indexfiles: skipping [" << *it << "] (nr)\n" ); LOGDEB("WebQueueIndexer::indexfiles: skipping [" << *it << "] (nr)\n" );
it++; continue; it++; continue;
} }
@ -360,7 +362,7 @@ bool BeagleQueueIndexer::indexFiles(list<string>& files)
} }
FsTreeWalker::Status FsTreeWalker::Status
BeagleQueueIndexer::processone(const string &path, WebQueueIndexer::processone(const string &path,
const struct stat *stp, const struct stat *stp,
FsTreeWalker::CbFlag flg) FsTreeWalker::CbFlag flg)
{ {
@ -374,9 +376,9 @@ BeagleQueueIndexer::processone(const string &path,
string dotpath = path_cat(path_getfather(path), string dotpath = path_cat(path_getfather(path),
string(".") + path_getsimple(path)); string(".") + path_getsimple(path));
LOGDEB("BeagleQueueIndexer: prc1: [" << (path) << "]\n" ); LOGDEB("WebQueueIndexer: prc1: [" << (path) << "]\n" );
BeagleDotFile dotfile(m_config, dotpath); WebQueueDotFile dotfile(m_config, dotpath);
Rcl::Doc dotdoc; Rcl::Doc dotdoc;
string udi, udipath; string udi, udipath;
if (!dotfile.toDoc(dotdoc)) if (!dotfile.toDoc(dotdoc))
@ -388,7 +390,7 @@ BeagleQueueIndexer::processone(const string &path,
udipath = path_cat(dotdoc.meta[Rcl::Doc::keybght], url_gpath(dotdoc.url)); udipath = path_cat(dotdoc.meta[Rcl::Doc::keybght], url_gpath(dotdoc.url));
make_udi(udipath, cstr_null, udi); make_udi(udipath, cstr_null, udi);
LOGDEB("BeagleQueueIndexer: prc1: udi [" << (udi) << "]\n" ); LOGDEB("WebQueueIndexer: prc1: udi [" << (udi) << "]\n" );
char ascdate[30]; char ascdate[30];
sprintf(ascdate, "%ld", long(stp->st_mtime)); sprintf(ascdate, "%ld", long(stp->st_mtime));
@ -410,7 +412,7 @@ BeagleQueueIndexer::processone(const string &path,
} else { } else {
Rcl::Doc doc; Rcl::Doc doc;
// Store the dotdoc fields in the future doc. In case someone wants // Store the dotdoc fields in the future doc. In case someone wants
// to use beagle-generated fields like beagle:inurl // to use fields generated by the browser plugin like inurl
doc.meta = dotdoc.meta; doc.meta = dotdoc.meta;
FileInterner interner(path, stp, m_config, FileInterner interner(path, stp, m_config,
@ -420,11 +422,11 @@ BeagleQueueIndexer::processone(const string &path,
try { try {
fis = interner.internfile(doc); fis = interner.internfile(doc);
} catch (CancelExcept) { } catch (CancelExcept) {
LOGERR("BeagleQueueIndexer: interrupted\n" ); LOGERR("WebQueueIndexer: interrupted\n" );
goto out; goto out;
} }
if (fis != FileInterner::FIDone && fis != FileInterner::FIAgain) { if (fis != FileInterner::FIDone && fis != FileInterner::FIAgain) {
LOGERR("BeagleQueueIndexer: bad status from internfile\n" ); LOGERR("WebQueueIndexer: bad status from internfile\n" );
// TOBEDONE: internfile can return FIAgain here if it is // TOBEDONE: internfile can return FIAgain here if it is
// paging a big text file, we should loop. Means we're // paging a big text file, we should loop. Means we're
// only indexing the first page for text/plain files // only indexing the first page for text/plain files
@ -457,11 +459,11 @@ BeagleQueueIndexer::processone(const string &path,
string fdata; string fdata;
file_to_string(path, fdata); file_to_string(path, fdata);
if (!m_cache || !m_cache->cc()) { if (!m_cache || !m_cache->cc()) {
LOGERR("BeagleQueueIndexer: cache initialization failed\n" ); LOGERR("WebQueueIndexer: cache initialization failed\n" );
goto out; goto out;
} }
if (!m_cache->cc()->put(udi, &dotfile.m_fields, fdata, 0)) { if (!m_cache->cc()->put(udi, &dotfile.m_fields, fdata, 0)) {
LOGERR("BeagleQueueIndexer::prc1: cache_put failed; " << (m_cache->cc()->getReason()) << "\n" ); LOGERR("WebQueueIndexer::prc1: cache_put failed; " << (m_cache->cc()->getReason()) << "\n" );
goto out; goto out;
} }
} }

View File

@ -14,18 +14,17 @@
* Free Software Foundation, Inc., * Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/ */
#ifndef _beaglequeue_h_included_ #ifndef _webqueue_h_included_
#define _beaglequeue_h_included_ #define _webqueue_h_included_
#include <list> #include <list>
/** /**
* Process the Beagle indexing queue. * Process the WEB indexing queue.
* *
* Beagle MUST NOT be running, else mayhem will ensue. * This was originally written to reuse the Beagle Firefox plug-in (which
* * copied visited pages and bookmarks to the queue), long dead and replaced by a
* This is mainly written to reuse the Beagle Firefox plug-in (which * recoll-specific plugin.
* copies visited pages and bookmarks to the queue).
*/ */
#include "fstreewalk.h" #include "fstreewalk.h"
@ -34,16 +33,16 @@
class DbIxStatusUpdater; class DbIxStatusUpdater;
class CirCache; class CirCache;
class RclConfig; class RclConfig;
class BeagleQueueCache; class WebStore;
namespace Rcl { namespace Rcl {
class Db; class Db;
} }
class BeagleQueueIndexer : public FsTreeWalkerCB { class WebQueueIndexer : public FsTreeWalkerCB {
public: public:
BeagleQueueIndexer(RclConfig *cnf, Rcl::Db *db, WebQueueIndexer(RclConfig *cnf, Rcl::Db *db,
DbIxStatusUpdater *updfunc = 0); DbIxStatusUpdater *updfunc = 0);
~BeagleQueueIndexer(); ~WebQueueIndexer();
/** This is called by the top indexer in recollindex. /** This is called by the top indexer in recollindex.
* Does the walking and the talking */ * Does the walking and the talking */
@ -68,7 +67,7 @@ public:
private: private:
RclConfig *m_config; RclConfig *m_config;
Rcl::Db *m_db; Rcl::Db *m_db;
BeagleQueueCache *m_cache; WebStore *m_cache;
string m_queuedir; string m_queuedir;
DbIxStatusUpdater *m_updater; DbIxStatusUpdater *m_updater;
bool m_nocacheindex; bool m_nocacheindex;
@ -77,4 +76,4 @@ private:
void updstatus(const string& udi); void updstatus(const string& udi);
}; };
#endif /* _beaglequeue_h_included_ */ #endif /* _webqueue_h_included_ */

View File

@ -16,23 +16,26 @@
*/ */
#include "autoconfig.h" #include "autoconfig.h"
#include "webqueuefetcher.h"
#include <mutex> #include <mutex>
#include "rcldoc.h" #include "rcldoc.h"
#include "fetcher.h" #include "fetcher.h"
#include "bglfetcher.h"
#include "log.h" #include "log.h"
#include "beaglequeuecache.h" #include "webstore.h"
// We use a single beagle cache object to access beagle data. We protect it using std::string;
// We use a single WebStore object to access the data. We protect it
// against multiple thread access. // against multiple thread access.
static std::mutex o_beagler_mutex; static std::mutex o_beagler_mutex;
bool BGLDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out) bool WQDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
{ {
string udi; string udi;
if (!idoc.getmeta(Rcl::Doc::keyudi, &udi) || udi.empty()) { if (!idoc.getmeta(Rcl::Doc::keyudi, &udi) || udi.empty()) {
LOGERR("BGLDocFetcher:: no udi in idoc\n" ); LOGERR("WQDocFetcher:: no udi in idoc\n" );
return false; return false;
} }
Rcl::Doc dotdoc; Rcl::Doc dotdoc;
@ -41,24 +44,23 @@ bool BGLDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
// Retrieve from our webcache (beagle data). The beagler // Retrieve from our webcache (beagle data). The beagler
// object is created at the first call of this routine and // object is created at the first call of this routine and
// deleted when the program exits. // deleted when the program exits.
static BeagleQueueCache o_beagler(cnf); static WebStore o_beagler(cnf);
if (!o_beagler.getFromCache(udi, dotdoc, out.data)) { if (!o_beagler.getFromCache(udi, dotdoc, out.data)) {
LOGINFO("BGLDocFetcher::fetch: failed for [" << (udi) << "]\n" ); LOGINFO("WQDocFetcher::fetch: failed for [" << udi << "]\n");
return false; return false;
} }
} }
if (dotdoc.mimetype.compare(idoc.mimetype)) { if (dotdoc.mimetype.compare(idoc.mimetype)) {
LOGINFO("BGLDocFetcher:: udi [" << (udi) << "], mimetp mismatch: in: [" << (idoc.mimetype) << "], bgl [" << (dotdoc.mimetype) << "]\n" ); LOGINFO("WQDocFetcher:: udi [" << udi << "], mimetp mismatch: in: [" <<
idoc.mimetype << "], bgl [" << dotdoc.mimetype << "]\n");
} }
out.kind = RawDoc::RDK_DATA; out.kind = RawDoc::RDK_DATA;
return true; return true;
} }
bool BGLDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig) bool WQDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig)
{ {
// Bgl sigs are empty // Web queue sigs are empty
sig.clear(); sig.clear();
return true; return true;
} }

View File

@ -14,18 +14,19 @@
* Free Software Foundation, Inc., * Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/ */
#ifndef _BGLFETCHER_H_INCLUDED_ #ifndef _WEBQUEUEFETCHER_H_INCLUDED_
#define _BGLFETCHER_H_INCLUDED_ #define _WEBQUEUEFETCHER_H_INCLUDED_
#include "fetcher.h" #include "fetcher.h"
/** /**
* The Beagle cache fetcher: * The WEB queue cache fetcher:
*/ */
class BGLDocFetcher : public DocFetcher{ class WQDocFetcher : public DocFetcher{
virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out); virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out);
virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc, virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc,
std::string& sig); std::string& sig);
virtual ~BGLDocFetcher() {} virtual ~WQDocFetcher() {}
}; };
#endif /* _BGLFETCHER_H_INCLUDED_ */ #endif /* _WEBQUEUEFETCHER_H_INCLUDED_ */

View File

@ -43,7 +43,7 @@
#include "recoll.h" #include "recoll.h"
#include "webcache.h" #include "webcache.h"
#include "beaglequeuecache.h" #include "webstore.h"
#include "circache.h" #include "circache.h"
#include "conftree.h" #include "conftree.h"
#include "rclmain_w.h" #include "rclmain_w.h"
@ -62,7 +62,7 @@ public:
class WebcacheModelInternal { class WebcacheModelInternal {
public: public:
std::shared_ptr<BeagleQueueCache> cache; std::shared_ptr<WebStore> cache;
vector<CEnt> all; vector<CEnt> all;
vector<CEnt> disp; vector<CEnt> disp;
}; };
@ -81,7 +81,7 @@ WebcacheModel::~WebcacheModel()
void WebcacheModel::reload() void WebcacheModel::reload()
{ {
m->cache = m->cache =
std::shared_ptr<BeagleQueueCache>(new BeagleQueueCache(theconfig)); std::shared_ptr<WebStore>(new WebStore(theconfig));
m->all.clear(); m->all.clear();
m->disp.clear(); m->disp.clear();