Removed most ancient references to Beagle in the code, replaced by WebQueue or such to clarify things. No real changes
This commit is contained in:
parent
be3ac2e180
commit
9f94b9ed20
@ -58,8 +58,8 @@ bincimapmime/mime-printbody.cc \
|
||||
bincimapmime/mime-utils.h \
|
||||
bincimapmime/mime.cc \
|
||||
bincimapmime/mime.h \
|
||||
common/beaglequeuecache.cpp \
|
||||
common/beaglequeuecache.h \
|
||||
common/webstore.cpp \
|
||||
common/webstore.h \
|
||||
common/conf_post.h \
|
||||
common/cstr.cpp \
|
||||
common/cstr.h \
|
||||
@ -76,10 +76,10 @@ common/unacpp.h \
|
||||
common/uproplist.h \
|
||||
common/utf8fn.cpp \
|
||||
common/utf8fn.h \
|
||||
index/beaglequeue.cpp \
|
||||
index/beaglequeue.h \
|
||||
index/bglfetcher.cpp \
|
||||
index/bglfetcher.h \
|
||||
index/webqueue.cpp \
|
||||
index/webqueue.h \
|
||||
index/webqueuefetcher.cpp \
|
||||
index/webqueuefetcher.h \
|
||||
index/checkretryfailed.cpp \
|
||||
index/checkretryfailed.h \
|
||||
index/exefetcher.cpp \
|
||||
|
||||
@ -17,10 +17,11 @@
|
||||
|
||||
#include "autoconfig.h"
|
||||
|
||||
#include "webstore.h"
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "cstr.h"
|
||||
#include "beaglequeuecache.h"
|
||||
#include "circache.h"
|
||||
#include "log.h"
|
||||
#include "rclconfig.h"
|
||||
@ -29,42 +30,43 @@
|
||||
|
||||
const string cstr_bgc_mimetype("mimetype");
|
||||
|
||||
BeagleQueueCache::BeagleQueueCache(RclConfig *cnf)
|
||||
WebStore::WebStore(RclConfig *cnf)
|
||||
{
|
||||
string ccdir = cnf->getWebcacheDir();
|
||||
|
||||
int maxmbs = 40;
|
||||
cnf->getConfParam("webcachemaxmbs", &maxmbs);
|
||||
if ((m_cache = new CirCache(ccdir)) == 0) {
|
||||
LOGERR("BeagleQueueCache: cant create CirCache object\n" );
|
||||
LOGERR("WebStore: cant create CirCache object\n" );
|
||||
return;
|
||||
}
|
||||
if (!m_cache->create(int64_t(maxmbs)*1000*1024, CirCache::CC_CRUNIQUE)) {
|
||||
LOGERR("BeagleQueueCache: cache file creation failed: " << (m_cache->getReason()) << "\n" );
|
||||
LOGERR("WebStore: cache file creation failed: " <<
|
||||
m_cache->getReason() << "\n");
|
||||
delete m_cache;
|
||||
m_cache = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
BeagleQueueCache::~BeagleQueueCache()
|
||||
WebStore::~WebStore()
|
||||
{
|
||||
delete m_cache;
|
||||
}
|
||||
|
||||
// Read document from cache. Return the metadata as an Rcl::Doc
|
||||
// @param htt Beagle Hit Type
|
||||
bool BeagleQueueCache::getFromCache(const string& udi, Rcl::Doc &dotdoc,
|
||||
// @param htt Web Hit Type
|
||||
bool WebStore::getFromCache(const string& udi, Rcl::Doc &dotdoc,
|
||||
string& data, string *htt)
|
||||
{
|
||||
string dict;
|
||||
|
||||
if (m_cache == 0) {
|
||||
LOGERR("BeagleQueueCache::getFromCache: cache is null\n" );
|
||||
LOGERR("WebStore::getFromCache: cache is null\n");
|
||||
return false;
|
||||
}
|
||||
if (!m_cache->get(udi, dict, &data)) {
|
||||
LOGDEB("BeagleQueueCache::getFromCache: get failed\n" );
|
||||
LOGDEB("WebStore::getFromCache: get failed\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@ -14,11 +14,10 @@
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
#ifndef _beaglequeuecache_h_included_
|
||||
#define _beaglequeuecache_h_included_
|
||||
#ifndef _webstore_h_included_
|
||||
#define _webstore_h_included_
|
||||
|
||||
#include <string>
|
||||
using std::string;
|
||||
|
||||
class RclConfig;
|
||||
namespace Rcl {
|
||||
@ -28,23 +27,24 @@ namespace Rcl {
|
||||
class CirCache;
|
||||
|
||||
/**
|
||||
* Manage the CirCache for the Beagle Queue indexer. Separated from the main
|
||||
* Manage the CirCache for the Web Queue indexer. Separated from the main
|
||||
* indexer code because it's also used for querying (getting the data for a
|
||||
* preview
|
||||
*/
|
||||
class BeagleQueueCache {
|
||||
class WebStore {
|
||||
public:
|
||||
BeagleQueueCache(RclConfig *config);
|
||||
~BeagleQueueCache();
|
||||
WebStore(RclConfig *config);
|
||||
~WebStore();
|
||||
|
||||
bool getFromCache(const string& udi, Rcl::Doc &doc, string& data,
|
||||
string *hittype = 0);
|
||||
bool getFromCache(const std::string& udi, Rcl::Doc &doc, std::string& data,
|
||||
std::string *hittype = 0);
|
||||
// We could write proxies for all the circache ops, but why bother?
|
||||
CirCache *cc() {return m_cache;}
|
||||
|
||||
private:
|
||||
CirCache *m_cache;
|
||||
};
|
||||
extern const string cstr_bgc_mimetype;
|
||||
|
||||
#endif /* _beaglequeuecache_h_included_ */
|
||||
extern const std::string cstr_bgc_mimetype;
|
||||
|
||||
#endif /* _webstore_h_included_ */
|
||||
|
||||
@ -16,13 +16,11 @@
|
||||
*/
|
||||
#include "autoconfig.h"
|
||||
|
||||
|
||||
#include "log.h"
|
||||
#include "rclconfig.h"
|
||||
|
||||
#include "fetcher.h"
|
||||
#include "fsfetcher.h"
|
||||
#include "bglfetcher.h"
|
||||
#include "webqueuefetcher.h"
|
||||
#include "exefetcher.h"
|
||||
|
||||
DocFetcher *docFetcherMake(RclConfig *config, const Rcl::Doc& idoc)
|
||||
@ -37,7 +35,7 @@ DocFetcher *docFetcherMake(RclConfig *config, const Rcl::Doc& idoc)
|
||||
return new FSDocFetcher;
|
||||
#ifndef DISABLE_WEB_INDEXER
|
||||
} else if (!backend.compare("BGL")) {
|
||||
return new BGLDocFetcher;
|
||||
return new WQDocFetcher;
|
||||
#endif
|
||||
} else {
|
||||
DocFetcher *f = exeDocFetcherMake(config, backend);
|
||||
|
||||
@ -27,7 +27,7 @@
|
||||
#include "indexer.h"
|
||||
#include "fsindexer.h"
|
||||
#ifndef DISABLE_WEB_INDEXER
|
||||
#include "beaglequeue.h"
|
||||
#include "webqueue.h"
|
||||
#endif
|
||||
#include "mimehandler.h"
|
||||
#include "pathut.h"
|
||||
@ -132,7 +132,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
|
||||
if (m_doweb && (typestorun & IxTWebQueue)) {
|
||||
runWebFilesMoverScript(m_config);
|
||||
deleteZ(m_webindexer);
|
||||
m_webindexer = new BeagleQueueIndexer(m_config, &m_db, m_updater);
|
||||
m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater);
|
||||
if (!m_webindexer || !m_webindexer->index()) {
|
||||
m_db.close();
|
||||
addIdxReason("indexer", "Web index creation failed. See log");
|
||||
@ -208,7 +208,7 @@ bool ConfIndexer::indexFiles(list<string>& ifiles, int flag)
|
||||
|
||||
if (m_doweb && !myfiles.empty() && !(flag & IxFNoWeb)) {
|
||||
if (!m_webindexer)
|
||||
m_webindexer = new BeagleQueueIndexer(m_config, &m_db, m_updater);
|
||||
m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater);
|
||||
if (m_webindexer) {
|
||||
ret = ret && m_webindexer->indexFiles(myfiles);
|
||||
} else {
|
||||
@ -267,7 +267,7 @@ bool ConfIndexer::purgeFiles(list<string> &files, int flag)
|
||||
#ifndef DISABLE_WEB_INDEXER
|
||||
if (m_doweb && !myfiles.empty() && !(flag & IxFNoWeb)) {
|
||||
if (!m_webindexer)
|
||||
m_webindexer = new BeagleQueueIndexer(m_config, &m_db, m_updater);
|
||||
m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater);
|
||||
if (m_webindexer) {
|
||||
ret = ret && m_webindexer->purgeFiles(myfiles);
|
||||
} else {
|
||||
|
||||
@ -29,7 +29,7 @@
|
||||
#include "idxstatus.h"
|
||||
|
||||
class FsIndexer;
|
||||
class BeagleQueueIndexer;
|
||||
class WebQueueIndexer;
|
||||
|
||||
/** Callback to say what we're doing. If the update func returns false, we
|
||||
* stop as soon as possible without corrupting state */
|
||||
@ -118,7 +118,7 @@ class ConfIndexer {
|
||||
Rcl::Db m_db;
|
||||
FsIndexer *m_fsindexer;
|
||||
bool m_doweb;
|
||||
BeagleQueueIndexer *m_webindexer;
|
||||
WebQueueIndexer *m_webindexer;
|
||||
DbIxStatusUpdater *m_updater;
|
||||
string m_reason;
|
||||
|
||||
|
||||
@ -49,7 +49,7 @@ using namespace std;
|
||||
#include "cancelcheck.h"
|
||||
#include "rcldb.h"
|
||||
#ifndef DISABLE_WEB_INDEXER
|
||||
#include "beaglequeue.h"
|
||||
#include "webqueue.h"
|
||||
#endif
|
||||
#include "recollindex.h"
|
||||
#include "fsindexer.h"
|
||||
|
||||
@ -16,6 +16,8 @@
|
||||
*/
|
||||
#include "autoconfig.h"
|
||||
|
||||
#include "webqueue.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
#include "safesysstat.h"
|
||||
@ -26,8 +28,7 @@
|
||||
#include "rclutil.h"
|
||||
#include "log.h"
|
||||
#include "fstreewalk.h"
|
||||
#include "beaglequeue.h"
|
||||
#include "beaglequeuecache.h"
|
||||
#include "webstore.h"
|
||||
#include "circache.h"
|
||||
#include "smallut.h"
|
||||
#include "fileudi.h"
|
||||
@ -44,12 +45,13 @@
|
||||
|
||||
using namespace std;
|
||||
|
||||
// Beagle creates a file named .xxx (where xxx is the name for the main file
|
||||
// in the queue), to hold external metadata (http or created by Beagle).
|
||||
// This class reads the .xxx, dotfile, and turns it into an Rcl::Doc holder
|
||||
class BeagleDotFile {
|
||||
// The browser plugin creates a file named .xxx (where xxx is the name
|
||||
// for the main file in the queue), to hold external metadata (http or
|
||||
// created by the plugin). This class reads the .xxx, dotfile, and turns
|
||||
// it into an Rcl::Doc holder
|
||||
class WebQueueDotFile {
|
||||
public:
|
||||
BeagleDotFile(RclConfig *conf, const string& fn)
|
||||
WebQueueDotFile(RclConfig *conf, const string& fn)
|
||||
: m_conf(conf), m_fn(fn)
|
||||
{}
|
||||
|
||||
@ -62,7 +64,7 @@ public:
|
||||
m_input.getline(cline, LL-1);
|
||||
if (!m_input.good()) {
|
||||
if (m_input.bad()) {
|
||||
LOGERR("beagleDotFileRead: input.bad()\n" );
|
||||
LOGERR("WebQueueDotFileRead: input.bad()\n" );
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@ -72,18 +74,18 @@ public:
|
||||
ll--;
|
||||
}
|
||||
line.assign(cline, ll);
|
||||
LOGDEB2("BeagleDotFile:readLine: [" << (line) << "]\n" );
|
||||
LOGDEB2("WebQueueDotFile:readLine: [" << (line) << "]\n" );
|
||||
return true;
|
||||
}
|
||||
|
||||
// Process a beagle dot file and set interesting stuff in the doc
|
||||
// Process a Web queue dot file and set interesting stuff in the doc
|
||||
bool toDoc(Rcl::Doc& doc)
|
||||
{
|
||||
string line;
|
||||
|
||||
m_input.open(m_fn.c_str(), ios::in);
|
||||
if (!m_input.good()) {
|
||||
LOGERR("BeagleDotFile: open failed for [" << (m_fn) << "]\n" );
|
||||
LOGERR("WebQueueDotFile: open failed for [" << (m_fn) << "]\n" );
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -173,24 +175,24 @@ public:
|
||||
|
||||
// Initialize. Compute paths and create a temporary directory that will be
|
||||
// used by internfile()
|
||||
BeagleQueueIndexer::BeagleQueueIndexer(RclConfig *cnf, Rcl::Db *db,
|
||||
WebQueueIndexer::WebQueueIndexer(RclConfig *cnf, Rcl::Db *db,
|
||||
DbIxStatusUpdater *updfunc)
|
||||
: m_config(cnf), m_db(db), m_cache(0), m_updater(updfunc),
|
||||
m_nocacheindex(false)
|
||||
{
|
||||
m_queuedir = m_config->getWebQueueDir();
|
||||
path_catslash(m_queuedir);
|
||||
m_cache = new BeagleQueueCache(cnf);
|
||||
m_cache = new WebStore(cnf);
|
||||
}
|
||||
|
||||
BeagleQueueIndexer::~BeagleQueueIndexer()
|
||||
WebQueueIndexer::~WebQueueIndexer()
|
||||
{
|
||||
LOGDEB("BeagleQueueIndexer::~\n" );
|
||||
LOGDEB("WebQueueIndexer::~\n" );
|
||||
deleteZ(m_cache);
|
||||
}
|
||||
|
||||
// Index document stored in the cache.
|
||||
bool BeagleQueueIndexer::indexFromCache(const string& udi)
|
||||
bool WebQueueIndexer::indexFromCache(const string& udi)
|
||||
{
|
||||
if (!m_db)
|
||||
return false;
|
||||
@ -202,12 +204,12 @@ bool BeagleQueueIndexer::indexFromCache(const string& udi)
|
||||
string hittype;
|
||||
|
||||
if (!m_cache || !m_cache->getFromCache(udi, dotdoc, data, &hittype)) {
|
||||
LOGERR("BeagleQueueIndexer::indexFromCache: cache failed\n" );
|
||||
LOGERR("WebQueueIndexer::indexFromCache: cache failed\n" );
|
||||
return false;
|
||||
}
|
||||
|
||||
if (hittype.empty()) {
|
||||
LOGERR("BeagleIndexer::index: cc entry has no hit type\n" );
|
||||
LOGERR("WebQueueIndexer::index: cc entry has no hit type\n" );
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -224,11 +226,11 @@ bool BeagleQueueIndexer::indexFromCache(const string& udi)
|
||||
try {
|
||||
fis = interner.internfile(doc);
|
||||
} catch (CancelExcept) {
|
||||
LOGERR("BeagleQueueIndexer: interrupted\n" );
|
||||
LOGERR("WebQueueIndexer: interrupted\n" );
|
||||
return false;
|
||||
}
|
||||
if (fis != FileInterner::FIDone) {
|
||||
LOGERR("BeagleQueueIndexer: bad status from internfile\n" );
|
||||
LOGERR("WebQueueIndexer: bad status from internfile\n" );
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -242,7 +244,7 @@ bool BeagleQueueIndexer::indexFromCache(const string& udi)
|
||||
}
|
||||
}
|
||||
|
||||
void BeagleQueueIndexer::updstatus(const string& udi)
|
||||
void WebQueueIndexer::updstatus(const string& udi)
|
||||
{
|
||||
if (m_updater) {
|
||||
++(m_updater->status.docsdone);
|
||||
@ -253,18 +255,18 @@ void BeagleQueueIndexer::updstatus(const string& udi)
|
||||
}
|
||||
}
|
||||
|
||||
bool BeagleQueueIndexer::index()
|
||||
bool WebQueueIndexer::index()
|
||||
{
|
||||
if (!m_db)
|
||||
return false;
|
||||
LOGDEB("BeagleQueueIndexer::processqueue: [" << (m_queuedir) << "]\n" );
|
||||
LOGDEB("WebQueueIndexer::processqueue: [" << (m_queuedir) << "]\n" );
|
||||
m_config->setKeyDir(m_queuedir);
|
||||
if (!path_makepath(m_queuedir, 0700)) {
|
||||
LOGERR("BeagleQueueIndexer:: can't create queuedir [" << (m_queuedir) << "] errno " << (errno) << "\n" );
|
||||
LOGERR("WebQueueIndexer:: can't create queuedir [" << (m_queuedir) << "] errno " << (errno) << "\n" );
|
||||
return false;
|
||||
}
|
||||
if (!m_cache || !m_cache->cc()) {
|
||||
LOGERR("BeagleQueueIndexer: cache initialization failed\n" );
|
||||
LOGERR("WebQueueIndexer: cache initialization failed\n" );
|
||||
return false;
|
||||
}
|
||||
CirCache *cc = m_cache->cc();
|
||||
@ -282,7 +284,7 @@ bool BeagleQueueIndexer::index()
|
||||
do {
|
||||
string udi;
|
||||
if (!cc->getCurrentUdi(udi)) {
|
||||
LOGERR("BeagleQueueIndexer:: cache file damaged\n" );
|
||||
LOGERR("WebQueueIndexer:: cache file damaged\n" );
|
||||
break;
|
||||
}
|
||||
if (udi.empty())
|
||||
@ -295,7 +297,7 @@ bool BeagleQueueIndexer::index()
|
||||
indexFromCache(udi);
|
||||
updstatus(udi);
|
||||
} catch (CancelExcept) {
|
||||
LOGERR("BeagleQueueIndexer: interrupted\n" );
|
||||
LOGERR("WebQueueIndexer: interrupted\n" );
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -307,17 +309,17 @@ bool BeagleQueueIndexer::index()
|
||||
FsTreeWalker walker(FsTreeWalker::FtwNoRecurse);
|
||||
walker.addSkippedName(".*");
|
||||
FsTreeWalker::Status status = walker.walk(m_queuedir, *this);
|
||||
LOGDEB("BeagleQueueIndexer::processqueue: done: status " << (status) << "\n" );
|
||||
LOGDEB("WebQueueIndexer::processqueue: done: status " << (status) << "\n" );
|
||||
return true;
|
||||
}
|
||||
|
||||
// Index a list of files (sent by the real time monitor)
|
||||
bool BeagleQueueIndexer::indexFiles(list<string>& files)
|
||||
bool WebQueueIndexer::indexFiles(list<string>& files)
|
||||
{
|
||||
LOGDEB("BeagleQueueIndexer::indexFiles\n" );
|
||||
LOGDEB("WebQueueIndexer::indexFiles\n" );
|
||||
|
||||
if (!m_db) {
|
||||
LOGERR("BeagleQueueIndexer::indexfiles no db??\n" );
|
||||
LOGERR("WebQueueIndexer::indexfiles no db??\n" );
|
||||
return false;
|
||||
}
|
||||
for (list<string>::iterator it = files.begin(); it != files.end();) {
|
||||
@ -326,7 +328,7 @@ bool BeagleQueueIndexer::indexFiles(list<string>& files)
|
||||
}
|
||||
string father = path_getfather(*it);
|
||||
if (father.compare(m_queuedir)) {
|
||||
LOGDEB("BeagleQueueIndexer::indexfiles: skipping [" << *it << "] (nq)\n" );
|
||||
LOGDEB("WebQueueIndexer::indexfiles: skipping [" << *it << "] (nq)\n" );
|
||||
it++; continue;
|
||||
}
|
||||
// Pb: we are often called with the dot file, before the
|
||||
@ -342,11 +344,11 @@ bool BeagleQueueIndexer::indexFiles(list<string>& files)
|
||||
}
|
||||
struct stat st;
|
||||
if (path_fileprops(*it, &st) != 0) {
|
||||
LOGERR("BeagleQueueIndexer::indexfiles: cant stat [" << *it << "]\n" );
|
||||
LOGERR("WebQueueIndexer::indexfiles: cant stat [" << *it << "]\n" );
|
||||
it++; continue;
|
||||
}
|
||||
if (!S_ISREG(st.st_mode)) {
|
||||
LOGDEB("BeagleQueueIndexer::indexfiles: skipping [" << *it << "] (nr)\n" );
|
||||
LOGDEB("WebQueueIndexer::indexfiles: skipping [" << *it << "] (nr)\n" );
|
||||
it++; continue;
|
||||
}
|
||||
|
||||
@ -360,7 +362,7 @@ bool BeagleQueueIndexer::indexFiles(list<string>& files)
|
||||
}
|
||||
|
||||
FsTreeWalker::Status
|
||||
BeagleQueueIndexer::processone(const string &path,
|
||||
WebQueueIndexer::processone(const string &path,
|
||||
const struct stat *stp,
|
||||
FsTreeWalker::CbFlag flg)
|
||||
{
|
||||
@ -374,9 +376,9 @@ BeagleQueueIndexer::processone(const string &path,
|
||||
|
||||
string dotpath = path_cat(path_getfather(path),
|
||||
string(".") + path_getsimple(path));
|
||||
LOGDEB("BeagleQueueIndexer: prc1: [" << (path) << "]\n" );
|
||||
LOGDEB("WebQueueIndexer: prc1: [" << (path) << "]\n" );
|
||||
|
||||
BeagleDotFile dotfile(m_config, dotpath);
|
||||
WebQueueDotFile dotfile(m_config, dotpath);
|
||||
Rcl::Doc dotdoc;
|
||||
string udi, udipath;
|
||||
if (!dotfile.toDoc(dotdoc))
|
||||
@ -388,7 +390,7 @@ BeagleQueueIndexer::processone(const string &path,
|
||||
udipath = path_cat(dotdoc.meta[Rcl::Doc::keybght], url_gpath(dotdoc.url));
|
||||
make_udi(udipath, cstr_null, udi);
|
||||
|
||||
LOGDEB("BeagleQueueIndexer: prc1: udi [" << (udi) << "]\n" );
|
||||
LOGDEB("WebQueueIndexer: prc1: udi [" << (udi) << "]\n" );
|
||||
char ascdate[30];
|
||||
sprintf(ascdate, "%ld", long(stp->st_mtime));
|
||||
|
||||
@ -410,7 +412,7 @@ BeagleQueueIndexer::processone(const string &path,
|
||||
} else {
|
||||
Rcl::Doc doc;
|
||||
// Store the dotdoc fields in the future doc. In case someone wants
|
||||
// to use beagle-generated fields like beagle:inurl
|
||||
// to use fields generated by the browser plugin like inurl
|
||||
doc.meta = dotdoc.meta;
|
||||
|
||||
FileInterner interner(path, stp, m_config,
|
||||
@ -420,11 +422,11 @@ BeagleQueueIndexer::processone(const string &path,
|
||||
try {
|
||||
fis = interner.internfile(doc);
|
||||
} catch (CancelExcept) {
|
||||
LOGERR("BeagleQueueIndexer: interrupted\n" );
|
||||
LOGERR("WebQueueIndexer: interrupted\n" );
|
||||
goto out;
|
||||
}
|
||||
if (fis != FileInterner::FIDone && fis != FileInterner::FIAgain) {
|
||||
LOGERR("BeagleQueueIndexer: bad status from internfile\n" );
|
||||
LOGERR("WebQueueIndexer: bad status from internfile\n" );
|
||||
// TOBEDONE: internfile can return FIAgain here if it is
|
||||
// paging a big text file, we should loop. Means we're
|
||||
// only indexing the first page for text/plain files
|
||||
@ -457,11 +459,11 @@ BeagleQueueIndexer::processone(const string &path,
|
||||
string fdata;
|
||||
file_to_string(path, fdata);
|
||||
if (!m_cache || !m_cache->cc()) {
|
||||
LOGERR("BeagleQueueIndexer: cache initialization failed\n" );
|
||||
LOGERR("WebQueueIndexer: cache initialization failed\n" );
|
||||
goto out;
|
||||
}
|
||||
if (!m_cache->cc()->put(udi, &dotfile.m_fields, fdata, 0)) {
|
||||
LOGERR("BeagleQueueIndexer::prc1: cache_put failed; " << (m_cache->cc()->getReason()) << "\n" );
|
||||
LOGERR("WebQueueIndexer::prc1: cache_put failed; " << (m_cache->cc()->getReason()) << "\n" );
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
@ -14,18 +14,17 @@
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
#ifndef _beaglequeue_h_included_
|
||||
#define _beaglequeue_h_included_
|
||||
#ifndef _webqueue_h_included_
|
||||
#define _webqueue_h_included_
|
||||
|
||||
#include <list>
|
||||
|
||||
/**
|
||||
* Process the Beagle indexing queue.
|
||||
* Process the WEB indexing queue.
|
||||
*
|
||||
* Beagle MUST NOT be running, else mayhem will ensue.
|
||||
*
|
||||
* This is mainly written to reuse the Beagle Firefox plug-in (which
|
||||
* copies visited pages and bookmarks to the queue).
|
||||
* This was originally written to reuse the Beagle Firefox plug-in (which
|
||||
* copied visited pages and bookmarks to the queue), long dead and replaced by a
|
||||
* recoll-specific plugin.
|
||||
*/
|
||||
|
||||
#include "fstreewalk.h"
|
||||
@ -34,16 +33,16 @@
|
||||
class DbIxStatusUpdater;
|
||||
class CirCache;
|
||||
class RclConfig;
|
||||
class BeagleQueueCache;
|
||||
class WebStore;
|
||||
namespace Rcl {
|
||||
class Db;
|
||||
}
|
||||
|
||||
class BeagleQueueIndexer : public FsTreeWalkerCB {
|
||||
class WebQueueIndexer : public FsTreeWalkerCB {
|
||||
public:
|
||||
BeagleQueueIndexer(RclConfig *cnf, Rcl::Db *db,
|
||||
WebQueueIndexer(RclConfig *cnf, Rcl::Db *db,
|
||||
DbIxStatusUpdater *updfunc = 0);
|
||||
~BeagleQueueIndexer();
|
||||
~WebQueueIndexer();
|
||||
|
||||
/** This is called by the top indexer in recollindex.
|
||||
* Does the walking and the talking */
|
||||
@ -68,7 +67,7 @@ public:
|
||||
private:
|
||||
RclConfig *m_config;
|
||||
Rcl::Db *m_db;
|
||||
BeagleQueueCache *m_cache;
|
||||
WebStore *m_cache;
|
||||
string m_queuedir;
|
||||
DbIxStatusUpdater *m_updater;
|
||||
bool m_nocacheindex;
|
||||
@ -77,4 +76,4 @@ private:
|
||||
void updstatus(const string& udi);
|
||||
};
|
||||
|
||||
#endif /* _beaglequeue_h_included_ */
|
||||
#endif /* _webqueue_h_included_ */
|
||||
|
||||
@ -16,23 +16,26 @@
|
||||
*/
|
||||
#include "autoconfig.h"
|
||||
|
||||
#include "webqueuefetcher.h"
|
||||
|
||||
#include <mutex>
|
||||
|
||||
#include "rcldoc.h"
|
||||
#include "fetcher.h"
|
||||
#include "bglfetcher.h"
|
||||
#include "log.h"
|
||||
#include "beaglequeuecache.h"
|
||||
#include "webstore.h"
|
||||
|
||||
// We use a single beagle cache object to access beagle data. We protect it
|
||||
using std::string;
|
||||
|
||||
// We use a single WebStore object to access the data. We protect it
|
||||
// against multiple thread access.
|
||||
static std::mutex o_beagler_mutex;
|
||||
|
||||
bool BGLDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
|
||||
bool WQDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
|
||||
{
|
||||
string udi;
|
||||
if (!idoc.getmeta(Rcl::Doc::keyudi, &udi) || udi.empty()) {
|
||||
LOGERR("BGLDocFetcher:: no udi in idoc\n" );
|
||||
LOGERR("WQDocFetcher:: no udi in idoc\n" );
|
||||
return false;
|
||||
}
|
||||
Rcl::Doc dotdoc;
|
||||
@ -41,24 +44,23 @@ bool BGLDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
|
||||
// Retrieve from our webcache (beagle data). The beagler
|
||||
// object is created at the first call of this routine and
|
||||
// deleted when the program exits.
|
||||
static BeagleQueueCache o_beagler(cnf);
|
||||
static WebStore o_beagler(cnf);
|
||||
if (!o_beagler.getFromCache(udi, dotdoc, out.data)) {
|
||||
LOGINFO("BGLDocFetcher::fetch: failed for [" << (udi) << "]\n" );
|
||||
LOGINFO("WQDocFetcher::fetch: failed for [" << udi << "]\n");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (dotdoc.mimetype.compare(idoc.mimetype)) {
|
||||
LOGINFO("BGLDocFetcher:: udi [" << (udi) << "], mimetp mismatch: in: [" << (idoc.mimetype) << "], bgl [" << (dotdoc.mimetype) << "]\n" );
|
||||
LOGINFO("WQDocFetcher:: udi [" << udi << "], mimetp mismatch: in: [" <<
|
||||
idoc.mimetype << "], bgl [" << dotdoc.mimetype << "]\n");
|
||||
}
|
||||
out.kind = RawDoc::RDK_DATA;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool BGLDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig)
|
||||
bool WQDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig)
|
||||
{
|
||||
// Bgl sigs are empty
|
||||
// Web queue sigs are empty
|
||||
sig.clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@ -14,18 +14,19 @@
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
#ifndef _BGLFETCHER_H_INCLUDED_
|
||||
#define _BGLFETCHER_H_INCLUDED_
|
||||
#ifndef _WEBQUEUEFETCHER_H_INCLUDED_
|
||||
#define _WEBQUEUEFETCHER_H_INCLUDED_
|
||||
|
||||
#include "fetcher.h"
|
||||
|
||||
/**
|
||||
* The Beagle cache fetcher:
|
||||
* The WEB queue cache fetcher:
|
||||
*/
|
||||
class BGLDocFetcher : public DocFetcher{
|
||||
class WQDocFetcher : public DocFetcher{
|
||||
virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out);
|
||||
virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc,
|
||||
std::string& sig);
|
||||
virtual ~BGLDocFetcher() {}
|
||||
virtual ~WQDocFetcher() {}
|
||||
};
|
||||
|
||||
#endif /* _BGLFETCHER_H_INCLUDED_ */
|
||||
#endif /* _WEBQUEUEFETCHER_H_INCLUDED_ */
|
||||
@ -43,7 +43,7 @@
|
||||
|
||||
#include "recoll.h"
|
||||
#include "webcache.h"
|
||||
#include "beaglequeuecache.h"
|
||||
#include "webstore.h"
|
||||
#include "circache.h"
|
||||
#include "conftree.h"
|
||||
#include "rclmain_w.h"
|
||||
@ -62,7 +62,7 @@ public:
|
||||
|
||||
class WebcacheModelInternal {
|
||||
public:
|
||||
std::shared_ptr<BeagleQueueCache> cache;
|
||||
std::shared_ptr<WebStore> cache;
|
||||
vector<CEnt> all;
|
||||
vector<CEnt> disp;
|
||||
};
|
||||
@ -81,7 +81,7 @@ WebcacheModel::~WebcacheModel()
|
||||
void WebcacheModel::reload()
|
||||
{
|
||||
m->cache =
|
||||
std::shared_ptr<BeagleQueueCache>(new BeagleQueueCache(theconfig));
|
||||
std::shared_ptr<WebStore>(new WebStore(theconfig));
|
||||
m->all.clear();
|
||||
m->disp.clear();
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user