Removed most ancient references to Beagle in the code, replaced by WebQueue or such to clarify things. No real changes
This commit is contained in:
parent
be3ac2e180
commit
9f94b9ed20
@ -58,8 +58,8 @@ bincimapmime/mime-printbody.cc \
|
|||||||
bincimapmime/mime-utils.h \
|
bincimapmime/mime-utils.h \
|
||||||
bincimapmime/mime.cc \
|
bincimapmime/mime.cc \
|
||||||
bincimapmime/mime.h \
|
bincimapmime/mime.h \
|
||||||
common/beaglequeuecache.cpp \
|
common/webstore.cpp \
|
||||||
common/beaglequeuecache.h \
|
common/webstore.h \
|
||||||
common/conf_post.h \
|
common/conf_post.h \
|
||||||
common/cstr.cpp \
|
common/cstr.cpp \
|
||||||
common/cstr.h \
|
common/cstr.h \
|
||||||
@ -76,10 +76,10 @@ common/unacpp.h \
|
|||||||
common/uproplist.h \
|
common/uproplist.h \
|
||||||
common/utf8fn.cpp \
|
common/utf8fn.cpp \
|
||||||
common/utf8fn.h \
|
common/utf8fn.h \
|
||||||
index/beaglequeue.cpp \
|
index/webqueue.cpp \
|
||||||
index/beaglequeue.h \
|
index/webqueue.h \
|
||||||
index/bglfetcher.cpp \
|
index/webqueuefetcher.cpp \
|
||||||
index/bglfetcher.h \
|
index/webqueuefetcher.h \
|
||||||
index/checkretryfailed.cpp \
|
index/checkretryfailed.cpp \
|
||||||
index/checkretryfailed.h \
|
index/checkretryfailed.h \
|
||||||
index/exefetcher.cpp \
|
index/exefetcher.cpp \
|
||||||
|
|||||||
@ -17,10 +17,11 @@
|
|||||||
|
|
||||||
#include "autoconfig.h"
|
#include "autoconfig.h"
|
||||||
|
|
||||||
|
#include "webstore.h"
|
||||||
|
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
|
||||||
#include "cstr.h"
|
#include "cstr.h"
|
||||||
#include "beaglequeuecache.h"
|
|
||||||
#include "circache.h"
|
#include "circache.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "rclconfig.h"
|
#include "rclconfig.h"
|
||||||
@ -29,42 +30,43 @@
|
|||||||
|
|
||||||
const string cstr_bgc_mimetype("mimetype");
|
const string cstr_bgc_mimetype("mimetype");
|
||||||
|
|
||||||
BeagleQueueCache::BeagleQueueCache(RclConfig *cnf)
|
WebStore::WebStore(RclConfig *cnf)
|
||||||
{
|
{
|
||||||
string ccdir = cnf->getWebcacheDir();
|
string ccdir = cnf->getWebcacheDir();
|
||||||
|
|
||||||
int maxmbs = 40;
|
int maxmbs = 40;
|
||||||
cnf->getConfParam("webcachemaxmbs", &maxmbs);
|
cnf->getConfParam("webcachemaxmbs", &maxmbs);
|
||||||
if ((m_cache = new CirCache(ccdir)) == 0) {
|
if ((m_cache = new CirCache(ccdir)) == 0) {
|
||||||
LOGERR("BeagleQueueCache: cant create CirCache object\n" );
|
LOGERR("WebStore: cant create CirCache object\n" );
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!m_cache->create(int64_t(maxmbs)*1000*1024, CirCache::CC_CRUNIQUE)) {
|
if (!m_cache->create(int64_t(maxmbs)*1000*1024, CirCache::CC_CRUNIQUE)) {
|
||||||
LOGERR("BeagleQueueCache: cache file creation failed: " << (m_cache->getReason()) << "\n" );
|
LOGERR("WebStore: cache file creation failed: " <<
|
||||||
|
m_cache->getReason() << "\n");
|
||||||
delete m_cache;
|
delete m_cache;
|
||||||
m_cache = 0;
|
m_cache = 0;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
BeagleQueueCache::~BeagleQueueCache()
|
WebStore::~WebStore()
|
||||||
{
|
{
|
||||||
delete m_cache;
|
delete m_cache;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read document from cache. Return the metadata as an Rcl::Doc
|
// Read document from cache. Return the metadata as an Rcl::Doc
|
||||||
// @param htt Beagle Hit Type
|
// @param htt Web Hit Type
|
||||||
bool BeagleQueueCache::getFromCache(const string& udi, Rcl::Doc &dotdoc,
|
bool WebStore::getFromCache(const string& udi, Rcl::Doc &dotdoc,
|
||||||
string& data, string *htt)
|
string& data, string *htt)
|
||||||
{
|
{
|
||||||
string dict;
|
string dict;
|
||||||
|
|
||||||
if (m_cache == 0) {
|
if (m_cache == 0) {
|
||||||
LOGERR("BeagleQueueCache::getFromCache: cache is null\n" );
|
LOGERR("WebStore::getFromCache: cache is null\n");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!m_cache->get(udi, dict, &data)) {
|
if (!m_cache->get(udi, dict, &data)) {
|
||||||
LOGDEB("BeagleQueueCache::getFromCache: get failed\n" );
|
LOGDEB("WebStore::getFromCache: get failed\n");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -14,11 +14,10 @@
|
|||||||
* Free Software Foundation, Inc.,
|
* Free Software Foundation, Inc.,
|
||||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
*/
|
*/
|
||||||
#ifndef _beaglequeuecache_h_included_
|
#ifndef _webstore_h_included_
|
||||||
#define _beaglequeuecache_h_included_
|
#define _webstore_h_included_
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
using std::string;
|
|
||||||
|
|
||||||
class RclConfig;
|
class RclConfig;
|
||||||
namespace Rcl {
|
namespace Rcl {
|
||||||
@ -28,23 +27,24 @@ namespace Rcl {
|
|||||||
class CirCache;
|
class CirCache;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Manage the CirCache for the Beagle Queue indexer. Separated from the main
|
* Manage the CirCache for the Web Queue indexer. Separated from the main
|
||||||
* indexer code because it's also used for querying (getting the data for a
|
* indexer code because it's also used for querying (getting the data for a
|
||||||
* preview
|
* preview
|
||||||
*/
|
*/
|
||||||
class BeagleQueueCache {
|
class WebStore {
|
||||||
public:
|
public:
|
||||||
BeagleQueueCache(RclConfig *config);
|
WebStore(RclConfig *config);
|
||||||
~BeagleQueueCache();
|
~WebStore();
|
||||||
|
|
||||||
bool getFromCache(const string& udi, Rcl::Doc &doc, string& data,
|
bool getFromCache(const std::string& udi, Rcl::Doc &doc, std::string& data,
|
||||||
string *hittype = 0);
|
std::string *hittype = 0);
|
||||||
// We could write proxies for all the circache ops, but why bother?
|
// We could write proxies for all the circache ops, but why bother?
|
||||||
CirCache *cc() {return m_cache;}
|
CirCache *cc() {return m_cache;}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
CirCache *m_cache;
|
CirCache *m_cache;
|
||||||
};
|
};
|
||||||
extern const string cstr_bgc_mimetype;
|
|
||||||
|
|
||||||
#endif /* _beaglequeuecache_h_included_ */
|
extern const std::string cstr_bgc_mimetype;
|
||||||
|
|
||||||
|
#endif /* _webstore_h_included_ */
|
||||||
|
|||||||
@ -16,13 +16,11 @@
|
|||||||
*/
|
*/
|
||||||
#include "autoconfig.h"
|
#include "autoconfig.h"
|
||||||
|
|
||||||
|
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "rclconfig.h"
|
#include "rclconfig.h"
|
||||||
|
|
||||||
#include "fetcher.h"
|
#include "fetcher.h"
|
||||||
#include "fsfetcher.h"
|
#include "fsfetcher.h"
|
||||||
#include "bglfetcher.h"
|
#include "webqueuefetcher.h"
|
||||||
#include "exefetcher.h"
|
#include "exefetcher.h"
|
||||||
|
|
||||||
DocFetcher *docFetcherMake(RclConfig *config, const Rcl::Doc& idoc)
|
DocFetcher *docFetcherMake(RclConfig *config, const Rcl::Doc& idoc)
|
||||||
@ -37,7 +35,7 @@ DocFetcher *docFetcherMake(RclConfig *config, const Rcl::Doc& idoc)
|
|||||||
return new FSDocFetcher;
|
return new FSDocFetcher;
|
||||||
#ifndef DISABLE_WEB_INDEXER
|
#ifndef DISABLE_WEB_INDEXER
|
||||||
} else if (!backend.compare("BGL")) {
|
} else if (!backend.compare("BGL")) {
|
||||||
return new BGLDocFetcher;
|
return new WQDocFetcher;
|
||||||
#endif
|
#endif
|
||||||
} else {
|
} else {
|
||||||
DocFetcher *f = exeDocFetcherMake(config, backend);
|
DocFetcher *f = exeDocFetcherMake(config, backend);
|
||||||
|
|||||||
@ -27,7 +27,7 @@
|
|||||||
#include "indexer.h"
|
#include "indexer.h"
|
||||||
#include "fsindexer.h"
|
#include "fsindexer.h"
|
||||||
#ifndef DISABLE_WEB_INDEXER
|
#ifndef DISABLE_WEB_INDEXER
|
||||||
#include "beaglequeue.h"
|
#include "webqueue.h"
|
||||||
#endif
|
#endif
|
||||||
#include "mimehandler.h"
|
#include "mimehandler.h"
|
||||||
#include "pathut.h"
|
#include "pathut.h"
|
||||||
@ -132,7 +132,7 @@ bool ConfIndexer::index(bool resetbefore, ixType typestorun, int flags)
|
|||||||
if (m_doweb && (typestorun & IxTWebQueue)) {
|
if (m_doweb && (typestorun & IxTWebQueue)) {
|
||||||
runWebFilesMoverScript(m_config);
|
runWebFilesMoverScript(m_config);
|
||||||
deleteZ(m_webindexer);
|
deleteZ(m_webindexer);
|
||||||
m_webindexer = new BeagleQueueIndexer(m_config, &m_db, m_updater);
|
m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater);
|
||||||
if (!m_webindexer || !m_webindexer->index()) {
|
if (!m_webindexer || !m_webindexer->index()) {
|
||||||
m_db.close();
|
m_db.close();
|
||||||
addIdxReason("indexer", "Web index creation failed. See log");
|
addIdxReason("indexer", "Web index creation failed. See log");
|
||||||
@ -208,7 +208,7 @@ bool ConfIndexer::indexFiles(list<string>& ifiles, int flag)
|
|||||||
|
|
||||||
if (m_doweb && !myfiles.empty() && !(flag & IxFNoWeb)) {
|
if (m_doweb && !myfiles.empty() && !(flag & IxFNoWeb)) {
|
||||||
if (!m_webindexer)
|
if (!m_webindexer)
|
||||||
m_webindexer = new BeagleQueueIndexer(m_config, &m_db, m_updater);
|
m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater);
|
||||||
if (m_webindexer) {
|
if (m_webindexer) {
|
||||||
ret = ret && m_webindexer->indexFiles(myfiles);
|
ret = ret && m_webindexer->indexFiles(myfiles);
|
||||||
} else {
|
} else {
|
||||||
@ -267,7 +267,7 @@ bool ConfIndexer::purgeFiles(list<string> &files, int flag)
|
|||||||
#ifndef DISABLE_WEB_INDEXER
|
#ifndef DISABLE_WEB_INDEXER
|
||||||
if (m_doweb && !myfiles.empty() && !(flag & IxFNoWeb)) {
|
if (m_doweb && !myfiles.empty() && !(flag & IxFNoWeb)) {
|
||||||
if (!m_webindexer)
|
if (!m_webindexer)
|
||||||
m_webindexer = new BeagleQueueIndexer(m_config, &m_db, m_updater);
|
m_webindexer = new WebQueueIndexer(m_config, &m_db, m_updater);
|
||||||
if (m_webindexer) {
|
if (m_webindexer) {
|
||||||
ret = ret && m_webindexer->purgeFiles(myfiles);
|
ret = ret && m_webindexer->purgeFiles(myfiles);
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@ -29,7 +29,7 @@
|
|||||||
#include "idxstatus.h"
|
#include "idxstatus.h"
|
||||||
|
|
||||||
class FsIndexer;
|
class FsIndexer;
|
||||||
class BeagleQueueIndexer;
|
class WebQueueIndexer;
|
||||||
|
|
||||||
/** Callback to say what we're doing. If the update func returns false, we
|
/** Callback to say what we're doing. If the update func returns false, we
|
||||||
* stop as soon as possible without corrupting state */
|
* stop as soon as possible without corrupting state */
|
||||||
@ -118,7 +118,7 @@ class ConfIndexer {
|
|||||||
Rcl::Db m_db;
|
Rcl::Db m_db;
|
||||||
FsIndexer *m_fsindexer;
|
FsIndexer *m_fsindexer;
|
||||||
bool m_doweb;
|
bool m_doweb;
|
||||||
BeagleQueueIndexer *m_webindexer;
|
WebQueueIndexer *m_webindexer;
|
||||||
DbIxStatusUpdater *m_updater;
|
DbIxStatusUpdater *m_updater;
|
||||||
string m_reason;
|
string m_reason;
|
||||||
|
|
||||||
|
|||||||
@ -49,7 +49,7 @@ using namespace std;
|
|||||||
#include "cancelcheck.h"
|
#include "cancelcheck.h"
|
||||||
#include "rcldb.h"
|
#include "rcldb.h"
|
||||||
#ifndef DISABLE_WEB_INDEXER
|
#ifndef DISABLE_WEB_INDEXER
|
||||||
#include "beaglequeue.h"
|
#include "webqueue.h"
|
||||||
#endif
|
#endif
|
||||||
#include "recollindex.h"
|
#include "recollindex.h"
|
||||||
#include "fsindexer.h"
|
#include "fsindexer.h"
|
||||||
|
|||||||
@ -16,6 +16,8 @@
|
|||||||
*/
|
*/
|
||||||
#include "autoconfig.h"
|
#include "autoconfig.h"
|
||||||
|
|
||||||
|
#include "webqueue.h"
|
||||||
|
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include "safesysstat.h"
|
#include "safesysstat.h"
|
||||||
@ -26,8 +28,7 @@
|
|||||||
#include "rclutil.h"
|
#include "rclutil.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "fstreewalk.h"
|
#include "fstreewalk.h"
|
||||||
#include "beaglequeue.h"
|
#include "webstore.h"
|
||||||
#include "beaglequeuecache.h"
|
|
||||||
#include "circache.h"
|
#include "circache.h"
|
||||||
#include "smallut.h"
|
#include "smallut.h"
|
||||||
#include "fileudi.h"
|
#include "fileudi.h"
|
||||||
@ -44,12 +45,13 @@
|
|||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
// Beagle creates a file named .xxx (where xxx is the name for the main file
|
// The browser plugin creates a file named .xxx (where xxx is the name
|
||||||
// in the queue), to hold external metadata (http or created by Beagle).
|
// for the main file in the queue), to hold external metadata (http or
|
||||||
// This class reads the .xxx, dotfile, and turns it into an Rcl::Doc holder
|
// created by the plugin). This class reads the .xxx, dotfile, and turns
|
||||||
class BeagleDotFile {
|
// it into an Rcl::Doc holder
|
||||||
|
class WebQueueDotFile {
|
||||||
public:
|
public:
|
||||||
BeagleDotFile(RclConfig *conf, const string& fn)
|
WebQueueDotFile(RclConfig *conf, const string& fn)
|
||||||
: m_conf(conf), m_fn(fn)
|
: m_conf(conf), m_fn(fn)
|
||||||
{}
|
{}
|
||||||
|
|
||||||
@ -62,7 +64,7 @@ public:
|
|||||||
m_input.getline(cline, LL-1);
|
m_input.getline(cline, LL-1);
|
||||||
if (!m_input.good()) {
|
if (!m_input.good()) {
|
||||||
if (m_input.bad()) {
|
if (m_input.bad()) {
|
||||||
LOGERR("beagleDotFileRead: input.bad()\n" );
|
LOGERR("WebQueueDotFileRead: input.bad()\n" );
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@ -72,18 +74,18 @@ public:
|
|||||||
ll--;
|
ll--;
|
||||||
}
|
}
|
||||||
line.assign(cline, ll);
|
line.assign(cline, ll);
|
||||||
LOGDEB2("BeagleDotFile:readLine: [" << (line) << "]\n" );
|
LOGDEB2("WebQueueDotFile:readLine: [" << (line) << "]\n" );
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process a beagle dot file and set interesting stuff in the doc
|
// Process a Web queue dot file and set interesting stuff in the doc
|
||||||
bool toDoc(Rcl::Doc& doc)
|
bool toDoc(Rcl::Doc& doc)
|
||||||
{
|
{
|
||||||
string line;
|
string line;
|
||||||
|
|
||||||
m_input.open(m_fn.c_str(), ios::in);
|
m_input.open(m_fn.c_str(), ios::in);
|
||||||
if (!m_input.good()) {
|
if (!m_input.good()) {
|
||||||
LOGERR("BeagleDotFile: open failed for [" << (m_fn) << "]\n" );
|
LOGERR("WebQueueDotFile: open failed for [" << (m_fn) << "]\n" );
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -173,24 +175,24 @@ public:
|
|||||||
|
|
||||||
// Initialize. Compute paths and create a temporary directory that will be
|
// Initialize. Compute paths and create a temporary directory that will be
|
||||||
// used by internfile()
|
// used by internfile()
|
||||||
BeagleQueueIndexer::BeagleQueueIndexer(RclConfig *cnf, Rcl::Db *db,
|
WebQueueIndexer::WebQueueIndexer(RclConfig *cnf, Rcl::Db *db,
|
||||||
DbIxStatusUpdater *updfunc)
|
DbIxStatusUpdater *updfunc)
|
||||||
: m_config(cnf), m_db(db), m_cache(0), m_updater(updfunc),
|
: m_config(cnf), m_db(db), m_cache(0), m_updater(updfunc),
|
||||||
m_nocacheindex(false)
|
m_nocacheindex(false)
|
||||||
{
|
{
|
||||||
m_queuedir = m_config->getWebQueueDir();
|
m_queuedir = m_config->getWebQueueDir();
|
||||||
path_catslash(m_queuedir);
|
path_catslash(m_queuedir);
|
||||||
m_cache = new BeagleQueueCache(cnf);
|
m_cache = new WebStore(cnf);
|
||||||
}
|
}
|
||||||
|
|
||||||
BeagleQueueIndexer::~BeagleQueueIndexer()
|
WebQueueIndexer::~WebQueueIndexer()
|
||||||
{
|
{
|
||||||
LOGDEB("BeagleQueueIndexer::~\n" );
|
LOGDEB("WebQueueIndexer::~\n" );
|
||||||
deleteZ(m_cache);
|
deleteZ(m_cache);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Index document stored in the cache.
|
// Index document stored in the cache.
|
||||||
bool BeagleQueueIndexer::indexFromCache(const string& udi)
|
bool WebQueueIndexer::indexFromCache(const string& udi)
|
||||||
{
|
{
|
||||||
if (!m_db)
|
if (!m_db)
|
||||||
return false;
|
return false;
|
||||||
@ -202,12 +204,12 @@ bool BeagleQueueIndexer::indexFromCache(const string& udi)
|
|||||||
string hittype;
|
string hittype;
|
||||||
|
|
||||||
if (!m_cache || !m_cache->getFromCache(udi, dotdoc, data, &hittype)) {
|
if (!m_cache || !m_cache->getFromCache(udi, dotdoc, data, &hittype)) {
|
||||||
LOGERR("BeagleQueueIndexer::indexFromCache: cache failed\n" );
|
LOGERR("WebQueueIndexer::indexFromCache: cache failed\n" );
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (hittype.empty()) {
|
if (hittype.empty()) {
|
||||||
LOGERR("BeagleIndexer::index: cc entry has no hit type\n" );
|
LOGERR("WebQueueIndexer::index: cc entry has no hit type\n" );
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -224,11 +226,11 @@ bool BeagleQueueIndexer::indexFromCache(const string& udi)
|
|||||||
try {
|
try {
|
||||||
fis = interner.internfile(doc);
|
fis = interner.internfile(doc);
|
||||||
} catch (CancelExcept) {
|
} catch (CancelExcept) {
|
||||||
LOGERR("BeagleQueueIndexer: interrupted\n" );
|
LOGERR("WebQueueIndexer: interrupted\n" );
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (fis != FileInterner::FIDone) {
|
if (fis != FileInterner::FIDone) {
|
||||||
LOGERR("BeagleQueueIndexer: bad status from internfile\n" );
|
LOGERR("WebQueueIndexer: bad status from internfile\n" );
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -242,7 +244,7 @@ bool BeagleQueueIndexer::indexFromCache(const string& udi)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void BeagleQueueIndexer::updstatus(const string& udi)
|
void WebQueueIndexer::updstatus(const string& udi)
|
||||||
{
|
{
|
||||||
if (m_updater) {
|
if (m_updater) {
|
||||||
++(m_updater->status.docsdone);
|
++(m_updater->status.docsdone);
|
||||||
@ -253,18 +255,18 @@ void BeagleQueueIndexer::updstatus(const string& udi)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool BeagleQueueIndexer::index()
|
bool WebQueueIndexer::index()
|
||||||
{
|
{
|
||||||
if (!m_db)
|
if (!m_db)
|
||||||
return false;
|
return false;
|
||||||
LOGDEB("BeagleQueueIndexer::processqueue: [" << (m_queuedir) << "]\n" );
|
LOGDEB("WebQueueIndexer::processqueue: [" << (m_queuedir) << "]\n" );
|
||||||
m_config->setKeyDir(m_queuedir);
|
m_config->setKeyDir(m_queuedir);
|
||||||
if (!path_makepath(m_queuedir, 0700)) {
|
if (!path_makepath(m_queuedir, 0700)) {
|
||||||
LOGERR("BeagleQueueIndexer:: can't create queuedir [" << (m_queuedir) << "] errno " << (errno) << "\n" );
|
LOGERR("WebQueueIndexer:: can't create queuedir [" << (m_queuedir) << "] errno " << (errno) << "\n" );
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!m_cache || !m_cache->cc()) {
|
if (!m_cache || !m_cache->cc()) {
|
||||||
LOGERR("BeagleQueueIndexer: cache initialization failed\n" );
|
LOGERR("WebQueueIndexer: cache initialization failed\n" );
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
CirCache *cc = m_cache->cc();
|
CirCache *cc = m_cache->cc();
|
||||||
@ -282,7 +284,7 @@ bool BeagleQueueIndexer::index()
|
|||||||
do {
|
do {
|
||||||
string udi;
|
string udi;
|
||||||
if (!cc->getCurrentUdi(udi)) {
|
if (!cc->getCurrentUdi(udi)) {
|
||||||
LOGERR("BeagleQueueIndexer:: cache file damaged\n" );
|
LOGERR("WebQueueIndexer:: cache file damaged\n" );
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if (udi.empty())
|
if (udi.empty())
|
||||||
@ -295,7 +297,7 @@ bool BeagleQueueIndexer::index()
|
|||||||
indexFromCache(udi);
|
indexFromCache(udi);
|
||||||
updstatus(udi);
|
updstatus(udi);
|
||||||
} catch (CancelExcept) {
|
} catch (CancelExcept) {
|
||||||
LOGERR("BeagleQueueIndexer: interrupted\n" );
|
LOGERR("WebQueueIndexer: interrupted\n" );
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -307,17 +309,17 @@ bool BeagleQueueIndexer::index()
|
|||||||
FsTreeWalker walker(FsTreeWalker::FtwNoRecurse);
|
FsTreeWalker walker(FsTreeWalker::FtwNoRecurse);
|
||||||
walker.addSkippedName(".*");
|
walker.addSkippedName(".*");
|
||||||
FsTreeWalker::Status status = walker.walk(m_queuedir, *this);
|
FsTreeWalker::Status status = walker.walk(m_queuedir, *this);
|
||||||
LOGDEB("BeagleQueueIndexer::processqueue: done: status " << (status) << "\n" );
|
LOGDEB("WebQueueIndexer::processqueue: done: status " << (status) << "\n" );
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Index a list of files (sent by the real time monitor)
|
// Index a list of files (sent by the real time monitor)
|
||||||
bool BeagleQueueIndexer::indexFiles(list<string>& files)
|
bool WebQueueIndexer::indexFiles(list<string>& files)
|
||||||
{
|
{
|
||||||
LOGDEB("BeagleQueueIndexer::indexFiles\n" );
|
LOGDEB("WebQueueIndexer::indexFiles\n" );
|
||||||
|
|
||||||
if (!m_db) {
|
if (!m_db) {
|
||||||
LOGERR("BeagleQueueIndexer::indexfiles no db??\n" );
|
LOGERR("WebQueueIndexer::indexfiles no db??\n" );
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
for (list<string>::iterator it = files.begin(); it != files.end();) {
|
for (list<string>::iterator it = files.begin(); it != files.end();) {
|
||||||
@ -326,7 +328,7 @@ bool BeagleQueueIndexer::indexFiles(list<string>& files)
|
|||||||
}
|
}
|
||||||
string father = path_getfather(*it);
|
string father = path_getfather(*it);
|
||||||
if (father.compare(m_queuedir)) {
|
if (father.compare(m_queuedir)) {
|
||||||
LOGDEB("BeagleQueueIndexer::indexfiles: skipping [" << *it << "] (nq)\n" );
|
LOGDEB("WebQueueIndexer::indexfiles: skipping [" << *it << "] (nq)\n" );
|
||||||
it++; continue;
|
it++; continue;
|
||||||
}
|
}
|
||||||
// Pb: we are often called with the dot file, before the
|
// Pb: we are often called with the dot file, before the
|
||||||
@ -342,11 +344,11 @@ bool BeagleQueueIndexer::indexFiles(list<string>& files)
|
|||||||
}
|
}
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (path_fileprops(*it, &st) != 0) {
|
if (path_fileprops(*it, &st) != 0) {
|
||||||
LOGERR("BeagleQueueIndexer::indexfiles: cant stat [" << *it << "]\n" );
|
LOGERR("WebQueueIndexer::indexfiles: cant stat [" << *it << "]\n" );
|
||||||
it++; continue;
|
it++; continue;
|
||||||
}
|
}
|
||||||
if (!S_ISREG(st.st_mode)) {
|
if (!S_ISREG(st.st_mode)) {
|
||||||
LOGDEB("BeagleQueueIndexer::indexfiles: skipping [" << *it << "] (nr)\n" );
|
LOGDEB("WebQueueIndexer::indexfiles: skipping [" << *it << "] (nr)\n" );
|
||||||
it++; continue;
|
it++; continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -360,7 +362,7 @@ bool BeagleQueueIndexer::indexFiles(list<string>& files)
|
|||||||
}
|
}
|
||||||
|
|
||||||
FsTreeWalker::Status
|
FsTreeWalker::Status
|
||||||
BeagleQueueIndexer::processone(const string &path,
|
WebQueueIndexer::processone(const string &path,
|
||||||
const struct stat *stp,
|
const struct stat *stp,
|
||||||
FsTreeWalker::CbFlag flg)
|
FsTreeWalker::CbFlag flg)
|
||||||
{
|
{
|
||||||
@ -374,9 +376,9 @@ BeagleQueueIndexer::processone(const string &path,
|
|||||||
|
|
||||||
string dotpath = path_cat(path_getfather(path),
|
string dotpath = path_cat(path_getfather(path),
|
||||||
string(".") + path_getsimple(path));
|
string(".") + path_getsimple(path));
|
||||||
LOGDEB("BeagleQueueIndexer: prc1: [" << (path) << "]\n" );
|
LOGDEB("WebQueueIndexer: prc1: [" << (path) << "]\n" );
|
||||||
|
|
||||||
BeagleDotFile dotfile(m_config, dotpath);
|
WebQueueDotFile dotfile(m_config, dotpath);
|
||||||
Rcl::Doc dotdoc;
|
Rcl::Doc dotdoc;
|
||||||
string udi, udipath;
|
string udi, udipath;
|
||||||
if (!dotfile.toDoc(dotdoc))
|
if (!dotfile.toDoc(dotdoc))
|
||||||
@ -388,7 +390,7 @@ BeagleQueueIndexer::processone(const string &path,
|
|||||||
udipath = path_cat(dotdoc.meta[Rcl::Doc::keybght], url_gpath(dotdoc.url));
|
udipath = path_cat(dotdoc.meta[Rcl::Doc::keybght], url_gpath(dotdoc.url));
|
||||||
make_udi(udipath, cstr_null, udi);
|
make_udi(udipath, cstr_null, udi);
|
||||||
|
|
||||||
LOGDEB("BeagleQueueIndexer: prc1: udi [" << (udi) << "]\n" );
|
LOGDEB("WebQueueIndexer: prc1: udi [" << (udi) << "]\n" );
|
||||||
char ascdate[30];
|
char ascdate[30];
|
||||||
sprintf(ascdate, "%ld", long(stp->st_mtime));
|
sprintf(ascdate, "%ld", long(stp->st_mtime));
|
||||||
|
|
||||||
@ -410,7 +412,7 @@ BeagleQueueIndexer::processone(const string &path,
|
|||||||
} else {
|
} else {
|
||||||
Rcl::Doc doc;
|
Rcl::Doc doc;
|
||||||
// Store the dotdoc fields in the future doc. In case someone wants
|
// Store the dotdoc fields in the future doc. In case someone wants
|
||||||
// to use beagle-generated fields like beagle:inurl
|
// to use fields generated by the browser plugin like inurl
|
||||||
doc.meta = dotdoc.meta;
|
doc.meta = dotdoc.meta;
|
||||||
|
|
||||||
FileInterner interner(path, stp, m_config,
|
FileInterner interner(path, stp, m_config,
|
||||||
@ -420,11 +422,11 @@ BeagleQueueIndexer::processone(const string &path,
|
|||||||
try {
|
try {
|
||||||
fis = interner.internfile(doc);
|
fis = interner.internfile(doc);
|
||||||
} catch (CancelExcept) {
|
} catch (CancelExcept) {
|
||||||
LOGERR("BeagleQueueIndexer: interrupted\n" );
|
LOGERR("WebQueueIndexer: interrupted\n" );
|
||||||
goto out;
|
goto out;
|
||||||
}
|
}
|
||||||
if (fis != FileInterner::FIDone && fis != FileInterner::FIAgain) {
|
if (fis != FileInterner::FIDone && fis != FileInterner::FIAgain) {
|
||||||
LOGERR("BeagleQueueIndexer: bad status from internfile\n" );
|
LOGERR("WebQueueIndexer: bad status from internfile\n" );
|
||||||
// TOBEDONE: internfile can return FIAgain here if it is
|
// TOBEDONE: internfile can return FIAgain here if it is
|
||||||
// paging a big text file, we should loop. Means we're
|
// paging a big text file, we should loop. Means we're
|
||||||
// only indexing the first page for text/plain files
|
// only indexing the first page for text/plain files
|
||||||
@ -457,11 +459,11 @@ BeagleQueueIndexer::processone(const string &path,
|
|||||||
string fdata;
|
string fdata;
|
||||||
file_to_string(path, fdata);
|
file_to_string(path, fdata);
|
||||||
if (!m_cache || !m_cache->cc()) {
|
if (!m_cache || !m_cache->cc()) {
|
||||||
LOGERR("BeagleQueueIndexer: cache initialization failed\n" );
|
LOGERR("WebQueueIndexer: cache initialization failed\n" );
|
||||||
goto out;
|
goto out;
|
||||||
}
|
}
|
||||||
if (!m_cache->cc()->put(udi, &dotfile.m_fields, fdata, 0)) {
|
if (!m_cache->cc()->put(udi, &dotfile.m_fields, fdata, 0)) {
|
||||||
LOGERR("BeagleQueueIndexer::prc1: cache_put failed; " << (m_cache->cc()->getReason()) << "\n" );
|
LOGERR("WebQueueIndexer::prc1: cache_put failed; " << (m_cache->cc()->getReason()) << "\n" );
|
||||||
goto out;
|
goto out;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -14,18 +14,17 @@
|
|||||||
* Free Software Foundation, Inc.,
|
* Free Software Foundation, Inc.,
|
||||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
*/
|
*/
|
||||||
#ifndef _beaglequeue_h_included_
|
#ifndef _webqueue_h_included_
|
||||||
#define _beaglequeue_h_included_
|
#define _webqueue_h_included_
|
||||||
|
|
||||||
#include <list>
|
#include <list>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Process the Beagle indexing queue.
|
* Process the WEB indexing queue.
|
||||||
*
|
*
|
||||||
* Beagle MUST NOT be running, else mayhem will ensue.
|
* This was originally written to reuse the Beagle Firefox plug-in (which
|
||||||
*
|
* copied visited pages and bookmarks to the queue), long dead and replaced by a
|
||||||
* This is mainly written to reuse the Beagle Firefox plug-in (which
|
* recoll-specific plugin.
|
||||||
* copies visited pages and bookmarks to the queue).
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include "fstreewalk.h"
|
#include "fstreewalk.h"
|
||||||
@ -34,16 +33,16 @@
|
|||||||
class DbIxStatusUpdater;
|
class DbIxStatusUpdater;
|
||||||
class CirCache;
|
class CirCache;
|
||||||
class RclConfig;
|
class RclConfig;
|
||||||
class BeagleQueueCache;
|
class WebStore;
|
||||||
namespace Rcl {
|
namespace Rcl {
|
||||||
class Db;
|
class Db;
|
||||||
}
|
}
|
||||||
|
|
||||||
class BeagleQueueIndexer : public FsTreeWalkerCB {
|
class WebQueueIndexer : public FsTreeWalkerCB {
|
||||||
public:
|
public:
|
||||||
BeagleQueueIndexer(RclConfig *cnf, Rcl::Db *db,
|
WebQueueIndexer(RclConfig *cnf, Rcl::Db *db,
|
||||||
DbIxStatusUpdater *updfunc = 0);
|
DbIxStatusUpdater *updfunc = 0);
|
||||||
~BeagleQueueIndexer();
|
~WebQueueIndexer();
|
||||||
|
|
||||||
/** This is called by the top indexer in recollindex.
|
/** This is called by the top indexer in recollindex.
|
||||||
* Does the walking and the talking */
|
* Does the walking and the talking */
|
||||||
@ -68,7 +67,7 @@ public:
|
|||||||
private:
|
private:
|
||||||
RclConfig *m_config;
|
RclConfig *m_config;
|
||||||
Rcl::Db *m_db;
|
Rcl::Db *m_db;
|
||||||
BeagleQueueCache *m_cache;
|
WebStore *m_cache;
|
||||||
string m_queuedir;
|
string m_queuedir;
|
||||||
DbIxStatusUpdater *m_updater;
|
DbIxStatusUpdater *m_updater;
|
||||||
bool m_nocacheindex;
|
bool m_nocacheindex;
|
||||||
@ -77,4 +76,4 @@ private:
|
|||||||
void updstatus(const string& udi);
|
void updstatus(const string& udi);
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif /* _beaglequeue_h_included_ */
|
#endif /* _webqueue_h_included_ */
|
||||||
|
|||||||
@ -16,23 +16,26 @@
|
|||||||
*/
|
*/
|
||||||
#include "autoconfig.h"
|
#include "autoconfig.h"
|
||||||
|
|
||||||
|
#include "webqueuefetcher.h"
|
||||||
|
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
|
|
||||||
#include "rcldoc.h"
|
#include "rcldoc.h"
|
||||||
#include "fetcher.h"
|
#include "fetcher.h"
|
||||||
#include "bglfetcher.h"
|
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "beaglequeuecache.h"
|
#include "webstore.h"
|
||||||
|
|
||||||
// We use a single beagle cache object to access beagle data. We protect it
|
using std::string;
|
||||||
|
|
||||||
|
// We use a single WebStore object to access the data. We protect it
|
||||||
// against multiple thread access.
|
// against multiple thread access.
|
||||||
static std::mutex o_beagler_mutex;
|
static std::mutex o_beagler_mutex;
|
||||||
|
|
||||||
bool BGLDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
|
bool WQDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
|
||||||
{
|
{
|
||||||
string udi;
|
string udi;
|
||||||
if (!idoc.getmeta(Rcl::Doc::keyudi, &udi) || udi.empty()) {
|
if (!idoc.getmeta(Rcl::Doc::keyudi, &udi) || udi.empty()) {
|
||||||
LOGERR("BGLDocFetcher:: no udi in idoc\n" );
|
LOGERR("WQDocFetcher:: no udi in idoc\n" );
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
Rcl::Doc dotdoc;
|
Rcl::Doc dotdoc;
|
||||||
@ -41,24 +44,23 @@ bool BGLDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
|
|||||||
// Retrieve from our webcache (beagle data). The beagler
|
// Retrieve from our webcache (beagle data). The beagler
|
||||||
// object is created at the first call of this routine and
|
// object is created at the first call of this routine and
|
||||||
// deleted when the program exits.
|
// deleted when the program exits.
|
||||||
static BeagleQueueCache o_beagler(cnf);
|
static WebStore o_beagler(cnf);
|
||||||
if (!o_beagler.getFromCache(udi, dotdoc, out.data)) {
|
if (!o_beagler.getFromCache(udi, dotdoc, out.data)) {
|
||||||
LOGINFO("BGLDocFetcher::fetch: failed for [" << (udi) << "]\n" );
|
LOGINFO("WQDocFetcher::fetch: failed for [" << udi << "]\n");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (dotdoc.mimetype.compare(idoc.mimetype)) {
|
if (dotdoc.mimetype.compare(idoc.mimetype)) {
|
||||||
LOGINFO("BGLDocFetcher:: udi [" << (udi) << "], mimetp mismatch: in: [" << (idoc.mimetype) << "], bgl [" << (dotdoc.mimetype) << "]\n" );
|
LOGINFO("WQDocFetcher:: udi [" << udi << "], mimetp mismatch: in: [" <<
|
||||||
|
idoc.mimetype << "], bgl [" << dotdoc.mimetype << "]\n");
|
||||||
}
|
}
|
||||||
out.kind = RawDoc::RDK_DATA;
|
out.kind = RawDoc::RDK_DATA;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool BGLDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig)
|
bool WQDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig)
|
||||||
{
|
{
|
||||||
// Bgl sigs are empty
|
// Web queue sigs are empty
|
||||||
sig.clear();
|
sig.clear();
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -14,18 +14,19 @@
|
|||||||
* Free Software Foundation, Inc.,
|
* Free Software Foundation, Inc.,
|
||||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
*/
|
*/
|
||||||
#ifndef _BGLFETCHER_H_INCLUDED_
|
#ifndef _WEBQUEUEFETCHER_H_INCLUDED_
|
||||||
#define _BGLFETCHER_H_INCLUDED_
|
#define _WEBQUEUEFETCHER_H_INCLUDED_
|
||||||
|
|
||||||
#include "fetcher.h"
|
#include "fetcher.h"
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Beagle cache fetcher:
|
* The WEB queue cache fetcher:
|
||||||
*/
|
*/
|
||||||
class BGLDocFetcher : public DocFetcher{
|
class WQDocFetcher : public DocFetcher{
|
||||||
virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out);
|
virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out);
|
||||||
virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc,
|
virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc,
|
||||||
std::string& sig);
|
std::string& sig);
|
||||||
virtual ~BGLDocFetcher() {}
|
virtual ~WQDocFetcher() {}
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif /* _BGLFETCHER_H_INCLUDED_ */
|
#endif /* _WEBQUEUEFETCHER_H_INCLUDED_ */
|
||||||
@ -43,7 +43,7 @@
|
|||||||
|
|
||||||
#include "recoll.h"
|
#include "recoll.h"
|
||||||
#include "webcache.h"
|
#include "webcache.h"
|
||||||
#include "beaglequeuecache.h"
|
#include "webstore.h"
|
||||||
#include "circache.h"
|
#include "circache.h"
|
||||||
#include "conftree.h"
|
#include "conftree.h"
|
||||||
#include "rclmain_w.h"
|
#include "rclmain_w.h"
|
||||||
@ -62,7 +62,7 @@ public:
|
|||||||
|
|
||||||
class WebcacheModelInternal {
|
class WebcacheModelInternal {
|
||||||
public:
|
public:
|
||||||
std::shared_ptr<BeagleQueueCache> cache;
|
std::shared_ptr<WebStore> cache;
|
||||||
vector<CEnt> all;
|
vector<CEnt> all;
|
||||||
vector<CEnt> disp;
|
vector<CEnt> disp;
|
||||||
};
|
};
|
||||||
@ -81,7 +81,7 @@ WebcacheModel::~WebcacheModel()
|
|||||||
void WebcacheModel::reload()
|
void WebcacheModel::reload()
|
||||||
{
|
{
|
||||||
m->cache =
|
m->cache =
|
||||||
std::shared_ptr<BeagleQueueCache>(new BeagleQueueCache(theconfig));
|
std::shared_ptr<WebStore>(new WebStore(theconfig));
|
||||||
m->all.clear();
|
m->all.clear();
|
||||||
m->disp.clear();
|
m->disp.clear();
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user