tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix problems that occurred when multiple threads were trying to read/convert files at the same time (ie: indexing and previewing threads in the GUI calling internfile()). Either get rid of or lock-protect all shared data, eliminate misc initialization possible conflicts by using static initializers. Hopefuly closes issue #51

Browse Source
This commit is contained in:
Jean-Francois Dockes
2011-04-28 10:58:33 +02:00
parent 01f24fa5fd
commit 55f124725f
33 changed files with 318 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/internfile/internfile.cpp
+36 -25
View File
@@ -45,6 +45,7 @@ using namespace std;
#include "beaglequeuecache.h"
#include "cancelcheck.h"
#include "copyfile.h"
#include "ptmutex.h"
#ifdef RCL_USE_XATTR
#include "pxattr.h"
@@ -76,9 +77,6 @@ static string colon_restore(const string& in)
return out;
}
set<string> FileInterner::o_missingExternal;
map<string, set<string> > FileInterner::o_typesForMissing;
#ifdef RCL_USE_XATTR
void FileInterner::reapXAttrs(const string& path)
{
@@ -192,7 +190,7 @@ void FileInterner::tmpcleanup()
FileInterner::FileInterner(const string &f, const struct stat *stp,
RclConfig *cnf,
TempDir& td, int flags, const string *imime)
: m_tdir(td), m_ok(false)
: m_tdir(td), m_ok(false), m_missingdatap(0)
{
initcommon(cnf, flags);
init(f, stp, cnf, flags, imime);
@@ -304,7 +302,7 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
// Setup from memory data (ie: out of the web cache). imime needs to be set.
FileInterner::FileInterner(const string &data, RclConfig *cnf,
TempDir& td, int flags, const string& imime)
: m_tdir(td), m_ok(false)
: m_tdir(td), m_ok(false), m_missingdatap(0)
{
initcommon(cnf, flags);
init(data, cnf, flags, imime);
@@ -366,9 +364,13 @@ void FileInterner::initcommon(RclConfig *cnf, int flags)
m_targetMType = stxtplain;
}
// We used a single beagle cache object to access beagle data. We protect it
// against multiple thread access.
static PTMutexInit o_lock;
FileInterner::FileInterner(const Rcl::Doc& idoc, RclConfig *cnf,
TempDir& td, int flags)
: m_tdir(td), m_ok(false)
: m_tdir(td), m_ok(false), m_missingdatap(0)
{
LOGDEB(("FileInterner::FileInterner(idoc)\n"));
initcommon(cnf, flags);
@@ -405,10 +407,6 @@ FileInterner::FileInterner(const Rcl::Doc& idoc, RclConfig *cnf,
}
init(fn, &st, cnf, flags, &idoc.mimetype);
} else if (!backend.compare("BGL")) {
// Retrieve from our webcache (beagle data). The beagler
// object is created at the first call of this routine and
// deleted when the program exits.
static BeagleQueueCache beagler(cnf);
string data;
Rcl::Doc dotdoc;
map<string,string>::const_iterator it =
@@ -418,11 +416,19 @@ FileInterner::FileInterner(const Rcl::Doc& idoc, RclConfig *cnf,
return;
}
string udi = it->second;
if (!beagler.getFromCache(udi, dotdoc, data)) {
LOGINFO(("FileInterner:: failed fetch from Beagle cache for [%s]\n",
udi.c_str()));
return;
}
{
PTMutexLocker locker(o_lock);
// Retrieve from our webcache (beagle data). The beagler
// object is created at the first call of this routine and
// deleted when the program exits.
static BeagleQueueCache beagler(cnf);
if (!beagler.getFromCache(udi, dotdoc, data)) {
LOGINFO(("FileInterner:: failed fetch from Beagle cache for [%s]\n",
udi.c_str()));
return;
}
}
if (dotdoc.mimetype.compare(idoc.mimetype)) {
LOGINFO(("FileInterner:: udi [%s], mimetp mismatch: in: [%s], bgl "
"[%s]\n", idoc.mimetype.c_str(), dotdoc.mimetype.c_str()));
@@ -485,7 +491,7 @@ bool FileInterner::dataToTempFile(const string& dt, const string& mt,
// accumulate helper name if it is
void FileInterner::checkExternalMissing(const string& msg, const string& mt)
{
if (msg.find("RECFILTERROR") == 0) {
if (m_missingdatap && msg.find("RECFILTERROR") == 0) {
list<string> lerr;
stringToStrings(msg, lerr);
if (lerr.size() > 2) {
@@ -495,28 +501,33 @@ void FileInterner::checkExternalMissing(const string& msg, const string& mt)
lerr.erase(it++);
string s;
stringsToString(lerr, s);
o_missingExternal.insert(s);
o_typesForMissing[s].insert(mt);
m_missingdatap->m_missingExternal.insert(s);
m_missingdatap->m_typesForMissing[s].insert(mt);
}
}
}
}
void FileInterner::getMissingExternal(string& out)
void FileInterner::getMissingExternal(FIMissingStore *st, string& out)
{
stringsToString(o_missingExternal, out);
if (st)
stringsToString(st->m_missingExternal, out);
}
void FileInterner::getMissingDescription(string& out)
void FileInterner::getMissingDescription(FIMissingStore *st, string& out)
{
if (st == 0)
return;
out.erase();
for (set<string>::const_iterator it = o_missingExternal.begin();
it != o_missingExternal.end(); it++) {
for (set<string>::const_iterator it =
st->m_missingExternal.begin();
it != st->m_missingExternal.end(); it++) {
out += *it;
map<string, set<string> >::const_iterator it2;
it2 = o_typesForMissing.find(*it);
if (it2 != o_typesForMissing.end()) {
it2 = st->m_typesForMissing.find(*it);
if (it2 != st->m_typesForMissing.end()) {
out += " (";
set<string>::const_iterator it3;
for (it3 = it2->second.begin();
src/internfile/internfile.h
+21 -6
View File
@@ -39,6 +39,19 @@ class Doc;
struct stat;
/** Storage for missing helper program info. We want to keep this out of the
* FileInterner class, because the data will typically be accumulated by several
* FileInterner objects. Can't use static member either (because there
* may be several separate usages of the class which shouldn't mix
* their data).
*/
class FIMissingStore {
public:
// Missing external programs
set<string> m_missingExternal;
map<string, set<string> > m_typesForMissing;
};
/**
* A class to convert data from a datastore (file-system, firefox
* history, etc.) into possibly one or severaldocuments in internal
@@ -110,6 +123,11 @@ class FileInterner {
~FileInterner();
void setMissingStore(FIMissingStore *st)
{
m_missingdatap = st;
}
/**
* Turn file or file part into Recoll document.
*
@@ -169,11 +187,10 @@ class FileInterner {
RclConfig *cnf, const Rcl::Doc& doc);
const string& getReason() const {return m_reason;}
static void getMissingExternal(string& missing);
static void getMissingDescription(string& desc);
static void getMissingExternal(FIMissingStore *st, string& missing);
static void getMissingDescription(FIMissingStore *st, string& desc);
bool ok() {return m_ok;}
private:
static const unsigned int MAXHANDLERS = 20;
RclConfig *m_cfg;
@@ -203,9 +220,7 @@ class FileInterner {
vector<TempFile> m_tempfiles;
// Error data if any
string m_reason;
// Missing external programs
static set<string> o_missingExternal;
static map<string, set<string> > o_typesForMissing;
FIMissingStore *m_missingdatap;
// Pseudo-constructors
void init(const string &fn, const struct stat *stp,
src/internfile/mh_mbox.cpp
+12 -2
View File
@@ -37,6 +37,7 @@
#include "rclconfig.h"
#include "md5.h"
#include "conftree.h"
#include "ptmutex.h"
using namespace std;
class FpKeeper {
@@ -52,6 +53,7 @@ public:
private: FILE **m_fpp;
};
static PTMutexInit o_mutex;
/**
* Handles a cache for message numbers to offset translations. Permits direct
@@ -78,6 +80,7 @@ public:
~MboxCache() {}
mbhoff_type get_offset(RclConfig *config, const string& udi, int msgnum)
{
PTMutexLocker locker(o_mutex);
LOGDEB0(("MboxCache::get_offsets: udi [%s] msgnum %d\n", udi.c_str(),
msgnum));
if (!ok(config)) {
@@ -125,6 +128,7 @@ public:
void put_offsets(RclConfig *config, const string& udi, mbhoff_type fsize,
vector<mbhoff_type>& offs)
{
PTMutexLocker locker(o_mutex);
LOGDEB0(("MboxCache::put_offsets: %u offsets\n", offs.size()));
if (!ok(config) || !maybemakedir())
return;
@@ -159,6 +163,7 @@ public:
// Check state, possibly initialize
bool ok(RclConfig *config) {
PTMutexLocker locker(o_mutex);
if (m_minfsize == -1)
return false;
if (!m_ok) {
@@ -218,7 +223,9 @@ private:
};
const size_t MboxCache::o_b1size = 1024;
static class MboxCache mcache;
static const string keyquirks("mhmboxquirks");
MimeHandlerMbox::~MimeHandlerMbox()
@@ -389,6 +396,7 @@ bool MimeHandlerMbox::next_document()
// avoid rereading the whole thing in this case. But I'm not sure
// we're ever used in this way (multiple retrieves on same
// object). So:
bool storeoffsets = true;
if (mtarg > 0) {
mbhoff_type off;
line_type line;
@@ -404,6 +412,7 @@ bool MimeHandlerMbox::next_document()
LOGDEB0(("MimeHandlerMbox: Cache: From_ Ok\n"));
fseeko(fp, (off_t)off, SEEK_SET);
m_msgnum = mtarg -1;
storeoffsets = false;
} else {
fseek(fp, 0, SEEK_SET);
m_msgnum = 0;
@@ -444,7 +453,8 @@ bool MimeHandlerMbox::next_document()
!regexec(&minifromregex, line, 0, 0, 0)) ) {
LOGDEB1(("MimeHandlerMbox: msgnum %d, "
"From_ at line %d: [%s]\n", m_msgnum, m_lineno, line));
m_offsets.push_back(message_end);
if (storeoffsets)
m_offsets.push_back(message_end);
m_msgnum++;
if ((mtarg <= 0 && m_msgnum > 1) ||
(mtarg > 0 && m_msgnum > mtarg)) {
@@ -477,7 +487,7 @@ bool MimeHandlerMbox::next_document()
if (iseof) {
LOGDEB2(("MimeHandlerMbox::next: eof hit\n"));
m_havedoc = false;
if (!m_udi.empty()) {
if (!m_udi.empty() && storeoffsets) {
mcache.put_offsets(m_config, m_udi, m_fsize, m_offsets);
}
}
src/internfile/mimehandler.cpp
+5 -24
View File
@@ -28,7 +28,6 @@ using namespace std;
#include "debuglog.h"
#include "rclconfig.h"
#include "smallut.h"
#include "pthread.h"
#include "mh_exec.h"
#include "mh_execm.h"
@@ -37,6 +36,7 @@ using namespace std;
#include "mh_mbox.h"
#include "mh_text.h"
#include "mh_unknown.h"
#include "ptmutex.h"
// Performance help: we use a pool of already known and created
// handlers. There can be several instances for a given mime type
@@ -48,26 +48,7 @@ using namespace std;
// simple lock should be enough as handlers are removed from the cache
// while in use and multiple copies are allowed
static multimap<string, Dijon::Filter*> o_handlers;
pthread_mutex_t o_handlers_mutex;
class HandlersLocker {
public:
HandlersLocker()
{
pthread_mutex_lock(&o_handlers_mutex);
}
~HandlersLocker()
{
pthread_mutex_unlock(&o_handlers_mutex);
}
};
class HandlersLockerInit {
public:
HandlersLockerInit()
{
pthread_mutex_init(&o_handlers_mutex, 0);
}
};
static HandlersLockerInit o_hli;
static PTMutexInit o_handlers_mutex;
/** For mime types set as "internal" in mimeconf:
* create appropriate handler object. */
@@ -163,7 +144,7 @@ void returnMimeHandler(Dijon::Filter *handler)
typedef multimap<string, Dijon::Filter*>::value_type value_type;
if (handler) {
handler->clear();
HandlersLocker locker;
PTMutexLocker locker(o_handlers_mutex);
o_handlers.insert(value_type(handler->get_mime_type(), handler));
}
}
@@ -172,7 +153,7 @@ void clearMimeHandlerCache()
{
typedef multimap<string, Dijon::Filter*>::value_type value_type;
map<string, Dijon::Filter *>::iterator it;
HandlersLocker locker;
PTMutexLocker locker(o_handlers_mutex);
for (it = o_handlers.begin(); it != o_handlers.end(); it++) {
delete it->second;
}
@@ -187,7 +168,7 @@ Dijon::Filter *getMimeHandler(const string &mtype, RclConfig *cfg,
LOGDEB2(("getMimeHandler: mtype [%s] filtertypes %d\n",
mtype.c_str(), filtertypes));
Dijon::Filter *h = 0;
HandlersLocker locker;
PTMutexLocker locker(o_handlers_mutex);
// Get handler definition for mime type. We do this even if an
// appropriate handler object may be in the cache (indexed by mime
src/internfile/myhtmlparse.cpp
+19 -16
View File
@@ -35,8 +35,6 @@
#include "debuglog.h"
#include "transcode.h"
map<string, string> MyHtmlParser::my_named_ents;
inline static bool
p_notdigit(char c)
{
@@ -151,20 +149,11 @@ static const char *epairs[] = {
"rsaquo", "\xe2\x80\xba", "euro", "\xe2\x82\xac",
NULL, NULL
};
MyHtmlParser::MyHtmlParser()
: in_script_tag(false),
in_style_tag(false),
in_body_tag(false),
in_pre_tag(false),
pending_space(false),
indexing_allowed(true)
{
// The default html document charset is iso-8859-1. We'll update
// this value from the encoding tag if found.
charset = "iso-8859-1";
if (my_named_ents.empty()) {
map<string, string> my_named_ents;
class NamedEntsInitializer {
public:
NamedEntsInitializer()
{
for (int i = 0;;) {
const char *ent;
const char *val;
@@ -177,6 +166,20 @@ MyHtmlParser::MyHtmlParser()
my_named_ents[string(ent)] = val;
}
}
};
static NamedEntsInitializer namedEntsInitializerInstance;
MyHtmlParser::MyHtmlParser()
: in_script_tag(false),
in_style_tag(false),
in_body_tag(false),
in_pre_tag(false),
pending_space(false),
indexing_allowed(true)
{
// The default html document charset is iso-8859-1. We'll update
// this value from the encoding tag if found.
charset = "iso-8859-1";
}
void MyHtmlParser::decode_entities(string &s)
src/internfile/myhtmlparse.h
-1
View File
@@ -40,7 +40,6 @@ class MyHtmlParser : public HtmlParser {
bool in_pre_tag;
bool pending_space;
map<string,string> meta;
static map<string, string> my_named_ents;
string dump, dmtime;
// This is the charset our caller thinks the doc used (initially
// comes from the environment/configuration, used as source for