tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

mh_mbox ckpt before switching to streams

Browse Source
This commit is contained in:
Jean-Francois Dockes 2019-08-08 09:40:51 +02:00
parent ad5beb43c2
commit 11152d2c9a
2 changed files with 138 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

265
src/internfile/mh_mbox.cpp
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2005 J.F.Dockes /* Copyright (C) 2005-2019 J.F.Dockes
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or * the Free Software Foundation; either version 2 of the License, or
@ -37,6 +37,7 @@
#include "pathut.h" #include "pathut.h"
using namespace std; using namespace std;
#ifdef _WIN32 #ifdef _WIN32
#define fseeko _fseeki64 #define fseeko _fseeki64
#define ftello _ftelli64 #define ftello _ftelli64
@ -45,6 +46,69 @@ using namespace std;
// Define maximum message size for safety. 100MB would seem reasonable // Define maximum message size for safety. 100MB would seem reasonable
static const unsigned int max_mbox_member_size = 100 * 1024 * 1024; static const unsigned int max_mbox_member_size = 100 * 1024 * 1024;
// The mbox format uses lines beginning with 'From ' as separator.
// Mailers are supposed to quote any other lines beginning with
// 'From ', turning it into '>From '. This should make it easy to detect
// message boundaries by matching a '^From ' regular expression
// Unfortunately this quoting is quite often incorrect in the real world.
//
// The rest of the format for the line is somewhat variable, but there will
// be a 4 digit year somewhere...
// The canonic format is the following, with a 24 characters date:
// From toto@tutu.com Sat Sep 30 16:44:06 2000
// This resulted into the pattern for versions up to 1.9.0:
// "^From .* [1-2][0-9][0-9][0-9]$"
//
// Some mailers add a time zone to the date, this is non-"standard",
// but happens, like in:
// From toto@truc.com Sat Sep 30 16:44:06 2000 -0400
//
// This is taken into account in the new regexp, which also matches more
// of the date format, to catch a few actual issues like
// From http://www.itu.int/newsroom/press/releases/1998/NP-2.html:
// Note that this *should* have been quoted.
//
// http://www.qmail.org/man/man5/mbox.html seems to indicate that the
// fact that From_ is normally preceded by a blank line should not be
// used, but we do it anyway (for now).
// The same source indicates that arbitrary data can follow the date field
//
// A variety of pathologic From_ lines:
// Bad date format:
// From uucp Wed May 22 11:28 GMT 1996
// Added timezone at the end (ok, part of the "any data" after the date)
// From qian2@fas.harvard.edu Sat Sep 30 16:44:06 2000 -0400
// Emacs VM botch ? Adds tz between hour and year
// From dockes Wed Feb 23 10:31:20 +0100 2005
// From dockes Fri Dec 1 20:36:39 +0100 2006
// The modified regexp gives the exact same results on the ietf mail archive
// and my own's.
// Update, 2008-08-29: some old? Thunderbird versions apparently use a date
// in "Date: " header format, like: From - Mon, 8 May 2006 10:57:32
// This was added as an alternative format. By the way it also fools "mail" and
// emacs-vm, Recoll is not alone
// Update: 2009-11-27: word after From may be quoted string: From "john bull"
static const string frompat{
"^From[ ]+([^ ]+|\"[^\"]+\")[ ]+" // 'From (toto@tutu|"john bull") '
"[[:alpha:]]{3}[ ]+[[:alpha:]]{3}[ ]+[0-3 ][0-9][ ]+" // Fri Oct 26
"[0-2][0-9]:[0-5][0-9](:[0-5][0-9])?[ ]+" // Time, seconds optional
"([^ ]+[ ]+)?" // Optional tz
"[12][0-9][0-9][0-9]" // Year, unanchored, more data may follow
"|" // Or standard mail Date: header format
"^From[ ]+[^ ]+[ ]+" // From toto@tutu
"[[:alpha:]]{3},[ ]+[0-3]?[0-9][ ]+[[:alpha:]]{3}[ ]+" // Mon, 8 May
"[12][0-9][0-9][0-9][ ]+" // Year
"[0-2][0-9]:[0-5][0-9](:[0-5][0-9])?" // Time, secs optional
};
// Extreme thunderbird brokiness. Will sometimes use From lines
// exactly like: From ^M (From followed by space and eol). We only
// test for this if QUIRKS_TBIRD is set
static const string miniTbirdFrom{"^From $"};
static SimpleRegexp fromregex(frompat, SimpleRegexp::SRE_NOSUB);
static SimpleRegexp minifromregex(miniTbirdFrom, SimpleRegexp::SRE_NOSUB);
// Automatic fp closing // Automatic fp closing
class FpKeeper { class FpKeeper {
public: public:
@ -79,11 +143,10 @@ static std::mutex o_mcache_mutex;
class MboxCache { class MboxCache {
public: public:
MboxCache() MboxCache() {
: m_ok(false), m_minfsize(0) { // Can't access rclconfig here, we're a static object, would
// Can't access rclconfig here, we're a static object, would // have to make sure it's initialized.
// have to make sure it's initialized. }
}
~MboxCache() {} ~MboxCache() {}
@ -192,12 +255,11 @@ public:
} }
private: private:
bool m_ok; bool m_ok{false};
// Place where we store things // Place where we store things
string m_dir; string m_dir;
// Don't cache smaller files. If -1, don't do anything. // Don't cache smaller files. If -1, don't do anything.
int64_t m_minfsize; int64_t m_minfsize{0};
// Create the cache directory if it does not exist // Create the cache directory if it does not exist
bool maybemakedir() { bool maybemakedir() {
@ -214,7 +276,6 @@ private:
MD5HexPrint(digest, xdigest); MD5HexPrint(digest, xdigest);
return path_cat(m_dir, xdigest); return path_cat(m_dir, xdigest);
} }
// Compute offset in cache file for the mbox offset of msgnum // Compute offset in cache file for the mbox offset of msgnum
// Msgnums are from 1 // Msgnums are from 1
int64_t cacheoffset(int msgnum) { int64_t cacheoffset(int msgnum) {
@ -234,18 +295,20 @@ MimeHandlerMbox::~MimeHandlerMbox()
void MimeHandlerMbox::clear_impl() void MimeHandlerMbox::clear_impl()
{ {
m_fn.erase(); m_fn.erase();
m_ipath.erase();
if (m_vfp) { if (m_vfp) {
fclose((FILE *)m_vfp); fclose((FILE *)m_vfp);
m_vfp = 0; m_vfp = 0;
} }
m_msgnum = m_lineno = 0; m_msgnum = m_lineno = m_fsize = 0;
m_ipath.erase();
m_offsets.clear(); m_offsets.clear();
m_quirks = 0;
} }
bool MimeHandlerMbox::set_document_file_impl(const string& mt, const string &fn) bool MimeHandlerMbox::set_document_file_impl(const string& mt, const string &fn)
{ {
LOGDEB("MimeHandlerMbox::set_document_file(" << fn << ")\n"); LOGDEB("MimeHandlerMbox::set_document_file(" << fn << ")\n");
clear_impl();
m_fn = fn; m_fn = fn;
if (m_vfp) { if (m_vfp) {
fclose((FILE *)m_vfp); fclose((FILE *)m_vfp);
@ -253,7 +316,7 @@ bool MimeHandlerMbox::set_document_file_impl(const string& mt, const string &fn)
} }
m_vfp = fopen(fn.c_str(), "rb"); m_vfp = fopen(fn.c_str(), "rb");
if (m_vfp == 0) { if (m_vfp == nullptr) {
LOGSYSERR("MimeHandlerMail::set_document_file", "fopen rb", fn); LOGSYSERR("MimeHandlerMail::set_document_file", "fopen rb", fn);
return false; return false;
} }
@ -262,12 +325,10 @@ bool MimeHandlerMbox::set_document_file_impl(const string& mt, const string &fn)
// perror("fcntl"); // perror("fcntl");
} }
#endif #endif
// Used to use ftell() here: no good beyond 2GB
m_fsize = path_filesize(fn); m_fsize = path_filesize(fn);
m_havedoc = true; m_havedoc = true;
m_offsets.clear();
m_quirks = 0;
// Check for location-based quirks: // Check for location-based quirks:
string quirks; string quirks;
if (m_config && m_config->getConfParam(cstr_keyquirks, quirks)) { if (m_config && m_config->getConfParam(cstr_keyquirks, quirks)) {
@ -279,16 +340,15 @@ bool MimeHandlerMbox::set_document_file_impl(const string& mt, const string &fn)
// And double check for thunderbird // And double check for thunderbird
string tbirdmsf = fn + ".msf"; string tbirdmsf = fn + ".msf";
if ((m_quirks&MBOXQUIRK_TBIRD) == 0 && path_exists(tbirdmsf)) { if (!(m_quirks & MBOXQUIRK_TBIRD) && path_exists(tbirdmsf)) {
LOGDEB("MimeHandlerMbox: detected unconfigured tbird mbox in " << fn << LOGDEB("MimeHandlerMbox: detected unconf'd tbird mbox in " << fn <<"\n");
"\n");
m_quirks |= MBOXQUIRK_TBIRD; m_quirks |= MBOXQUIRK_TBIRD;
} }
return true; return true;
} }
#define LL 1024 #define LL 20000
typedef char line_type[LL+10]; typedef char line_type[LL+10];
static inline void stripendnl(line_type& line, int& ll) static inline void stripendnl(line_type& line, int& ll)
{ {
@ -302,79 +362,60 @@ static inline void stripendnl(line_type& line, int& ll)
} }
} }
// The mbox format uses lines beginning with 'From ' as separator. bool MimeHandlerMbox::tryUseCache(int mtarg)
// Mailers are supposed to quote any other lines beginning with {
// 'From ', turning it into '>From '. This should make it easy to detect bool cachefound = false;
// message boundaries by matching a '^From ' regular expression
// Unfortunately this quoting is quite often incorrect in the real world. int64_t off;
// line_type line;
// The rest of the format for the line is somewhat variable, but there will LOGDEB0("MimeHandlerMbox::next_doc: mtarg " << mtarg << " m_udi[" <<
// be a 4 digit year somewhere... m_udi << "]\n");
// The canonic format is the following, with a 24 characters date: if (m_udi.empty()) {
// From toto@tutu.com Sat Sep 30 16:44:06 2000 goto out;
// This resulted into the pattern for versions up to 1.9.0: }
// "^From .* [1-2][0-9][0-9][0-9]$" if ((off = o_mcache.get_offset(m_config, m_udi, mtarg)) < 0) {
// goto out;
// Some mailers add a time zone to the date, this is non-"standard", }
// but happens, like in: LOGDEB1("MimeHandlerMbox::next_doc: got offset " << off <<
// From toto@truc.com Sat Sep 30 16:44:06 2000 -0400 " from cache\n");
// if (fseeko((FILE*)m_vfp, off, SEEK_SET) < 0) {
// This is taken into account in the new regexp, which also matches more goto out;
// of the date format, to catch a few actual issues like }
// From http://www.itu.int/newsroom/press/releases/1998/NP-2.html: LOGDEB1("MimeHandlerMbox::next_doc: fseeko ok\n");
// Note that this *should* have been quoted. if (!fgets(line, LL, (FILE*)m_vfp)) {
// goto out;
// http://www.qmail.org/man/man5/mbox.html seems to indicate that the }
// fact that From_ is normally preceded by a blank line should not be LOGDEB1("MimeHandlerMbox::next_doc: fgets ok. line:[" << line << "]\n");
// used, but we do it anyway (for now).
// The same source indicates that arbitrary data can follow the date field
//
// A variety of pathologic From_ lines:
// Bad date format:
// From uucp Wed May 22 11:28 GMT 1996
// Added timezone at the end (ok, part of the "any data" after the date)
// From qian2@fas.harvard.edu Sat Sep 30 16:44:06 2000 -0400
// Emacs VM botch ? Adds tz between hour and year
// From dockes Wed Feb 23 10:31:20 +0100 2005
// From dockes Fri Dec 1 20:36:39 +0100 2006
// The modified regexp gives the exact same results on the ietf mail archive
// and my own's.
// Update, 2008-08-29: some old? Thunderbird versions apparently use a date
// in "Date: " header format, like: From - Mon, 8 May 2006 10:57:32
// This was added as an alternative format. By the way it also fools "mail" and
// emacs-vm, Recoll is not alone
// Update: 2009-11-27: word after From may be quoted string: From "john bull"
static const string frompat{
"^From[ ]+([^ ]+|\"[^\"]+\")[ ]+" // 'From (toto@tutu|"john bull") '
"[[:alpha:]]{3}[ ]+[[:alpha:]]{3}[ ]+[0-3 ][0-9][ ]+" // Fri Oct 26
"[0-2][0-9]:[0-5][0-9](:[0-5][0-9])?[ ]+" // Time, seconds optional
"([^ ]+[ ]+)?" // Optional tz
"[12][0-9][0-9][0-9]" // Year, unanchored, more data may follow
"|" // Or standard mail Date: header format
"^From[ ]+[^ ]+[ ]+" // From toto@tutu
"[[:alpha:]]{3},[ ]+[0-3]?[0-9][ ]+[[:alpha:]]{3}[ ]+" // Mon, 8 May
"[12][0-9][0-9][0-9][ ]+" // Year
"[0-2][0-9]:[0-5][0-9](:[0-5][0-9])?" // Time, secs optional
};
// Extreme thunderbird brokiness. Will sometimes use From lines if ((fromregex(line) ||
// exactly like: From ^M (From followed by space and eol). We only ((m_quirks & MBOXQUIRK_TBIRD) && minifromregex(line))) ) {
// test for this if QUIRKS_TBIRD is set LOGDEB0("MimeHandlerMbox: Cache: From_ Ok\n");
static const string miniTbirdFrom{"^From $"}; fseeko((FILE*)m_vfp, off, SEEK_SET);
m_msgnum = mtarg -1;
cachefound = true;
} else {
LOGDEB0("MimeHandlerMbox: cache: regex failed for [" << line << "]\n");
}
static SimpleRegexp fromregex(frompat, SimpleRegexp::SRE_NOSUB); out:
static SimpleRegexp minifromregex(miniTbirdFrom, SimpleRegexp::SRE_NOSUB); if (!cachefound) {
// No cached result: scan.
fseek((FILE*)m_vfp, 0, SEEK_SET);
m_msgnum = 0;
}
return cachefound;
}
bool MimeHandlerMbox::next_document() bool MimeHandlerMbox::next_document()
{ {
if (m_vfp == 0) { if (nullptr == m_vfp) {
LOGERR("MimeHandlerMbox::next_document: not open\n"); LOGERR("MimeHandlerMbox::next_document: not open\n");
return false; return false;
} }
if (!m_havedoc) { if (!m_havedoc) {
return false; return false;
} }
FILE *fp = (FILE *)m_vfp;
int mtarg = 0; int mtarg = 0;
if (!m_ipath.empty()) { if (!m_ipath.empty()) {
sscanf(m_ipath.c_str(), "%d", &mtarg); sscanf(m_ipath.c_str(), "%d", &mtarg);
@ -385,63 +426,27 @@ bool MimeHandlerMbox::next_document()
} }
LOGDEB0("MimeHandlerMbox::next_document: fn " << m_fn << ", msgnum " << LOGDEB0("MimeHandlerMbox::next_document: fn " << m_fn << ", msgnum " <<
m_msgnum << " mtarg " << mtarg << " \n"); m_msgnum << " mtarg " << mtarg << " \n");
if (mtarg == 0) if (mtarg == 0)
mtarg = -1; mtarg = -1;
// If we are called to retrieve a specific message, try to use the
// If we are called to retrieve a specific message, seek to bof // offsets cache to try and position to the right header.
// (then scan up to the message). This is for the case where the
// same object is reused to fetch several messages (else the fp is
// just opened no need for a seek). We could also check if the
// current message number is lower than the requested one and
// avoid rereading the whole thing in this case. But I'm not sure
// we're ever used in this way (multiple retrieves on same
// object). So:
bool storeoffsets = true; bool storeoffsets = true;
if (mtarg > 0) { if (mtarg > 0) {
int64_t off; storeoffsets = !tryUseCache(mtarg);
line_type line;
LOGDEB0("MimeHandlerMbox::next_doc: mtarg " << mtarg << " m_udi[" <<
m_udi << "]\n");
if (!m_udi.empty()) {
LOGDEB("MimeHandlerMbox::next_doc: udi not empty\n");
if ((off = o_mcache.get_offset(m_config, m_udi, mtarg)) >= 0) {
LOGDEB1("MimeHandlerMbox::next_doc: got offset " << off <<
" from cache\n");
if (fseeko(fp, off, SEEK_SET) >= 0) {
LOGDEB1("MimeHandlerMbox::next_doc: fseeko ok\n");
if (fgets(line, LL, fp)) {
LOGDEB1("MimeHandlerMbox::next_doc: fgets ok. line:[" <<
line << "]\n");
if ((fromregex(line) || ((m_quirks & MBOXQUIRK_TBIRD) &&
minifromregex(line))) ) {
LOGDEB0("MimeHandlerMbox: Cache: From_ Ok\n");
fseeko(fp, off, SEEK_SET);
m_msgnum = mtarg -1;
storeoffsets = false;
} else {
LOGDEB0("MimeHandlerMbox: cache: regex failed\n");
}
}
}
}
}
if (storeoffsets) {
// No cached result: scan.
fseek(fp, 0, SEEK_SET);
m_msgnum = 0;
}
} }
off_t message_end = 0; int64_t message_end = 0;
int64_t message_end1 = 0;
bool iseof = false; bool iseof = false;
bool hademptyline = true; bool hademptyline = true;
string& msgtxt = m_metaData[cstr_dj_keycontent]; string& msgtxt = m_metaData[cstr_dj_keycontent];
msgtxt.erase(); msgtxt.erase();
line_type line; line_type line;
for (;;) { for (;;) {
message_end = ftello(fp); message_end = ftello((FILE*)m_vfp);
if (!fgets(line, LL, fp)) { if (!fgets(line, LL, (FILE*)m_vfp)) {
LOGDEB2("MimeHandlerMbox:next: eof\n"); LOGDEB2("MimeHandlerMbox:next: eof\n");
iseof = true; iseof = true;
m_msgnum++; m_msgnum++;
@ -469,10 +474,10 @@ bool MimeHandlerMbox::next_document()
((m_quirks & MBOXQUIRK_TBIRD) && minifromregex(line))) ((m_quirks & MBOXQUIRK_TBIRD) && minifromregex(line)))
) { ) {
LOGDEB0("MimeHandlerMbox: msgnum " << m_msgnum << LOGDEB0("MimeHandlerMbox: msgnum " << m_msgnum <<
", From_ at line " << m_lineno << ": [" << line ", From_ at line " << m_lineno << " foffset " <<
<< "]\n"); message_end << " line: [" << line << "]\n");
if (storeoffsets) { if (storeoffsets) {
LOGDEB1("Pushing offset: " << message_end << endl);
m_offsets.push_back(message_end); m_offsets.push_back(message_end);
} }
m_msgnum++; m_msgnum++;

4
src/internfile/mh_mbox.h
View File

@ -48,14 +48,16 @@ protected:
private: private:
std::string m_fn; // File name std::string m_fn; // File name
std::string m_ipath;
void *m_vfp; // File pointer for folder void *m_vfp; // File pointer for folder
int m_msgnum; // Current message number in folder. Starts at 1 int m_msgnum; // Current message number in folder. Starts at 1
std::string m_ipath;
int64_t m_lineno; // debug int64_t m_lineno; // debug
int64_t m_fsize; int64_t m_fsize;
std::vector<int64_t> m_offsets; std::vector<int64_t> m_offsets;
enum Quirks {MBOXQUIRK_TBIRD=1}; enum Quirks {MBOXQUIRK_TBIRD=1};
int m_quirks; int m_quirks;
bool tryUseCache(int mtarg);
}; };
#endif /* _MBOX_H_INCLUDED_ */ #endif /* _MBOX_H_INCLUDED_ */