tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

merge

Browse Source
This commit is contained in:
Jean-Francois Dockes 2019-08-08 13:50:21 +02:00
commit d88b2a8bc9
1 changed files with 77 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
src/internfile/mh_mbox.cpp
View File

@ -14,9 +14,10 @@
* Free Software Foundation, Inc., * Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/ */
#include "autoconfig.h"
#include <stdio.h> #include "autoconfig.h"
#define _FILE_OFFSET_BITS 64
#include <errno.h> #include <errno.h>
#include <sys/types.h> #include <sys/types.h>
#include <time.h> #include <time.h>
@ -24,11 +25,11 @@
#include <cstring> #include <cstring>
#include <map> #include <map>
#include <mutex> #include <mutex>
#include <fstream>
#include "cstr.h" #include "cstr.h"
#include "mimehandler.h" #include "mimehandler.h"
#include "log.h" #include "log.h"
#include "readfile.h"
#include "mh_mbox.h" #include "mh_mbox.h"
#include "smallut.h" #include "smallut.h"
#include "rclconfig.h" #include "rclconfig.h"
@ -38,11 +39,6 @@
using namespace std; using namespace std;
#ifdef _WIN32
#define fseeko _fseeki64
#define ftello _ftelli64
#endif
// Define maximum message size for safety. 100MB would seem reasonable // Define maximum message size for safety. 100MB would seem reasonable
static const unsigned int max_mbox_member_size = 100 * 1024 * 1024; static const unsigned int max_mbox_member_size = 100 * 1024 * 1024;
@ -109,21 +105,6 @@ static const string miniTbirdFrom{"^From $"};
static SimpleRegexp fromregex(frompat, SimpleRegexp::SRE_NOSUB); static SimpleRegexp fromregex(frompat, SimpleRegexp::SRE_NOSUB);
static SimpleRegexp minifromregex(miniTbirdFrom, SimpleRegexp::SRE_NOSUB); static SimpleRegexp minifromregex(miniTbirdFrom, SimpleRegexp::SRE_NOSUB);
// Automatic fp closing
class FpKeeper {
public:
FpKeeper(FILE **fpp)
: m_fpp(fpp) {}
~FpKeeper() {
if (m_fpp && *m_fpp) {
fclose(*m_fpp);
*m_fpp = 0;
}
}
private:
FILE **m_fpp;
};
static std::mutex o_mcache_mutex; static std::mutex o_mcache_mutex;
/** /**
@ -154,20 +135,19 @@ public:
LOGDEB0("MboxCache::get_offsets: udi [" << udi << "] msgnum " LOGDEB0("MboxCache::get_offsets: udi [" << udi << "] msgnum "
<< msgnum << "\n"); << msgnum << "\n");
if (!ok(config)) { if (!ok(config)) {
LOGDEB0("MboxCache::get_offsets: init failed\n"); LOGDEB("MboxCache::get_offsets: init failed\n");
return -1; return -1;
} }
std::unique_lock<std::mutex> locker(o_mcache_mutex); std::unique_lock<std::mutex> locker(o_mcache_mutex);
string fn = makefilename(udi); string fn = makefilename(udi);
FILE *fp = 0; ifstream instream(fn.c_str(), std::ifstream::binary);
if ((fp = fopen(fn.c_str(), "rb")) == 0) { if (!instream.good()) {
LOGSYSERR("MboxCache::get_offsets", "open", fn); LOGSYSERR("MboxCache::get_offsets", "open", fn);
return -1; return false;
} }
FpKeeper keeper(&fp);
char blk1[M_o_b1size]; char blk1[M_o_b1size];
if (fread(blk1, M_o_b1size, 1, fp) != 1) { instream.read(blk1, M_o_b1size);
if (!instream.good()) {
LOGSYSERR("MboxCache::get_offsets", "read blk1", ""); LOGSYSERR("MboxCache::get_offsets", "read blk1", "");
return -1; return -1;
} }
@ -180,14 +160,15 @@ public:
} }
LOGDEB1("MboxCache::get_offsets: reading offsets file at offs " LOGDEB1("MboxCache::get_offsets: reading offsets file at offs "
<< cacheoffset(msgnum) << "\n"); << cacheoffset(msgnum) << "\n");
if (fseeko(fp, cacheoffset(msgnum), SEEK_SET) != 0) { instream.seekg(cacheoffset(msgnum));
if (!instream.good()) {
LOGSYSERR("MboxCache::get_offsets", "seek", LOGSYSERR("MboxCache::get_offsets", "seek",
lltodecstr(cacheoffset(msgnum))); lltodecstr(cacheoffset(msgnum)));
return -1; return -1;
} }
int64_t offset = -1; int64_t offset = -1;
size_t ret; instream.read((char*)&offset, sizeof(int64_t));
if ((ret = fread(&offset, sizeof(int64_t), 1, fp)) != 1) { if (!instream.good()) {
LOGSYSERR("MboxCache::get_offsets", "read", ""); LOGSYSERR("MboxCache::get_offsets", "read", "");
return -1; return -1;
} }
@ -202,35 +183,41 @@ public:
if (!ok(config) || !maybemakedir()) if (!ok(config) || !maybemakedir())
return; return;
if (fsize < m_minfsize) { if (fsize < m_minfsize) {
LOGDEB0("MboxCache::put_offsets: fsize " << fsize << " < minsize " << LOGDEB0("MboxCache::put_offsets: fsize " << fsize << " < minsize "
m_minfsize << endl); << m_minfsize << endl);
return; return;
} }
std::unique_lock<std::mutex> locker(o_mcache_mutex); std::unique_lock<std::mutex> locker(o_mcache_mutex);
string fn = makefilename(udi); string fn = makefilename(udi);
FILE *fp; std::ofstream os(fn.c_str(), std::ios::out|std::ios::binary);
if ((fp = fopen(fn.c_str(), "wb")) == 0) { if (!os.good()) {
LOGSYSERR("MboxCache::put_offsets", "fopen", fn); LOGSYSERR("MboxCache::put_offsets", "open", fn);
return; return;
} }
FpKeeper keeper(&fp);
string blk1("udi="); string blk1("udi=");
blk1.append(udi); blk1.append(udi);
blk1.append(cstr_newline); blk1.append(cstr_newline);
blk1.resize(M_o_b1size, 0); blk1.resize(M_o_b1size, 0);
if (fwrite(blk1.c_str(), M_o_b1size, 1, fp) != 1) { os.write(blk1.c_str(), M_o_b1size);
LOGSYSERR("MboxCache::put_offsets", "fwrite blk1", ""); if (!os.good()) {
LOGSYSERR("MboxCache::put_offsets", "write blk1", "");
return; return;
} }
for (const auto& off : offs) { for (const auto& off : offs) {
LOGDEB1("MboxCache::put_offsets: writing value " << off << LOGDEB1("MboxCache::put_offsets: writing value " << off <<
" at offset " << ftello(fp) << endl); " at offset " << ftello(fp) << endl);
if (fwrite((char*)&off, sizeof(int64_t), 1, fp) != 1) { os.write((char*)&off, sizeof(int64_t));
LOGSYSERR("MboxCache::put_offsets", "fwrite", ""); if (!os.good()) {
LOGSYSERR("MboxCache::put_offsets", "write", "");
return; return;
} }
} }
os.flush();
if (!os.good()) {
LOGSYSERR("MboxCache::put_offsets", "flush", "");
return;
}
} }
// Check state, possibly initialize // Check state, possibly initialize
@ -294,7 +281,7 @@ public:
Internal(MimeHandlerMbox *p) : pthis(p) {} Internal(MimeHandlerMbox *p) : pthis(p) {}
std::string fn; // File name std::string fn; // File name
std::string ipath; std::string ipath;
void *vfp{nullptr}; // File pointer for folder ifstream instream;
int msgnum{0}; // Current message number in folder. Starts at 1 int msgnum{0}; // Current message number in folder. Starts at 1
int64_t lineno{0}; // debug int64_t lineno{0}; // debug
int64_t fsize{0}; int64_t fsize{0};
@ -322,10 +309,7 @@ void MimeHandlerMbox::clear_impl()
{ {
m->fn.erase(); m->fn.erase();
m->ipath.erase(); m->ipath.erase();
if (m->vfp) { m->instream = ifstream();
fclose((FILE *)m->vfp);
m->vfp = 0;
}
m->msgnum = m->lineno = m->fsize = 0; m->msgnum = m->lineno = m->fsize = 0;
m->offsets.clear(); m->offsets.clear();
m->quirks = 0; m->quirks = 0;
@ -341,17 +325,14 @@ bool MimeHandlerMbox::set_document_file_impl(const string& mt, const string &fn)
LOGDEB("MimeHandlerMbox::set_document_file(" << fn << ")\n"); LOGDEB("MimeHandlerMbox::set_document_file(" << fn << ")\n");
clear_impl(); clear_impl();
m->fn = fn; m->fn = fn;
if (m->vfp) { m->instream = ifstream(fn.c_str(), std::ifstream::binary);
fclose((FILE *)m->vfp); if (!m->instream.good()) {
m->vfp = 0; LOGSYSERR("MimeHandlerMail::set_document_file", "ifstream", fn);
}
m->vfp = fopen(fn.c_str(), "rb");
if (m->vfp == nullptr) {
LOGSYSERR("MimeHandlerMail::set_document_file", "fopen rb", fn);
return false; return false;
} }
#if defined O_NOATIME && O_NOATIME != 0
// TBD
#if 0 && defined O_NOATIME && O_NOATIME != 0
if (fcntl(fileno((FILE *)m->vfp), F_SETFL, O_NOATIME) < 0) { if (fcntl(fileno((FILE *)m->vfp), F_SETFL, O_NOATIME) < 0) {
// perror("fcntl"); // perror("fcntl");
} }
@ -372,33 +353,19 @@ bool MimeHandlerMbox::set_document_file_impl(const string& mt, const string &fn)
// And double check for thunderbird // And double check for thunderbird
string tbirdmsf = fn + ".msf"; string tbirdmsf = fn + ".msf";
if (!(m->quirks & MBOXQUIRK_TBIRD) && path_exists(tbirdmsf)) { if (!(m->quirks & MBOXQUIRK_TBIRD) && path_exists(tbirdmsf)) {
LOGDEB("MimeHandlerMbox: detected unconf'd tbird mbox in " << fn <<"\n"); LOGDEB("MimeHandlerMbox: detected unconf'd tbird mbox in "<< fn <<"\n");
m->quirks |= MBOXQUIRK_TBIRD; m->quirks |= MBOXQUIRK_TBIRD;
} }
return true; return true;
} }
#define LL 20000
typedef char line_type[LL+10];
static inline void stripendnl(line_type& line, int& ll)
{
ll = int(strlen(line));
while (ll > 0) {
if (line[ll-1] == '\n' || line[ll-1] == '\r') {
line[ll-1] = 0;
ll--;
} else
break;
}
}
bool MimeHandlerMbox::Internal::tryUseCache(int mtarg) bool MimeHandlerMbox::Internal::tryUseCache(int mtarg)
{ {
bool cachefound = false; bool cachefound = false;
string line;
int64_t off; int64_t off;
line_type line;
LOGDEB0("MimeHandlerMbox::next_doc: mtarg " << mtarg << " m_udi[" << LOGDEB0("MimeHandlerMbox::next_doc: mtarg " << mtarg << " m_udi[" <<
pthis->m_udi << "]\n"); pthis->m_udi << "]\n");
if (pthis->m_udi.empty()) { if (pthis->m_udi.empty()) {
@ -407,21 +374,22 @@ bool MimeHandlerMbox::Internal::tryUseCache(int mtarg)
if ((off = o_mcache.get_offset(pthis->m_config, pthis->m_udi, mtarg)) < 0) { if ((off = o_mcache.get_offset(pthis->m_config, pthis->m_udi, mtarg)) < 0) {
goto out; goto out;
} }
LOGDEB1("MimeHandlerMbox::next_doc: got offset " << off << instream.seekg(off);
" from cache\n"); if (!instream.good()) {
if (fseeko((FILE*)vfp, off, SEEK_SET) < 0) { LOGSYSERR("tryUseCache", "seekg", "");
goto out; goto out;
} }
LOGDEB1("MimeHandlerMbox::next_doc: fseeko ok\n"); getline(instream, line, '\n');
if (!fgets(line, LL, (FILE*)vfp)) { if (!instream.good()) {
LOGSYSERR("tryUseCache", "getline", "");
goto out; goto out;
} }
LOGDEB1("MimeHandlerMbox::next_doc: fgets ok. line:[" << line << "]\n"); LOGDEB1("MimeHandlerMbox::tryUseCache:getl ok. line:[" << line << "]\n");
if ((fromregex(line) || if ((fromregex(line) ||
((quirks & MBOXQUIRK_TBIRD) && minifromregex(line))) ) { ((quirks & MBOXQUIRK_TBIRD) && minifromregex(line))) ) {
LOGDEB0("MimeHandlerMbox: Cache: From_ Ok\n"); LOGDEB0("MimeHandlerMbox: Cache: From_ Ok\n");
fseeko((FILE*)vfp, off, SEEK_SET); instream.seekg(off);
msgnum = mtarg -1; msgnum = mtarg -1;
cachefound = true; cachefound = true;
} else { } else {
@ -431,7 +399,7 @@ bool MimeHandlerMbox::Internal::tryUseCache(int mtarg)
out: out:
if (!cachefound) { if (!cachefound) {
// No cached result: scan. // No cached result: scan.
fseek((FILE*)vfp, 0, SEEK_SET); instream.seekg(0);
msgnum = 0; msgnum = 0;
} }
return cachefound; return cachefound;
@ -439,7 +407,7 @@ out:
bool MimeHandlerMbox::next_document() bool MimeHandlerMbox::next_document()
{ {
if (nullptr == m->vfp) { if (!m->instream.good()) {
LOGERR("MimeHandlerMbox::next_document: not open\n"); LOGERR("MimeHandlerMbox::next_document: not open\n");
return false; return false;
} }
@ -469,25 +437,40 @@ bool MimeHandlerMbox::next_document()
} }
int64_t message_end = 0; int64_t message_end = 0;
int64_t message_end1 = 0;
bool iseof = false; bool iseof = false;
bool hademptyline = true; bool hademptyline = true;
string& msgtxt = m_metaData[cstr_dj_keycontent]; string& msgtxt = m_metaData[cstr_dj_keycontent];
msgtxt.erase(); msgtxt.erase();
line_type line; string line;
for (;;) { for (;;) {
message_end = ftello((FILE*)m->vfp); message_end = m->instream.tellg();
if (!fgets(line, LL, (FILE*)m->vfp)) { getline(m->instream, line, '\n');
LOGDEB2("MimeHandlerMbox:next: eof\n"); if (!m->instream.good()) {
ifstream::iostate st = m->instream.rdstate();
if (st & std::ifstream::eofbit) {
LOGDEB0("MimeHandlerMbox:next: eof\n");
}
if (st & std::ifstream::failbit) {
LOGDEB0("MimeHandlerMbox:next: fail\n");
LOGSYSERR("MimeHandlerMbox:next:", "", "");
}
if (st & std::ifstream::badbit) {
LOGDEB0("MimeHandlerMbox:next: bad\n");
LOGSYSERR("MimeHandlerMbox:next:", "", "");
}
if (st & std::ifstream::goodbit) {
LOGDEB0("MimeHandlerMbox:next: good\n");
}
LOGDEB0("MimeHandlerMbox:next: eof at " << message_end << endl);
iseof = true; iseof = true;
m->msgnum++; m->msgnum++;
break; break;
} }
m->lineno++; m->lineno++;
int ll; rtrimstring(line, "\r\n");
stripendnl(line, ll); int ll = line.size();
LOGDEB2("mhmbox:next: hadempty " << hademptyline << " lineno " << LOGDEB2("mhmbox:next: hadempty " << hademptyline << " lineno " <<
m->lineno << " ll " << ll << " Line: [" << line << "]\n"); m->lineno << " ll " << ll << " Line: [" << line << "]\n");
if (hademptyline) { if (hademptyline) {
if (ll > 0) { if (ll > 0) {
// Non-empty line with empty line flag set, reset flag // Non-empty line with empty line flag set, reset flag
@ -504,7 +487,7 @@ bool MimeHandlerMbox::next_document()
fromregex(line) || fromregex(line) ||
((m->quirks & MBOXQUIRK_TBIRD) && minifromregex(line))) ((m->quirks & MBOXQUIRK_TBIRD) && minifromregex(line)))
) { ) {
LOGDEB0("MimeHandlerMbox: msgnum " << m->msgnum << LOGDEB1("MimeHandlerMbox: msgnum " << m->msgnum <<
", From_ at line " << m->lineno << " foffset " << ", From_ at line " << m->lineno << " foffset " <<
message_end << " line: [" << line << "]\n"); message_end << " line: [" << line << "]\n");
@ -527,8 +510,7 @@ bool MimeHandlerMbox::next_document()
if (mtarg <= 0 || m->msgnum == mtarg) { if (mtarg <= 0 || m->msgnum == mtarg) {
// Accumulate message lines // Accumulate message lines
line[ll] = '\n'; line += '\n';
line[ll+1] = 0;
msgtxt += line; msgtxt += line;
if (msgtxt.size() > max_mbox_member_size) { if (msgtxt.size() > max_mbox_member_size) {
LOGERR("mh_mbox: huge message (more than " << LOGERR("mh_mbox: huge message (more than " <<