tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

mh_mbox: fix on Windows.

Browse Source 
The 2 main issues were that the files were fopen'd in text mode, and that
the std-based SimpleRegexp implementation was wrong (match instead of search).
Also, make sure that we are using 64 bits interfaces.
This commit is contained in:
Jean-Francois Dockes 2019-08-07 17:07:06 +02:00
parent 4a5bdd98c3
commit ad5beb43c2
3 changed files with 65 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
src/internfile/mh_mbox.cpp
View File

@ -19,7 +19,6 @@
#include <stdio.h> #include <stdio.h>
#include <errno.h> #include <errno.h>
#include <sys/types.h> #include <sys/types.h>
#include "safesysstat.h"
#include <time.h> #include <time.h>
#include <cstring> #include <cstring>
@ -38,6 +37,10 @@
#include "pathut.h" #include "pathut.h"
using namespace std; using namespace std;
#ifdef _WIN32
#define fseeko _fseeki64
#define ftello _ftelli64
#endif
// Define maximum message size for safety. 100MB would seem reasonable // Define maximum message size for safety. 100MB would seem reasonable
static const unsigned int max_mbox_member_size = 100 * 1024 * 1024; static const unsigned int max_mbox_member_size = 100 * 1024 * 1024;
@ -72,16 +75,10 @@ static std::mutex o_mcache_mutex;
* binary, values are not even byte-swapped to be proc-idependant. * binary, values are not even byte-swapped to be proc-idependant.
*/ */
#ifdef _WIN32
// vc++ does not let define an array of size o_b1size because non-const??
#define M_o_b1size 1024 #define M_o_b1size 1024
#else
#define M_o_b1size o_b1size
#endif
class MboxCache { class MboxCache {
public: public:
typedef MimeHandlerMbox::mbhoff_type mbhoff_type;
MboxCache() MboxCache()
: m_ok(false), m_minfsize(0) { : m_ok(false), m_minfsize(0) {
// Can't access rclconfig here, we're a static object, would // Can't access rclconfig here, we're a static object, would
@ -90,7 +87,7 @@ public:
~MboxCache() {} ~MboxCache() {}
mbhoff_type get_offset(RclConfig *config, const string& udi, int msgnum) { int64_t get_offset(RclConfig *config, const string& udi, int msgnum) {
LOGDEB0("MboxCache::get_offsets: udi [" << udi << "] msgnum " LOGDEB0("MboxCache::get_offsets: udi [" << udi << "] msgnum "
<< msgnum << "\n"); << msgnum << "\n");
if (!ok(config)) { if (!ok(config)) {
@ -100,43 +97,44 @@ public:
std::unique_lock<std::mutex> locker(o_mcache_mutex); std::unique_lock<std::mutex> locker(o_mcache_mutex);
string fn = makefilename(udi); string fn = makefilename(udi);
FILE *fp = 0; FILE *fp = 0;
if ((fp = fopen(fn.c_str(), "r")) == 0) { if ((fp = fopen(fn.c_str(), "rb")) == 0) {
LOGSYSERR("MboxCache::get_offsets", "open", fn); LOGSYSERR("MboxCache::get_offsets", "open", fn);
return -1; return -1;
} }
FpKeeper keeper(&fp); FpKeeper keeper(&fp);
char blk1[M_o_b1size]; char blk1[M_o_b1size];
if (fread(blk1, 1, o_b1size, fp) != o_b1size) { if (fread(blk1, M_o_b1size, 1, fp) != 1) {
LOGSYSERR("MboxCache::get_offsets", "read blk1", ""); LOGSYSERR("MboxCache::get_offsets", "read blk1", "");
return -1; return -1;
} }
ConfSimple cf(string(blk1, o_b1size)); ConfSimple cf(string(blk1, M_o_b1size));
string fudi; string fudi;
if (!cf.get("udi", fudi) || fudi.compare(udi)) { if (!cf.get("udi", fudi) || fudi.compare(udi)) {
LOGINFO("MboxCache::get_offset:badudi fn " << fn << " udi [" LOGINFO("MboxCache::get_offset:badudi fn " << fn << " udi ["
<< udi << "], fudi [" << fudi << "]\n"); << udi << "], fudi [" << fudi << "]\n");
return -1; return -1;
} }
LOGDEB1("MboxCache::get_offsets: reading offsets file at offs "
<< cacheoffset(msgnum) << "\n");
if (fseeko(fp, cacheoffset(msgnum), SEEK_SET) != 0) { if (fseeko(fp, cacheoffset(msgnum), SEEK_SET) != 0) {
LOGSYSERR("MboxCache::get_offsets", "seek", LOGSYSERR("MboxCache::get_offsets", "seek",
lltodecstr(cacheoffset(msgnum))); lltodecstr(cacheoffset(msgnum)));
return -1; return -1;
} }
mbhoff_type offset = -1; int64_t offset = -1;
size_t ret; size_t ret;
if ((ret = fread(&offset, 1, sizeof(mbhoff_type), fp)) if ((ret = fread(&offset, sizeof(int64_t), 1, fp)) != 1) {
!= sizeof(mbhoff_type)) {
LOGSYSERR("MboxCache::get_offsets", "read", ""); LOGSYSERR("MboxCache::get_offsets", "read", "");
return -1; return -1;
} }
LOGDEB0("MboxCache::get_offsets: ret " << lltodecstr(offset) << "\n"); LOGDEB0("MboxCache::get_offsets: ret " << offset << "\n");
return offset; return offset;
} }
// Save array of offsets for a given file, designated by Udi // Save array of offsets for a given file, designated by Udi
void put_offsets(RclConfig *config, const string& udi, mbhoff_type fsize, void put_offsets(RclConfig *config, const string& udi, int64_t fsize,
vector<mbhoff_type>& offs) { vector<int64_t>& offs) {
LOGDEB0("MboxCache::put_offsets: " << offs.size() << " offsets\n"); LOGDEB0("MboxCache::put_offsets: " << offs.size() << " offsets\n");
if (!ok(config) || !maybemakedir()) if (!ok(config) || !maybemakedir())
return; return;
@ -148,7 +146,7 @@ public:
std::unique_lock<std::mutex> locker(o_mcache_mutex); std::unique_lock<std::mutex> locker(o_mcache_mutex);
string fn = makefilename(udi); string fn = makefilename(udi);
FILE *fp; FILE *fp;
if ((fp = fopen(fn.c_str(), "w")) == 0) { if ((fp = fopen(fn.c_str(), "wb")) == 0) {
LOGSYSERR("MboxCache::put_offsets", "fopen", fn); LOGSYSERR("MboxCache::put_offsets", "fopen", fn);
return; return;
} }
@ -156,15 +154,16 @@ public:
string blk1("udi="); string blk1("udi=");
blk1.append(udi); blk1.append(udi);
blk1.append(cstr_newline); blk1.append(cstr_newline);
blk1.resize(o_b1size, 0); blk1.resize(M_o_b1size, 0);
if (fwrite(blk1.c_str(), 1, o_b1size, fp) != o_b1size) { if (fwrite(blk1.c_str(), M_o_b1size, 1, fp) != 1) {
LOGSYSERR("MboxCache::put_offsets", "fwrite blk1", ""); LOGSYSERR("MboxCache::put_offsets", "fwrite blk1", "");
return; return;
} }
for (const auto& off : offs) { for (const auto& off : offs) {
if (fwrite((char*)&off, 1, sizeof(mbhoff_type), fp) != LOGDEB1("MboxCache::put_offsets: writing value " << off <<
sizeof(mbhoff_type)) { " at offset " << ftello(fp) << endl);
if (fwrite((char*)&off, sizeof(int64_t), 1, fp) != 1) {
LOGSYSERR("MboxCache::put_offsets", "fwrite", ""); LOGSYSERR("MboxCache::put_offsets", "fwrite", "");
return; return;
} }
@ -198,8 +197,7 @@ private:
// Place where we store things // Place where we store things
string m_dir; string m_dir;
// Don't cache smaller files. If -1, don't do anything. // Don't cache smaller files. If -1, don't do anything.
mbhoff_type m_minfsize; int64_t m_minfsize;
static const size_t o_b1size;
// Create the cache directory if it does not exist // Create the cache directory if it does not exist
bool maybemakedir() { bool maybemakedir() {
@ -219,12 +217,11 @@ private:
// Compute offset in cache file for the mbox offset of msgnum // Compute offset in cache file for the mbox offset of msgnum
// Msgnums are from 1 // Msgnums are from 1
mbhoff_type cacheoffset(int msgnum) { int64_t cacheoffset(int msgnum) {
return o_b1size + (msgnum-1) * sizeof(mbhoff_type); return M_o_b1size + (msgnum-1) * sizeof(int64_t);
} }
}; };
const size_t MboxCache::o_b1size = 1024;
static class MboxCache o_mcache; static class MboxCache o_mcache;
static const string cstr_keyquirks("mhmboxquirks"); static const string cstr_keyquirks("mhmboxquirks");
@ -255,9 +252,9 @@ bool MimeHandlerMbox::set_document_file_impl(const string& mt, const string &fn)
m_vfp = 0; m_vfp = 0;
} }
m_vfp = fopen(fn.c_str(), "r"); m_vfp = fopen(fn.c_str(), "rb");
if (m_vfp == 0) { if (m_vfp == 0) {
LOGSYSERR("MimeHandlerMail::set_document_file", "fopen r", fn); LOGSYSERR("MimeHandlerMail::set_document_file", "fopen rb", fn);
return false; return false;
} }
#if defined O_NOATIME && O_NOATIME != 0 #if defined O_NOATIME && O_NOATIME != 0
@ -266,13 +263,7 @@ bool MimeHandlerMbox::set_document_file_impl(const string& mt, const string &fn)
} }
#endif #endif
// Used to use ftell() here: no good beyond 2GB // Used to use ftell() here: no good beyond 2GB
{struct stat st; m_fsize = path_filesize(fn);
if (fstat(fileno((FILE*)m_vfp), &st) < 0) {
LOGSYSERR("MimeHandlerMbox:setdocfile", "fstat", fn);
return false;
}
m_fsize = st.st_size;
}
m_havedoc = true; m_havedoc = true;
m_offsets.clear(); m_offsets.clear();
m_quirks = 0; m_quirks = 0;
@ -408,21 +399,35 @@ bool MimeHandlerMbox::next_document()
// object). So: // object). So:
bool storeoffsets = true; bool storeoffsets = true;
if (mtarg > 0) { if (mtarg > 0) {
mbhoff_type off; int64_t off;
line_type line; line_type line;
LOGDEB0("MimeHandlerMbox::next_doc: mtarg " << mtarg << " m_udi[" << LOGDEB0("MimeHandlerMbox::next_doc: mtarg " << mtarg << " m_udi[" <<
m_udi << "]\n"); m_udi << "]\n");
if (!m_udi.empty() && if (!m_udi.empty()) {
(off = o_mcache.get_offset(m_config, m_udi, mtarg)) >= 0 && LOGDEB("MimeHandlerMbox::next_doc: udi not empty\n");
fseeko(fp, (off_t)off, SEEK_SET) >= 0 && if ((off = o_mcache.get_offset(m_config, m_udi, mtarg)) >= 0) {
fgets(line, LL, fp) && LOGDEB1("MimeHandlerMbox::next_doc: got offset " << off <<
(fromregex(line) || ((m_quirks & MBOXQUIRK_TBIRD) && " from cache\n");
minifromregex(line))) ) { if (fseeko(fp, off, SEEK_SET) >= 0) {
LOGDEB0("MimeHandlerMbox: Cache: From_ Ok\n"); LOGDEB1("MimeHandlerMbox::next_doc: fseeko ok\n");
fseeko(fp, (off_t)off, SEEK_SET); if (fgets(line, LL, fp)) {
m_msgnum = mtarg -1; LOGDEB1("MimeHandlerMbox::next_doc: fgets ok. line:[" <<
storeoffsets = false; line << "]\n");
} else { if ((fromregex(line) || ((m_quirks & MBOXQUIRK_TBIRD) &&
minifromregex(line))) ) {
LOGDEB0("MimeHandlerMbox: Cache: From_ Ok\n");
fseeko(fp, off, SEEK_SET);
m_msgnum = mtarg -1;
storeoffsets = false;
} else {
LOGDEB0("MimeHandlerMbox: cache: regex failed\n");
}
}
}
}
}
if (storeoffsets) {
// No cached result: scan.
fseek(fp, 0, SEEK_SET); fseek(fp, 0, SEEK_SET);
m_msgnum = 0; m_msgnum = 0;
} }
@ -466,8 +471,10 @@ bool MimeHandlerMbox::next_document()
LOGDEB0("MimeHandlerMbox: msgnum " << m_msgnum << LOGDEB0("MimeHandlerMbox: msgnum " << m_msgnum <<
", From_ at line " << m_lineno << ": [" << line ", From_ at line " << m_lineno << ": [" << line
<< "]\n"); << "]\n");
if (storeoffsets) if (storeoffsets) {
LOGDEB1("Pushing offset: " << message_end << endl);
m_offsets.push_back(message_end); m_offsets.push_back(message_end);
}
m_msgnum++; m_msgnum++;
if ((mtarg <= 0 && m_msgnum > 1) || if ((mtarg <= 0 && m_msgnum > 1) ||
(mtarg > 0 && m_msgnum > mtarg)) { (mtarg > 0 && m_msgnum > mtarg)) {

8
src/internfile/mh_mbox.h
View File

@ -19,6 +19,7 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include <inttypes.h>
#include "mimehandler.h" #include "mimehandler.h"
@ -40,7 +41,6 @@ public:
return true; return true;
} }
virtual void clear_impl() override; virtual void clear_impl() override;
typedef long long mbhoff_type;
protected: protected:
virtual bool set_document_file_impl(const std::string&, virtual bool set_document_file_impl(const std::string&,
@ -51,9 +51,9 @@ private:
void *m_vfp; // File pointer for folder void *m_vfp; // File pointer for folder
int m_msgnum; // Current message number in folder. Starts at 1 int m_msgnum; // Current message number in folder. Starts at 1
std::string m_ipath; std::string m_ipath;
int m_lineno; // debug int64_t m_lineno; // debug
mbhoff_type m_fsize; int64_t m_fsize;
std::vector<mbhoff_type> m_offsets; std::vector<int64_t> m_offsets;
enum Quirks {MBOXQUIRK_TBIRD=1}; enum Quirks {MBOXQUIRK_TBIRD=1};
int m_quirks; int m_quirks;
}; };

10
src/utils/pathut.cpp
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2004 J.F.Dockes /* Copyright (C) 2004-2019 J.F.Dockes
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by * it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or * the Free Software Foundation; either version 2.1 of the License, or
@ -57,9 +57,9 @@
#include "safesysstat.h" #include "safesysstat.h"
#include "transcode.h" #include "transcode.h"
#define STAT _wstat #define STAT _wstati64
#define LSTAT _wstat #define LSTAT _wstati64
#define STATBUF _stat #define STATBUF _stati64
#define ACCESS _waccess #define ACCESS _waccess
#else // Not windows -> #else // Not windows ->
@ -656,7 +656,7 @@ bool path_makepath(const string& ipath, int mode)
path = "/"; path = "/";
for (const auto& elem : elems) { for (const auto& elem : elems) {
#ifdef _WIN32 #ifdef _WIN32
if (path == "/" && path_strlookslikedrive(*it)) { if (path == "/" && path_strlookslikedrive(elem)) {
path = ""; path = "";
} }
#endif #endif