tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

TempFile: clean-up interface by using internal ref-counted class member. Uncomp: add interface to clear cache

Browse Source
This commit is contained in:
Jean-Francois Dockes
2018-05-17 10:24:01 +02:00
parent 40fd63662b
commit d69d2abbde
12 changed files with 147 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/internfile/internfile.cpp
+25 -25
View File
@@ -47,6 +47,7 @@ using namespace std;
#include "copyfile.h"
#include "fetcher.h"
#include "extrameta.h"
#include "uncomp.h"
// The internal path element separator. This can't be the same as the rcldb
// file to ipath separator : "|"
@@ -188,7 +189,7 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
int maxkbs = -1;
if (!m_cfg->getConfParam("compressedfilemaxkbs", &maxkbs) ||
maxkbs < 0 || !stp || int(stp->st_size / 1024) < maxkbs) {
if (!m_uncomp.uncompressfile(m_fn, ucmd, m_tfile)) {
if (!m_uncomp->uncompressfile(m_fn, ucmd, m_tfile)) {
return;
}
LOGDEB1("FileInterner:: after ucomp: tfile " << m_tfile <<"\n");
@@ -293,8 +294,8 @@ void FileInterner::init(const string &data, RclConfig *cnf,
result = df->set_document_data(m_mimetype, data.c_str(), data.length());
} else if (df->is_data_input_ok(Dijon::Filter::DOCUMENT_FILE_NAME)) {
TempFile temp = dataToTempFile(data, m_mimetype);
if (temp &&
(result = df->set_document_file(m_mimetype, temp->filename()))) {
if (temp.ok() &&
(result = df->set_document_file(m_mimetype, temp.filename()))) {
m_tmpflgs[m_handlers.size()] = true;
m_tempfiles.push_back(temp);
}
@@ -312,7 +313,8 @@ void FileInterner::init(const string &data, RclConfig *cnf,
void FileInterner::initcommon(RclConfig *cnf, int flags)
{
m_cfg = cnf;
m_uncomp = m_forPreview = ((flags & FIF_forPreview) != 0);
m_forPreview = ((flags & FIF_forPreview) != 0);
m_uncomp = new Uncomp(m_forPreview);
// Initialize handler stack.
m_handlers.reserve(MAXHANDLERS);
for (unsigned int i = 0; i < MAXHANDLERS; i++)
@@ -373,10 +375,10 @@ bool FileInterner::makesig(RclConfig *cnf, const Rcl::Doc& idoc, string& sig)
FileInterner::~FileInterner()
{
for (vector<RecollFilter*>::iterator it = m_handlers.begin();
it != m_handlers.end(); it++) {
returnMimeHandler(*it);
for (auto& entry: m_handlers) {
returnMimeHandler(entry);
}
delete m_uncomp;
// m_tempfiles will take care of itself
}
@@ -386,14 +388,14 @@ FileInterner::~FileInterner()
TempFile FileInterner::dataToTempFile(const string& dt, const string& mt)
{
// Create temp file with appropriate suffix for mime type
TempFile temp(new TempFileInternal(m_cfg->getSuffixFromMimeType(mt)));
if (!temp->ok()) {
TempFile temp(m_cfg->getSuffixFromMimeType(mt));
if (!temp.ok()) {
LOGERR("FileInterner::dataToTempFile: cant create tempfile: " <<
temp->getreason() << "\n");
temp.getreason() << "\n");
return TempFile();
}
string reason;
if (!stringtofile(dt, temp->filename(), reason)) {
if (!stringtofile(dt, temp.filename(), reason)) {
LOGERR("FileInterner::dataToTempFile: stringtofile: " <<reason << "\n");
return TempFile();
}
@@ -723,8 +725,8 @@ int FileInterner::addHandler()
setres = newflt->set_document_data(mimetype,txt->c_str(),txt->length());
} else if (newflt->is_data_input_ok(Dijon::Filter::DOCUMENT_FILE_NAME)) {
TempFile temp = dataToTempFile(*txt, mimetype);
if (temp &&
(setres = newflt->set_document_file(mimetype, temp->filename()))) {
if (temp.ok() &&
(setres = newflt->set_document_file(mimetype, temp.filename()))) {
m_tmpflgs[m_handlers.size()] = true;
m_tempfiles.push_back(temp);
// Hack here, but really helps perfs: if we happen to
@@ -765,7 +767,7 @@ FileInterner::Status FileInterner::internfile(Rcl::Doc& doc,const string& ipath)
LOGDEB("FileInterner::internfile. ipath [" << ipath << "]\n");
// Get rid of possible image tempfile from older call
m_imgtmp.reset();
m_imgtmp = TempFile();
if (m_handlers.size() < 1) {
// Just means the constructor failed
@@ -916,9 +918,8 @@ FileInterner::Status FileInterner::internfile(Rcl::Doc& doc,const string& ipath)
bool FileInterner::tempFileForMT(TempFile& otemp, RclConfig* cnf,
const string& mimetype)
{
TempFile temp(new TempFileInternal(
cnf->getSuffixFromMimeType(mimetype)));
if (!temp->ok()) {
TempFile temp(cnf->getSuffixFromMimeType(mimetype));
if (!temp.ok()) {
LOGERR("FileInterner::tempFileForMT: can't create temp file\n");
return false;
}
@@ -970,7 +971,7 @@ bool FileInterner::topdocToFile(
if (!tempFileForMT(temp, cnf, idoc.mimetype)) {
return false;
}
filename = temp->filename();
filename = temp.filename();
} else {
filename = tofile.c_str();
}
@@ -985,7 +986,7 @@ bool FileInterner::topdocToFile(
return false;
}
}
fn = temp ? temp->filename() : rawdoc.data;
fn = temp.ok() ? temp.filename() : rawdoc.data;
if (!copyfile(fn.c_str(), filename, reason)) {
LOGERR("FileInterner::idocToFile: copyfile: " << reason << "\n");
return false;
@@ -1040,7 +1041,7 @@ bool FileInterner::interntofile(TempFile& otemp, const string& tofile,
if (!tempFileForMT(temp, m_cfg, mimetype)) {
return false;
}
filename = temp->filename();
filename = temp.filename();
} else {
filename = tofile.c_str();
}
@@ -1106,9 +1107,8 @@ bool FileInterner::maybeUncompressToTemp(TempFile& temp, const string& fn,
" kbs\n");
return false;
}
temp =
TempFile(new TempFileInternal(cnf->getSuffixFromMimeType(doc.mimetype)));
if (!temp->ok()) {
temp = TempFile(cnf->getSuffixFromMimeType(doc.mimetype));
if (!temp.ok()) {
LOGERR("FileInterner: cant create temporary file\n");
return false;
}
@@ -1123,9 +1123,9 @@ bool FileInterner::maybeUncompressToTemp(TempFile& temp, const string& fn,
// reason for this, but it's not nice here. Have to move, the
// uncompressed file, hopefully staying on the same dev.
string reason;
if (!renameormove(uncomped.c_str(), temp->filename(), reason)) {
if (!renameormove(uncomped.c_str(), temp.filename(), reason)) {
LOGERR("FileInterner::maybeUncompress: move [" << uncomped <<
"] -> [" << temp->filename() << "] failed: " << reason << "\n");
"] -> [" << temp.filename() << "] failed: " << reason << "\n");
return false;
}
return true;
src/internfile/internfile.h
+3 -2
View File
@@ -28,14 +28,15 @@ using std::map;
using std::set;
#include "mimehandler.h"
#include "uncomp.h"
#include "pathut.h"
#include "rclutil.h"
class RclConfig;
namespace Rcl {
class Doc;
}
class Uncomp;
struct stat;
/** Storage for missing helper program info. We want to keep this out of the
@@ -277,7 +278,7 @@ class FileInterner {
string m_reason;
FIMissingStore *m_missingdatap{nullptr};
Uncomp m_uncomp;
Uncomp *m_uncomp{nullptr};
bool m_noxattrs; // disable xattrs usage
bool m_direct; // External app did the extraction
src/internfile/uncomp.cpp
+29 -7
View File
@@ -35,6 +35,12 @@ using std::vector;
Uncomp::UncompCache Uncomp::o_cache;
Uncomp::Uncomp(bool docache)
: m_docache(docache)
{
LOGDEB0("Uncomp::Uncomp: m_docache: " << m_docache << "\n");
}
bool Uncomp::uncompressfile(const string& ifn,
const vector<string>& cmdv, string& tfile)
{
@@ -57,7 +63,8 @@ bool Uncomp::uncompressfile(const string& ifn,
}
// Make sure tmp dir is empty. we guarantee this to filters
if (!m_dir || !m_dir->ok() || !m_dir->wipe()) {
LOGERR("uncompressfile: can't clear temp dir " << (m_dir->dirname()) << "\n" );
LOGERR("uncompressfile: can't clear temp dir " << m_dir->dirname() <<
"\n");
return false;
}
@@ -66,12 +73,14 @@ bool Uncomp::uncompressfile(const string& ifn,
int pc;
long long availmbs;
if (!fsocc(m_dir->dirname(), &pc, &availmbs)) {
LOGERR("uncompressfile: can't retrieve avail space for " << (m_dir->dirname()) << "\n" );
LOGERR("uncompressfile: can't retrieve avail space for " <<
m_dir->dirname() << "\n");
// Hope for the best
} else {
long long fsize = path_filesize(ifn);
if (fsize < 0) {
LOGERR("uncompressfile: stat input file " << (ifn) << " errno " << (errno) << "\n" );
LOGERR("uncompressfile: stat input file " << ifn << " errno " <<
errno << "\n");
return false;
}
// We need at least twice the file size for the uncompressed
@@ -83,7 +92,9 @@ bool Uncomp::uncompressfile(const string& ifn,
long long filembs = fsize / (1024 * 1024);
if (availmbs < 2 * filembs + 1) {
LOGERR("uncompressfile. " << (lltodecstr(availmbs)) << " MBs available in " << (m_dir->dirname()) << " not enough to uncompress " << (ifn) << " of size " << (lltodecstr(filembs)) << " mbs\n" );
LOGERR("uncompressfile. " << availmbs << " MBs available in " <<
m_dir->dirname() << " not enough to uncompress " <<
ifn << " of size " << filembs << " MBs\n");
return false;
}
}
@@ -107,9 +118,10 @@ bool Uncomp::uncompressfile(const string& ifn,
ExecCmd ex;
int status = ex.doexec(cmd, args, 0, &tfile);
if (status || tfile.empty()) {
LOGERR("uncompressfile: doexec: failed for [" << (ifn) << "] status 0x" << (status) << "\n" );
LOGERR("uncompressfile: doexec: failed for [" << ifn << "] status 0x" <<
status << "\n");
if (!m_dir->wipe()) {
LOGERR("uncompressfile: wipedir failed\n" );
LOGERR("uncompressfile: wipedir failed\n");
}
return false;
}
@@ -122,6 +134,8 @@ bool Uncomp::uncompressfile(const string& ifn,
Uncomp::~Uncomp()
{
LOGDEB0("Uncomp::~Uncomp: m_docache: " << m_docache << " m_dir " <<
(m_dir?m_dir->dirname():"(null)") << "\n");
if (m_docache) {
std::unique_lock<std::mutex> lock(o_cache.m_lock);
delete o_cache.m_dir;
@@ -133,4 +147,12 @@ Uncomp::~Uncomp()
}
}
void Uncomp::clearcache()
{
LOGDEB0("Uncomp::clearcache\n");
std::unique_lock<std::mutex> lock(o_cache.m_lock);
delete o_cache.m_dir;
o_cache.m_dir = 0;
o_cache.m_tfile.clear();
o_cache.m_srcpath.clear();
}
src/internfile/uncomp.h
+7 -13
View File
@@ -27,10 +27,7 @@
/// Uncompression script interface.
class Uncomp {
public:
Uncomp(bool docache = false)
: m_dir(0), m_docache(docache)
{
}
explicit Uncomp(bool docache = false);
~Uncomp();
/** Uncompress the input file into a temporary one, by executing the
@@ -41,25 +38,22 @@ public:
bool uncompressfile(const std::string& ifn,
const std::vector<std::string>& cmdv,
std::string& tfile);
static void clearcache();
private:
TempDir *m_dir;
TempDir *m_dir{0};
std::string m_tfile;
std::string m_srcpath;
bool m_docache;
class UncompCache {
public:
UncompCache()
: m_dir(0)
{
}
~UncompCache()
{
UncompCache() {}
~UncompCache() {
delete m_dir;
}
std::mutex m_lock;
TempDir *m_dir;
TempDir *m_dir{0};
std::string m_tfile;
std::string m_srcpath;
};