tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

simplified temp file handling for compressed documents and, for querying, implemented caching for last file uncompressed

Browse Source
This commit is contained in:
Jean-Francois Dockes
2013-03-06 18:52:57 +01:00
parent 2a22e864cc
commit 3c80e51940
14 changed files with 221 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/internfile/internfile.cpp
+16 -75
View File
@@ -151,60 +151,6 @@ string FileInterner::getLastIpathElt(const string& ipath)
}
}
// Uncompress input file into a temporary one, by executing the appropriate
// script.
static bool uncompressfile(RclConfig *conf, const string& ifn,
const vector<string>& cmdv, TempDir& tdir,
string& tfile)
{
// Make sure tmp dir is empty. we guarantee this to filters
if (!tdir.ok() || !tdir.wipe()) {
LOGERR(("uncompressfile: can't clear temp dir %s\n", tdir.dirname()));
return false;
}
string cmd = cmdv.front();
// Substitute file name and temp dir in command elements
vector<string>::const_iterator it = cmdv.begin();
++it;
vector<string> args;
map<char, string> subs;
subs['f'] = ifn;
subs['t'] = tdir.dirname();
for (; it != cmdv.end(); it++) {
string ns;
pcSubst(*it, ns, subs);
args.push_back(ns);
}
// Execute command and retrieve output file name, check that it exists
ExecCmd ex;
int status = ex.doexec(cmd, args, 0, &tfile);
if (status || tfile.empty()) {
LOGERR(("uncompressfile: doexec: failed for [%s] status 0x%x\n",
ifn.c_str(), status));
if (!tdir.wipe()) {
LOGERR(("uncompressfile: wipedir failed\n"));
}
return false;
}
if (tfile[tfile.length() - 1] == '\n')
tfile.erase(tfile.length() - 1, 1);
return true;
}
// Delete temporary uncompressed file
void FileInterner::tmpcleanup()
{
if (m_tfile.empty())
return;
if (unlink(m_tfile.c_str()) < 0) {
LOGERR(("FileInterner::tmpcleanup: unlink(%s) errno %d\n",
m_tfile.c_str(), errno));
return;
}
}
// Constructor: identify the input file, possibly create an
// uncompressed temporary copy, and create the top filter for the
// uncompressed file type.
@@ -213,9 +159,8 @@ void FileInterner::tmpcleanup()
// processed by the first call to internfile().
// Split into "constructor calls init()" to allow use from other constructor
FileInterner::FileInterner(const string &fn, const struct stat *stp,
RclConfig *cnf,
TempDir& td, int flags, const string *imime)
: m_tdir(td), m_ok(false), m_missingdatap(0)
RclConfig *cnf, int flags, const string *imime)
: m_ok(false), m_missingdatap(0), m_uncomp((flags & FIF_forPreview) != 0)
{
LOGDEB0(("FileInterner::FileInterner(fn=%s)\n", fn.c_str()));
if (fn.empty()) {
@@ -285,11 +230,11 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
int maxkbs = -1;
if (!m_cfg->getConfParam("compressedfilemaxkbs", &maxkbs) ||
maxkbs < 0 || !stp || int(stp->st_size / 1024) < maxkbs) {
if (!uncompressfile(m_cfg, m_fn, ucmd, m_tdir, m_tfile)) {
if (!m_uncomp.uncompressfile(m_fn, ucmd, m_tfile)) {
return;
}
LOGDEB1(("FileInterner:: after ucomp: m_tdir %s, tfile %s\n",
m_tdir.dirname(), m_tfile.c_str()));
LOGDEB1(("FileInterner:: after ucomp: tfile %s\n",
m_tfile.c_str()));
m_fn = m_tfile;
// Stat the uncompressed file, mainly to get the size
struct stat ucstat;
@@ -352,8 +297,8 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
// Setup from memory data (ie: out of the web cache). imime needs to be set.
FileInterner::FileInterner(const string &data, RclConfig *cnf,
TempDir& td, int flags, const string& imime)
: m_tdir(td), m_ok(false), m_missingdatap(0)
int flags, const string& imime)
: m_ok(false), m_missingdatap(0), m_uncomp((flags & FIF_forPreview) != 0)
{
LOGDEB0(("FileInterner::FileInterner(data)\n"));
initcommon(cnf, flags);
@@ -416,9 +361,8 @@ void FileInterner::initcommon(RclConfig *cnf, int flags)
m_targetMType = cstr_textplain;
}
FileInterner::FileInterner(const Rcl::Doc& idoc, RclConfig *cnf,
TempDir& td, int flags)
: m_tdir(td), m_ok(false), m_missingdatap(0)
FileInterner::FileInterner(const Rcl::Doc& idoc, RclConfig *cnf, int flags)
: m_ok(false), m_missingdatap(0), m_uncomp(((flags & FIF_forPreview) != 0))
{
LOGDEB0(("FileInterner::FileInterner(idoc)\n"));
initcommon(cnf, flags);
@@ -462,7 +406,6 @@ bool FileInterner::makesig(RclConfig *cnf, const Rcl::Doc& idoc, string& sig)
FileInterner::~FileInterner()
{
tmpcleanup();
for (vector<Dijon::Filter*>::iterator it = m_handlers.begin();
it != m_handlers.end(); it++) {
returnMimeHandler(*it);
@@ -1012,12 +955,10 @@ bool FileInterner::idocToFile(TempFile& otemp, const string& tofile,
LOGDEB(("FileInterner::idocToFile\n"));
idoc.dump();
TempDir tmpdir;
// We set FIF_forPreview for consistency with the previous version
// which determined this by looking at mtype!=null. Probably
// doesn't change anything in this case.
FileInterner interner(idoc, cnf, tmpdir, FIF_forPreview);
FileInterner interner(idoc, cnf, FIF_forPreview);
interner.setTargetMType(idoc.mimetype);
return interner.interntofile(otemp, tofile, idoc.ipath, idoc.mimetype);
}
@@ -1102,6 +1043,7 @@ bool FileInterner::isCompressed(const string& fn, RclConfig *cnf)
return false;
}
// Static.
bool FileInterner::maybeUncompressToTemp(TempFile& temp, const string& fn,
RclConfig *cnf, const Rcl::Doc& doc)
{
@@ -1131,16 +1073,16 @@ bool FileInterner::maybeUncompressToTemp(TempFile& temp, const string& fn,
fn.c_str(), maxkbs));
return false;
}
TempDir tmpdir;
temp =
TempFile(new TempFileInternal(cnf->getSuffixFromMimeType(doc.mimetype)));
if (!tmpdir.ok() || !temp->ok()) {
LOGERR(("FileInterner: cant create temporary file/dir"));
if (!temp->ok()) {
LOGERR(("FileInterner: cant create temporary file"));
return false;
}
Uncomp uncomp;
string uncomped;
if (!uncompressfile(cnf, fn, ucmd, tmpdir, uncomped)) {
if (!uncomp.uncompressfile(fn, ucmd, uncomped)) {
return false;
}
@@ -1233,8 +1175,7 @@ int main(int argc, char **argv)
perror("stat");
exit(1);
}
TempDir tmp;
FileInterner interner(fn, &st, config, tmp, 0);
FileInterner interner(fn, &st, config, 0);
Rcl::Doc doc;
FileInterner::Status status = interner.internfile(doc, ipath);
switch (status) {
src/internfile/internfile.h
+8 -10
View File
@@ -16,6 +16,8 @@
*/
#ifndef _INTERNFILE_H_INCLUDED_
#define _INTERNFILE_H_INCLUDED_
// The class changes according to RCL_USE_XATTR
#include "autoconfig.h"
#include <string>
#include <vector>
@@ -27,8 +29,7 @@ using std::map;
using std::set;
#include "Filter.h"
// The class changes according to RCL_USE_XATTR
#include "autoconfig.h"
#include "uncomp.h"
#include "pathut.h"
class RclConfig;
@@ -114,15 +115,14 @@ class FileInterner {
* mime type for the uncompressed version.
*/
FileInterner(const string &fn, const struct stat *stp,
RclConfig *cnf, TempDir &td, int flags,
const string *mtype = 0);
RclConfig *cnf, int flags, const string *mtype = 0);
/**
* Alternate constructor for the case where the data is in memory.
* This is mainly for data extracted from the web cache. The mime type
* must be set, input must be already uncompressed.
*/
FileInterner(const string &data, RclConfig *cnf, TempDir &td,
FileInterner(const string &data, RclConfig *cnf,
int flags, const string& mtype);
/**
@@ -135,8 +135,7 @@ class FileInterner {
* ipath...) to retrieve the file or a file reference, which we
* then process normally.
*/
FileInterner(const Rcl::Doc& idoc, RclConfig *cnf, TempDir &td,
int flags);
FileInterner(const Rcl::Doc& idoc, RclConfig *cnf, int flags);
~FileInterner();
@@ -251,8 +250,6 @@ class FileInterner {
TempFile m_imgtmp; // Possible reference to an image temp file
string m_targetMType;
string m_reachedMType; // target or text/plain
// m_tdir and m_tfile are used only for decompressing input file if needed
TempDir &m_tdir;
string m_tfile;
bool m_ok; // Set after construction if ok
#ifdef RCL_USE_XATTR
@@ -273,6 +270,8 @@ class FileInterner {
string m_reason;
FIMissingStore *m_missingdatap;
Uncomp m_uncomp;
// Pseudo-constructors
void init(const string &fn, const struct stat *stp,
RclConfig *cnf, int flags, const string *mtype = 0);
@@ -280,7 +279,6 @@ class FileInterner {
const string& mtype);
void initcommon(RclConfig *cnf, int flags);
void tmpcleanup();
bool dijontorcl(Rcl::Doc&);
void collectIpathAndMT(Rcl::Doc&) const;
TempFile dataToTempFile(const string& data, const string& mt);
src/internfile/uncomp.cpp
+104
View File
@@ -0,0 +1,104 @@
/* Copyright (C) 2013 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "autoconfig.h"
#include <string>
#include <vector>
#include <map>
using std::map;
using std::string;
using std::vector;
#include "uncomp.h"
#include "debuglog.h"
#include "smallut.h"
#include "execmd.h"
Uncomp::UncompCache Uncomp::o_cache;
bool Uncomp::uncompressfile(const string& ifn,
const vector<string>& cmdv, string& tfile)
{
if (m_docache) {
PTMutexLocker lock(o_cache.m_lock);
if (!o_cache.m_srcpath.compare(ifn)) {
m_dir = o_cache.m_dir;
m_tfile = tfile = o_cache.m_tfile;
m_srcpath = ifn;
o_cache.m_dir = 0;
o_cache.m_srcpath.clear();
return true;
}
}
m_srcpath.clear();
m_tfile.clear();
if (m_dir == 0) {
m_dir = new TempDir;
}
// Make sure tmp dir is empty. we guarantee this to filters
if (!m_dir || !m_dir->ok() || !m_dir->wipe()) {
LOGERR(("uncompressfile: can't clear temp dir %s\n", m_dir->dirname()));
return false;
}
string cmd = cmdv.front();
// Substitute file name and temp dir in command elements
vector<string>::const_iterator it = cmdv.begin();
++it;
vector<string> args;
map<char, string> subs;
subs['f'] = ifn;
subs['t'] = m_dir->dirname();
for (; it != cmdv.end(); it++) {
string ns;
pcSubst(*it, ns, subs);
args.push_back(ns);
}
// Execute command and retrieve output file name, check that it exists
ExecCmd ex;
int status = ex.doexec(cmd, args, 0, &tfile);
if (status || tfile.empty()) {
LOGERR(("uncompressfile: doexec: failed for [%s] status 0x%x\n",
ifn.c_str(), status));
if (!m_dir->wipe()) {
LOGERR(("uncompressfile: wipedir failed\n"));
}
return false;
}
if (tfile[tfile.length() - 1] == '\n')
tfile.erase(tfile.length() - 1, 1);
m_tfile = tfile;
m_srcpath = ifn;
return true;
}
Uncomp::~Uncomp()
{
if (m_docache) {
PTMutexLocker lock(o_cache.m_lock);
delete o_cache.m_dir;
o_cache.m_dir = m_dir;
o_cache.m_tfile = m_tfile;
o_cache.m_srcpath = m_srcpath;
} else {
delete m_dir;
}
}
src/internfile/uncomp.h
+68
View File
@@ -0,0 +1,68 @@
/* Copyright (C) 2013 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef _UNCOMP_H_INCLUDED_
#define _UNCOMP_H_INCLUDED_
#include <vector>
#include <string>
#include "pathut.h"
#include "ptmutex.h"
/// Uncompression script interface.
class Uncomp {
public:
Uncomp(bool docache = false)
: m_dir(0), m_docache(docache)
{
}
~Uncomp();
/** Uncompress the input file into a temporary one, by executing the
* script given as input.
* Return the path to the uncompressed file (which is inside a
* temporary directory).
*/
bool uncompressfile(const std::string& ifn,
const std::vector<std::string>& cmdv,
std::string& tfile);
private:
TempDir *m_dir;
string m_tfile;
string m_srcpath;
bool m_docache;
class UncompCache {
public:
UncompCache()
: m_dir(0)
{
}
~UncompCache()
{
delete m_dir;
}
PTMutexInit m_lock;
TempDir *m_dir;
string m_tfile;
string m_srcpath;
};
static UncompCache o_cache;
};
#endif /* _UNCOMP_H_INCLUDED_ */