tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

make sure that python rclextract.idoctofile always retrieves an uncompressed file of the correct MIME type. + misc comments

Browse Source
This commit is contained in:
Jean-Francois Dockes
2017-07-20 12:52:24 +02:00
parent 32e79d301b
commit 29c6f75423
6 changed files with 114 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/internfile/internfile.cpp
+52 -47
View File
@@ -118,7 +118,6 @@ bool FileInterner::ipathContains(const string& parent, const string& child)
// Split into "constructor calls init()" to allow use from other constructor
FileInterner::FileInterner(const string &fn, const struct stat *stp,
RclConfig *cnf, int flags, const string *imime)
: m_ok(false), m_missingdatap(0), m_uncomp((flags & FIF_forPreview) != 0)
{
LOGDEB0("FileInterner::FileInterner(fn=" << fn << ")\n");
if (fn.empty()) {
@@ -219,8 +218,18 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
LOGDEB0("FileInterner:: no mime: [" << m_fn << "]\n");
}
// Look for appropriate handler (might still return empty)
// Get fields computed from extended attributes. We use the
// original file, not the m_fn which may be the uncompressed temp
// file
if (!m_noxattrs)
reapXAttrs(m_cfg, f, m_XAttrsFields);
// Gather metadata from external commands as configured.
reapMetaCmds(m_cfg, f, m_cmdFields);
m_mimetype = l_mime;
// Look for appropriate handler (might still return empty)
RecollFilter *df = getMimeHandler(l_mime, m_cfg, !m_forPreview);
if (!df || df->is_unknown()) {
@@ -234,15 +243,6 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
m_forPreview ? "view" : "index");
df->set_property(Dijon::Filter::DJF_UDI, udi);
// Get fields computed from extended attributes. We use the
// original file, not the m_fn which may be the uncompressed temp
// file
if (!m_noxattrs)
reapXAttrs(m_cfg, f, m_XAttrsFields);
// Gather metadata from external commands as configured.
reapMetaCmds(m_cfg, f, m_cmdFields);
df->set_docsize(docsize);
if (!df->set_document_file(l_mime, m_fn)) {
delete df;
@@ -258,7 +258,6 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
// Setup from memory data (ie: out of the web cache). imime needs to be set.
FileInterner::FileInterner(const string &data, RclConfig *cnf,
int flags, const string& imime)
: m_ok(false), m_missingdatap(0), m_uncomp((flags & FIF_forPreview) != 0)
{
LOGDEB0("FileInterner::FileInterner(data)\n");
initcommon(cnf, flags);
@@ -313,7 +312,7 @@ void FileInterner::init(const string &data, RclConfig *cnf,
void FileInterner::initcommon(RclConfig *cnf, int flags)
{
m_cfg = cnf;
m_forPreview = ((flags & FIF_forPreview) != 0);
m_uncomp = m_forPreview = ((flags & FIF_forPreview) != 0);
// Initialize handler stack.
m_handlers.reserve(MAXHANDLERS);
for (unsigned int i = 0; i < MAXHANDLERS; i++)
@@ -324,7 +323,6 @@ void FileInterner::initcommon(RclConfig *cnf, int flags)
}
FileInterner::FileInterner(const Rcl::Doc& idoc, RclConfig *cnf, int flags)
: m_ok(false), m_missingdatap(0), m_uncomp(((flags & FIF_forPreview) != 0))
{
LOGDEB0("FileInterner::FileInterner(idoc)\n");
initcommon(cnf, flags);
@@ -347,6 +345,9 @@ FileInterner::FileInterner(const Rcl::Doc& idoc, RclConfig *cnf, int flags)
init(rawdoc.data, cnf, flags, idoc.mimetype);
break;
case DocFetcher::RawDoc::RDK_DATADIRECT:
// Note: only used for demo with the sample python external
// mbox indexer at this point. The external program is
// responsible for all the extraction process.
init(rawdoc.data, cnf, flags, idoc.mimetype);
m_direct = true;
break;
@@ -735,8 +736,8 @@ int FileInterner::addHandler()
}
}
if (!setres) {
LOGINFO("FileInterner::addHandler: set_doc failed inside " << m_fn <<
" for mtype " << mimetype << "\n");
LOGINFO("FileInterner::addHandler: set_doc failed inside [" << m_fn <<
"] for mtype " << mimetype << "\n");
delete newflt;
if (m_forPreview)
return ADD_ERROR;
@@ -918,36 +919,24 @@ bool FileInterner::tempFileForMT(TempFile& otemp, RclConfig* cnf,
TempFile temp(new TempFileInternal(
cnf->getSuffixFromMimeType(mimetype)));
if (!temp->ok()) {
LOGERR("FileInterner::interntofile: can't create temp file\n");
LOGERR("FileInterner::tempFileForMT: can't create temp file\n");
return false;
}
otemp = temp;
return true;
}
// Extract document (typically subdoc of multidoc) into temporary file.
// We do the usual internfile stuff: create a temporary directory,
// then create an interner and call internfile. The target mtype is set to
// the input mtype, so that no data conversion is performed.
// We then write the data out of the resulting document into the output file.
// There are two temporary objects:
// - The internfile temporary directory gets destroyed by its destructor
// - The output temporary file which is held in a reference-counted
// object and will be deleted when done with.
//
// If the ipath is null, maybe we're called because the file is not
// stored in the regular file system. We use the docfetcher to get a
// copy (in topdocToFile())
//
// We currently don't handle the case of an internal doc of a non-fs document.
bool FileInterner::idocToFile(TempFile& otemp, const string& tofile,
RclConfig *cnf, const Rcl::Doc& idoc)
// Static method, creates a FileInterner object to do the job.
bool FileInterner::idocToFile(
TempFile& otemp, const string& tofile, RclConfig *cnf,
const Rcl::Doc& idoc, bool uncompress)
{
LOGDEB("FileInterner::idocToFile\n");
if (idoc.ipath.empty()) {
return topdocToFile(otemp, tofile, cnf, idoc);
// Because of the mandatory first conversion in the
// FileInterner constructor, need to use a specific method.
return topdocToFile(otemp, tofile, cnf, idoc, uncompress);
}
// We set FIF_forPreview for consistency with the previous version
@@ -958,17 +947,21 @@ bool FileInterner::idocToFile(TempFile& otemp, const string& tofile,
return interner.interntofile(otemp, tofile, idoc.ipath, idoc.mimetype);
}
bool FileInterner::topdocToFile(TempFile& otemp, const string& tofile,
RclConfig *cnf, const Rcl::Doc& idoc)
// This is only needed because the FileInterner constructor always performs
// the first conversion, so that we need another approach for accessing the
// original document (targetmtype won't do).
bool FileInterner::topdocToFile(
TempFile& otemp, const string& tofile,
RclConfig *cnf, const Rcl::Doc& idoc, bool uncompress)
{
DocFetcher *fetcher = docFetcherMake(cnf, idoc);
if (fetcher == 0) {
LOGERR("FileInterner::idocToFile no backend\n");
LOGERR("FileInterner::topdocToFile no backend\n");
return false;
}
DocFetcher::RawDoc rawdoc;
if (!fetcher->fetch(cnf, idoc, rawdoc)) {
LOGERR("FileInterner::idocToFile fetcher failed\n");
LOGERR("FileInterner::topdocToFile fetcher failed\n");
return false;
}
const char *filename = "";
@@ -983,13 +976,24 @@ bool FileInterner::topdocToFile(TempFile& otemp, const string& tofile,
}
string reason;
switch (rawdoc.kind) {
case DocFetcher::RawDoc::RDK_FILENAME:
if (!copyfile(rawdoc.data.c_str(), filename, reason)) {
case DocFetcher::RawDoc::RDK_FILENAME: {
string fn(rawdoc.data);
TempFile temp;
if (uncompress && isCompressed(fn, cnf)) {
if (!maybeUncompressToTemp(temp, fn, cnf, idoc)) {
LOGERR("FileInterner::idocToFile: uncompress failed\n");
return false;
}
}
fn = temp ? temp->filename() : rawdoc.data;
if (!copyfile(fn.c_str(), filename, reason)) {
LOGERR("FileInterner::idocToFile: copyfile: " << reason << "\n");
return false;
}
}
break;
case DocFetcher::RawDoc::RDK_DATA:
case DocFetcher::RawDoc::RDK_DATADIRECT:
if (!stringtofile(rawdoc.data, filename, reason)) {
LOGERR("FileInterner::idocToFile: stringtofile: " << reason <<"\n");
return false;
@@ -1019,11 +1023,12 @@ bool FileInterner::interntofile(TempFile& otemp, const string& tofile,
}
// Specialcase text/html. This is to work around a bug that will
// get fixed some day: internfile initialisation does not check
// targetmtype, so that at least one conversion is always
// performed. A common case would be an "Open" on an html file
// (we'd end up with text/plain content). As the html version is
// saved in this case, use it.
// get fixed some day: the internfile constructor always loads the
// first handler so that at least one conversion is always
// performed (and the access to the original data may be lost). A
// common case is an "Open" on an HTML file (we end up
// with text/plain content). As the HTML version is saved in this
// case, use it.
if (!stringlowercmp(cstr_texthtml, mimetype) && !get_html().empty()) {
doc.text = get_html();
doc.mimetype = cstr_texthtml;
src/internfile/internfile.h
+24 -18
View File
@@ -118,21 +118,20 @@ class FileInterner {
/**
* Alternate constructor for the case where the data is in memory.
* This is mainly for data extracted from the web cache. The mime type
* must be set, input must be already uncompressed.
* This is mainly for data extracted from the web cache.
* The MIME type must be set, and the data must be uncompressed.
*/
FileInterner(const string &data, RclConfig *cnf,
int flags, const string& mtype);
/**
* Alternate constructor used at query time. We don't know where
* the data was stored, this is determined from the Rcl::Doc data
* the data was stored, and use the fetcher interface to reach it.
*
* @param idoc Rcl::Doc object built from index data. The back-end
* storage identifier (rclbes field) is used to build the
* appropriate fetcher which uses the rest of the Doc fields (url,
* ipath...) to retrieve the file or a file reference, which we
* then process normally.
* storage identifier (rclbes field) is used by the fetcher factory
* to build the appropriate object to return a file name or data which
* is then used with the appropriate init method.
*/
FileInterner(const Rcl::Doc& idoc, RclConfig *cnf, int flags);
@@ -214,10 +213,15 @@ class FileInterner {
* to query the right backend. Used to check up-to-dateness at query time */
static bool makesig(RclConfig *cnf, const Rcl::Doc& idoc, string& sig);
/** Extract internal document into temporary file.
* This is used mainly for starting an external viewer for a
* subdocument (ie: mail attachment). This really would not need to be
* a member. It creates a FileInterner object to do the actual work
/** Extract internal document into temporary file, without converting the
* data.
*
* This is used mainly for starting an external viewer for a
* subdocument (ie: mail attachment), but, for consistency, it also
* works with a top level (null ipath) document.
* This would not actually need to be a member method, it creates a
* FileInterner object to do the actual work.
*
* @return true for success.
* @param temp output reference-counted temp file object (goes
* away magically). Only used if tofile.empty()
@@ -225,13 +229,14 @@ class FileInterner {
* @param cnf The recoll config
* @param doc Doc data taken from the index. We use it to construct a
* FileInterner object.
* @param uncompress if true, uncompress compressed original doc. Only does
* anything for a top level document.
*/
static bool idocToFile(TempFile& temp, const string& tofile,
RclConfig *cnf, const Rcl::Doc& doc);
RclConfig *cnf, const Rcl::Doc& doc,
bool uncompress = true);
/**
* Does file appear to be the compressed version of a document?
*/
/** Does file appear to be the compressed version of a document? */
static bool isCompressed(const string& fn, RclConfig *cnf);
/**
@@ -253,7 +258,7 @@ class FileInterner {
string m_targetMType;
string m_reachedMType; // target or text/plain
string m_tfile;
bool m_ok; // Set after construction if ok
bool m_ok{false}; // Set after construction if ok
// Fields found in file extended attributes. This is kept here,
// not in the file-level handler because we are only interested in
// the top-level file, not any temp file necessitated by
@@ -270,7 +275,7 @@ class FileInterner {
vector<TempFile> m_tempfiles;
// Error data if any
string m_reason;
FIMissingStore *m_missingdatap;
FIMissingStore *m_missingdatap{nullptr};
Uncomp m_uncomp;
@@ -294,7 +299,8 @@ class FileInterner {
static bool tempFileForMT(TempFile& otemp, RclConfig *cnf,
const std::string& mimetype);
static bool topdocToFile(TempFile& otemp, const std::string& tofile,
RclConfig *cnf, const Rcl::Doc& idoc);
RclConfig *cnf, const Rcl::Doc& idoc,
bool uncompress);
};