tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
recoll/src/internfile/internfile.cpp

673 lines
19 KiB
C++
Raw Blame History

#ifndef lint
static char rcsid[] = "@(#$Id: internfile.cpp,v 1.28 2007-02-19 18:05:25 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef TEST_INTERNFILE
#include <unistd.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <string>
#include <iostream>
#ifndef NO_NAMESPACES
using namespace std;
#endif /* NO_NAMESPACES */
#include "internfile.h"
#include "rcldoc.h"
#include "mimetype.h"
#include "debuglog.h"
#include "mimehandler.h"
#include "execmd.h"
#include "pathut.h"
#include "wipedir.h"
#include "rclconfig.h"
// The internal path element separator. This can't be the same as the rcldb
// file to ipath separator : "|"
static const string isep(":");
// Execute the command to uncompress a file into a temporary one.
static bool uncompressfile(RclConfig *conf, const string& ifn,
const list<string>& cmdv, const string& tdir,
string& tfile)
{
// Make sure tmp dir is empty. we guarantee this to filters
if (wipedir(tdir) != 0) {
LOGERR(("uncompressfile: can't clear temp dir %s\n", tdir.c_str()));
return false;
}
string cmd = cmdv.front();
// Substitute file name and temp dir in command elements
list<string>::const_iterator it = cmdv.begin();
++it;
list<string> args;
for (; it != cmdv.end(); it++) {
string s = *it;
string ns;
string::const_iterator it1;
for (it1 = s.begin(); it1 != s.end();it1++) {
if (*it1 == '%') {
if (++it1 == s.end()) {
ns += '%';
break;
}
if (*it1 == '%')
ns += '%';
if (*it1 == 'f')
ns += ifn;
if (*it1 == 't')
ns += tdir;
} else {
ns += *it1;
}
}
args.push_back(ns);
}
// Execute command and retrieve output file name, check that it exists
ExecCmd ex;
int status = ex.doexec(cmd, args, 0, &tfile);
if (status || tfile.empty()) {
LOGERR(("uncompressfile: doexec: status 0x%x\n", status));
rmdir(tdir.c_str());
return false;
}
if (tfile[tfile.length() - 1] == '\n')
tfile.erase(tfile.length() - 1, 1);
return true;
}
void FileInterner::tmpcleanup()
{
if (m_tdir.empty() || m_tfile.empty())
return;
if (unlink(m_tfile.c_str()) < 0) {
LOGERR(("FileInterner::tmpcleanup: unlink(%s) errno %d\n",
m_tfile.c_str(), errno));
return;
}
}
// Handler==0 on return says we're in error, will be handled when calling
// internfile
FileInterner::FileInterner(const std::string &f, const struct stat *stp,
RclConfig *cnf,
const string& td, const string *imime)
: m_cfg(cnf), m_fn(f), m_forPreview(imime?true:false), m_tdir(td)
{
bool usfci = false;
cnf->getConfParam("usesystemfilecommand", &usfci);
LOGDEB(("FileInterner::FileInterner: [%s] mime [%s] preview %d\n",
f.c_str(), imime?imime->c_str() : "(null)", m_forPreview));
// We need to run mime type identification in any case to check
// for a compressed file.
string l_mime = mimetype(m_fn, stp, m_cfg, usfci);
// If identification fails, try to use the input parameter. This
// is then normally not a compressed type (it's the mime type from
// the db), and is only set when previewing, not for indexing
if (l_mime.empty() && imime)
l_mime = *imime;
if (!l_mime.empty()) {
// Has mime: check for a compressed file. If so, create a
// temporary uncompressed file, and rerun the mime type
// identification, then do the rest with the temp file.
list<string>ucmd;
if (m_cfg->getUncompressor(l_mime, ucmd)) {
if (!uncompressfile(m_cfg, m_fn, ucmd, m_tdir, m_tfile)) {
return;
}
LOGDEB1(("internfile: after ucomp: m_tdir %s, tfile %s\n",
m_tdir.c_str(), m_tfile.c_str()));
m_fn = m_tfile;
// Note: still using the original file's stat. right ?
l_mime = mimetype(m_fn, stp, m_cfg, usfci);
if (l_mime.empty() && imime)
l_mime = *imime;
}
}
if (l_mime.empty()) {
// No mime type. We let it through as config may warrant that
// we index all file names
LOGDEB(("internfile: (no mime) [%s]\n", m_fn.c_str()));
}
// Look for appropriate handler (might still return empty)
m_mimetype = l_mime;
Dijon::Filter *df = getMimeHandler(l_mime, m_cfg);
if (!df) {
// No handler for this type, for now :( if indexallfilenames
// is set in the config, this normally wont happen (we get mh_unknown)
LOGDEB(("FileInterner:: no handler for %s\n", l_mime.c_str()));
return;
}
df->set_property(Dijon::Filter::OPERATING_MODE,
m_forPreview ? "view" : "index");
string charset = m_cfg->getDefCharset();
df->set_property(Dijon::Filter::DEFAULT_CHARSET, charset);
if (!df->set_document_file(m_fn)) {
LOGERR(("FileInterner:: error parsing %s\n", m_fn.c_str()));
return;
}
m_handlers.reserve(MAXHANDLERS);
for (unsigned int i = 0; i < MAXHANDLERS; i++)
m_tmpflgs[i] = false;
m_handlers.push_back(df);
LOGDEB(("FileInterner::FileInterner: %s [%s]\n", l_mime.c_str(),
m_fn.c_str()));
m_targetMType = "text/plain";
}
FileInterner::~FileInterner()
{
tmpcleanup();
for (vector<Dijon::Filter*>::iterator it = m_handlers.begin();
it != m_handlers.end(); it++)
delete *it;
// m_tempfiles will take care of itself
}
bool FileInterner::dataToTempFile(const string& dt, const string& mt,
string& fn)
{
// Find appropriate suffix for mime type
TempFile temp(new TempFileInternal(m_cfg->getSuffixFromMimeType(mt)));
if (temp->ok()) {
m_tmpflgs[m_handlers.size()-1] = true;
m_tempfiles.push_back(temp);
} else {
LOGERR(("FileInterner::dataToTempFile: cant create tempfile\n"));
return false;
}
int fd = open(temp->filename(), O_WRONLY);
if (fd < 0) {
LOGERR(("FileInterner::dataToTempFile: open(%s) failed errno %d\n",
temp->filename(), errno));
return false;
}
if (write(fd, dt.c_str(), dt.length()) != (int)dt.length()) {
close(fd);
LOGERR(("FileInterner::dataToTempFile: write to %s failed errno %d\n",
temp->filename(), errno));
return false;
}
close(fd);
fn = temp->filename();
return true;
}
// See if the error string is formatted as a missing helper message,
// accumulate helper name if it is
void FileInterner::maybeExternalMissing(const string& msg)
{
if (msg.find("RECFILTERROR") == 0) {
list<string> lerr;
stringToStrings(msg, lerr);
if (lerr.size() > 2) {
list<string>::iterator it = lerr.begin();
it++;
if (*it == "HELPERNOTFOUND") {
it++;
m_missingExternal.push_back(it->c_str());
}
}
}
}
const list<string>& FileInterner::getMissingExternal()
{
m_missingExternal.sort();
m_missingExternal.unique();
return m_missingExternal;
}
void FileInterner::getMissingExternal(string& out)
{
m_missingExternal.sort();
m_missingExternal.unique();
stringsToString(m_missingExternal, out);
}
static inline bool getKeyValue(const map<string, string>& docdata,
const string& key, string& value)
{
map<string,string>::const_iterator it;
it = docdata.find(key);
if (it != docdata.end()) {
value = it->second;
return true;
}
return false;
}
static const string keyab("abstract");
static const string keyau("author");
static const string keycs("charset");
static const string keyct("content");
static const string keyfn("filename");
static const string keykw("keywords");
static const string keymd("modificationdate");
static const string keymt("mimetype");
static const string keyoc("origcharset");
static const string keysm("sample");
static const string keytt("title");
bool FileInterner::dijontorcl(Rcl::Doc& doc)
{
Dijon::Filter *df = m_handlers.back();
const std::map<std::string, std::string>& docdata = df->get_meta_data();
getKeyValue(docdata, keyau, doc.author);
getKeyValue(docdata, keyoc, doc.origcharset);
getKeyValue(docdata, keyct, doc.text);
getKeyValue(docdata, keytt, doc.title);
getKeyValue(docdata, keykw, doc.keywords);
getKeyValue(docdata, keymd, doc.dmtime);
if (!getKeyValue(docdata, keyab, doc.abstract))
getKeyValue(docdata, keysm, doc.abstract);
LOGDEB1(("FILENAME: %s\n", doc.utf8fn.c_str()));
return true;
}
// Collect the ipath stack.
// While we're at it, we also set the mimetype and filename, which are special
// properties: we want to get them from the topmost doc
// with an ipath, not the last one which is usually text/plain
void FileInterner::collectIpathAndMT(Rcl::Doc& doc, string& ipath) const
{
bool hasipath = false;
// If there is no ipath stack, the mimetype is the one from the file
doc.mimetype = m_mimetype;
LOGDEB2(("INITIAL mimetype: %s\n", doc.mimetype.c_str()));
string ipathel;
for (vector<Dijon::Filter*>::const_iterator hit = m_handlers.begin();
hit != m_handlers.end(); hit++) {
const map<string, string>& docdata = (*hit)->get_meta_data();
if (getKeyValue(docdata, "ipath", ipathel)) {
if (!ipathel.empty()) {
// We have a non-empty ipath
hasipath = true;
getKeyValue(docdata, keymt, doc.mimetype);
getKeyValue(docdata, keyfn, doc.utf8fn);
}
ipath += ipathel + isep;
} else {
ipath += isep;
}
}
// Trim empty tail elements in ipath.
if (hasipath) {
LOGDEB2(("IPATH [%s]\n", ipath.c_str()));
string::size_type sit = ipath.find_last_not_of(isep);
if (sit == string::npos)
ipath.erase();
else if (sit < ipath.length() -1)
ipath.erase(sit+1);
} else {
ipath.erase();
}
}
// Remove handler from stack. Clean up temp file if needed.
void FileInterner::popHandler()
{
if (m_handlers.empty())
return;
int i = m_handlers.size()-1;
if (m_tmpflgs[i]) {
m_tempfiles.pop_back();
m_tmpflgs[i] = false;
}
delete m_handlers.back();
m_handlers.pop_back();
}
FileInterner::Status FileInterner::internfile(Rcl::Doc& doc, string& ipath)
{
LOGDEB(("FileInterner::internfile. ipath [%s]\n", ipath.c_str()));
if (m_handlers.size() < 1) {
LOGERR(("FileInterner::internfile: bad stack size %d !!\n",
m_handlers.size()));
return FIError;
}
// Ipath vector.
// Note that the vector is big enough for the maximum stack. All values
// over the last significant one are ""
// We set the ipath for the first handler here, others are set
// when they're pushed on the stack
vector<string> vipath(MAXHANDLERS);
int vipathidx = 0;
if (!ipath.empty()) {
list<string> lipath;
stringToTokens(ipath, lipath, isep, true);
vipath.insert(vipath.begin(), lipath.begin(), lipath.end());
if (!m_handlers.back()->skip_to_document(vipath[m_handlers.size()-1])){
LOGERR(("FileInterner::internfile: can't skip\n"));
return FIError;
}
}
/* Try to get doc from the topmost filter */
// Security counter: we try not to loop but ...
int loop = 0;
while (!m_handlers.empty()) {
if (loop++ > 30) {
LOGERR(("FileInterner:: looping!\n"));
return FIError;
}
if (!m_handlers.back()->has_documents()) {
// No docs at the current top level. Pop and see if there
// is something at the previous one
popHandler();
continue;
}
// Don't stop on next_document() error. There might be ie an
// error while decoding an attachment, but we still want to
// process the rest of the mbox!
if (!m_handlers.back()->next_document()) {
Rcl::Doc doc; string ipath;
collectIpathAndMT(doc, ipath);
m_reason = m_handlers.back()->get_error();
maybeExternalMissing(m_reason);
LOGERR(("FileInterner::internfile: next_document error [%s%s%s] %s\n",
m_fn.c_str(), ipath.empty()?"":"|", ipath.c_str(),
m_reason.c_str()));
// If fetching a specific document, this is fatal
if (m_forPreview) {
return FIError;
}
popHandler();
continue;
}
// Look at what we've got
const std::map<std::string, std::string>& docdata =
m_handlers.back()->get_meta_data();
string charset, mimetype;
getKeyValue(docdata, keycs, charset);
getKeyValue(docdata, keymt, mimetype);
LOGDEB(("FileInterner::internfile: next_doc is %s\n",
mimetype.c_str()));
// If we find a text/plain doc, we're done
if (!stringicmp(mimetype, m_targetMType))
break;
// Got a non text/plain doc. We need to stack another
// filter. Check current size
if (m_handlers.size() > MAXHANDLERS) {
// Stack too big. Skip this and go on to check if there is
// something else in the current back()
LOGINFO(("FileInterner::internfile: stack too high\n"));
continue;
}
Dijon::Filter *again = getMimeHandler(mimetype, m_cfg);
if (!again) {
// If we can't find a filter, this doc can't be handled
// but there can be other ones so we go on
LOGINFO(("FileInterner::internfile: no filter for [%s]\n",
mimetype.c_str()));
continue;
}
again->set_property(Dijon::Filter::OPERATING_MODE,
m_forPreview ? "view" : "index");
again->set_property(Dijon::Filter::DEFAULT_CHARSET,
charset);
string ns;
const string *txt = &ns;
map<string,string>::const_iterator it;
it = docdata.find("content");
if (it != docdata.end())
txt = &it->second;
bool setres = false;
if (again->is_data_input_ok(Dijon::Filter::DOCUMENT_STRING)) {
setres = again->set_document_string(*txt);
} else if (again->is_data_input_ok(Dijon::Filter::DOCUMENT_DATA)) {
setres = again->set_document_data(txt->c_str(), txt->length());
}else if(again->is_data_input_ok(Dijon::Filter::DOCUMENT_FILE_NAME)) {
string filename;
if (dataToTempFile(*txt, mimetype, filename)) {
if (!(setres = again->set_document_file(filename))) {
m_tmpflgs[m_handlers.size()-1] = false;
m_tempfiles.pop_back();
}
}
}
if (!setres) {
LOGINFO(("FileInterner::internfile: set_doc failed inside %s\n",
m_fn.c_str()));
delete again;
if (m_forPreview)
return FIError;
continue;
}
// add filter and go on, maybe this one will give us text...
m_handlers.push_back(again);
if (!ipath.empty() &&
!m_handlers.back()->skip_to_document(vipath[m_handlers.size()-1])){
LOGERR(("FileInterner::internfile: can't skip\n"));
return FIError;
}
}
if (m_handlers.empty()) {
LOGERR(("FileInterner::internfile: stack empty\n"));
return FIError;
}
// If indexing compute ipath and significant mimetype
// Note that ipath is returned through the parameter not doc.ipath
if (!m_forPreview)
collectIpathAndMT(doc, ipath);
dijontorcl(doc);
// Destack what can be
while (!m_handlers.empty() && !m_handlers.back()->has_documents()) {
popHandler();
}
if (m_handlers.empty())
return FIDone;
else
return FIAgain;
}
class DirWiper {
public:
string dir;
bool do_it;
DirWiper(string d) : dir(d), do_it(true) {}
~DirWiper() {
if (do_it) {
wipedir(dir);
rmdir(dir.c_str());
}
}
};
bool FileInterner::idocTempFile(TempFile& otemp, RclConfig *cnf,
const string& fn,
const string& ipath,
const string& mtype)
{
string tmpdir, reason;
if (!maketmpdir(tmpdir, reason))
return false;
DirWiper wiper(tmpdir);
struct stat st;
if (stat(fn.c_str(), &st) < 0) {
LOGERR(("FileInterner::idocTempFile: can't stat [%s]\n", fn.c_str()));
return false;
}
FileInterner interner(fn, &st, cnf, tmpdir, &mtype);
interner.setTargetMType(mtype);
Rcl::Doc doc;
string mipath = ipath;
Status ret = interner.internfile(doc, mipath);
if (ret == FileInterner::FIError) {
LOGERR(("FileInterner::idocTempFile: internfile() failed\n"));
return false;
}
TempFile temp(new TempFileInternal(cnf->getSuffixFromMimeType(mtype)));
if (!temp->ok()) {
LOGERR(("FileInterner::idocTempFile: cannot create temporary file"));
return false;
}
int fd = open(temp->filename(), O_WRONLY);
if (fd < 0) {
LOGERR(("FileInterner::idocTempFile: open(%s) failed errno %d\n",
temp->filename(), errno));
return false;
}
const string& dt = doc.text;
if (write(fd, dt.c_str(), dt.length()) != (int)dt.length()) {
close(fd);
LOGERR(("FileInterner::idocTempFile: write to %s failed errno %d\n",
temp->filename(), errno));
return false;
}
close(fd);
otemp = temp;
return true;
}
#else
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <string>
using namespace std;
#include "debuglog.h"
#include "rclinit.h"
#include "internfile.h"
#include "rclconfig.h"
#include "rcldoc.h"
static string thisprog;
static string usage =
" internfile <filename> [ipath]\n"
" \n\n"
;
static void
Usage(void)
{
cerr << thisprog << ": usage:\n" << usage;
exit(1);
}
static int op_flags;
#define OPT_q 0x1
int main(int argc, char **argv)
{
thisprog = argv[0];
argc--; argv++;
while (argc > 0 && **argv == '-') {
(*argv)++;
if (!(**argv))
/* Cas du "adb - core" */
Usage();
while (**argv)
switch (*(*argv)++) {
default: Usage(); break;
}
argc--; argv++;
}
DebugLog::getdbl()->setloglevel(DEBDEB1);
DebugLog::setfilename("stderr");
if (argc < 1)
Usage();
string fn(*argv++);
argc--;
string ipath;
if (argc >= 1) {
ipath.append(*argv++);
argc--;
}
string reason;
RclConfig *config = recollinit(0, 0, reason);
if (config == 0 || !config->ok()) {
string str = "Configuration problem: ";
str += reason;
fprintf(stderr, "%s\n", str.c_str());
exit(1);
}
FileInterner interner(fn, config, "/tmp");
Rcl::Doc doc;
FileInterner::Status status = interner.internfile(doc, ipath);
switch (status) {
case FileInterner::FIDone:
case FileInterner::FIAgain:
break;
case FileInterner::FIError:
default:
fprintf(stderr, "internfile failed\n");
exit(1);
}
cout << "doc.url [[[[" << doc.url <<
"]]]]\n-----------------------------------------------------\n" <<
"doc.ipath [[[[" << doc.ipath <<
"]]]]\n-----------------------------------------------------\n" <<
"doc.mimetype [[[[" << doc.mimetype <<
"]]]]\n-----------------------------------------------------\n" <<
"doc.fmtime [[[[" << doc.fmtime <<
"]]]]\n-----------------------------------------------------\n" <<
"doc.dmtime [[[[" << doc.dmtime <<
"]]]]\n-----------------------------------------------------\n" <<
"doc.origcharset [[[[" << doc.origcharset <<
"]]]]\n-----------------------------------------------------\n" <<
"doc.title [[[[" << doc.title <<
"]]]]\n-----------------------------------------------------\n" <<
"doc.keywords [[[[" << doc.keywords <<
"]]]]\n-----------------------------------------------------\n" <<
"doc.abstract [[[[" << doc.abstract <<
"]]]]\n-----------------------------------------------------\n" <<
"doc.text [[[[" << doc.text << "]]]]\n";
}
#endif // TEST_INTERNFILE
Reference in New Issue View Git Blame Copy Permalink