tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

prettified LOG lines

Browse Source
This commit is contained in:
Jean-Francois Dockes 2017-07-19 19:15:29 +02:00
parent c9c3158121
commit 9f02bc8119
1 changed files with 81 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
src/internfile/internfile.cpp
View File

@ -77,7 +77,8 @@ static string colon_restore(const string& in)
// (ie message having a given attachment)
bool FileInterner::getEnclosingUDI(const Rcl::Doc &doc, string& udi)
{
LOGDEB("FileInterner::getEnclosingUDI(): url [" << (doc.url) << "] ipath [" << (doc.ipath) << "]\n" );
LOGDEB("FileInterner::getEnclosingUDI(): url [" << doc.url <<
"] ipath [" << doc.ipath << "]\n");
string eipath = doc.ipath;
string::size_type colon;
if (eipath.empty())
@ -119,9 +120,9 @@ FileInterner::FileInterner(const string &fn, const struct stat *stp,
RclConfig *cnf, int flags, const string *imime)
: m_ok(false), m_missingdatap(0), m_uncomp((flags & FIF_forPreview) != 0)
{
LOGDEB0("FileInterner::FileInterner(fn=" << (fn) << ")\n" );
LOGDEB0("FileInterner::FileInterner(fn=" << fn << ")\n");
if (fn.empty()) {
LOGERR("FileInterner::FileInterner: empty file name!\n" );
LOGERR("FileInterner::FileInterner: empty file name!\n");
return;
}
initcommon(cnf, flags);
@ -132,7 +133,7 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
int flags, const string *imime)
{
if (f.empty()) {
LOGERR("FileInterner::init: empty file name!\n" );
LOGERR("FileInterner::init: empty file name!\n");
return;
}
m_fn = f;
@ -154,17 +155,17 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
// document, but this can be part of a compound document, and
// we're dealing with the top level file here, or this could be a
// compressed file. The flag tells us we really can use it
// (e.g. the beagle indexer sets it).
// (e.g. the web indexer sets it).
if (flags & FIF_doUseInputMimetype) {
if (!imime) {
LOGERR("FileInterner:: told to use null imime\n" );
LOGERR("FileInterner:: told to use null imime\n");
return;
}
l_mime = *imime;
} else {
LOGDEB("FileInterner::init fn [" << f << "] mime [" <<
(imime?imime->c_str() : "(null)") << "] preview " << m_forPreview
<< "\n" );
LOGDEB("FileInterner::init fn [" << f << "] mime [" <<
(imime ? imime->c_str() : "(null)") << "] preview " <<
m_forPreview << "\n");
// Run mime type identification in any case (see comment above).
l_mime = mimetype(m_fn, stp, m_cfg, usfci);
@ -191,12 +192,13 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
if (!m_uncomp.uncompressfile(m_fn, ucmd, m_tfile)) {
return;
}
LOGDEB1("FileInterner:: after ucomp: tfile " << (m_tfile) << "\n" );
LOGDEB1("FileInterner:: after ucomp: tfile " << m_tfile <<"\n");
m_fn = m_tfile;
// Stat the uncompressed file, mainly to get the size
struct stat ucstat;
if (path_fileprops(m_fn, &ucstat) != 0) {
LOGERR("FileInterner: can't stat the uncompressed file[" << (m_fn) << "] errno " << (errno) << "\n" );
LOGERR("FileInterner: can't stat the uncompressed file[" <<
m_fn << "] errno " << errno << "\n");
return;
} else {
docsize = ucstat.st_size;
@ -205,7 +207,8 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
if (l_mime.empty() && imime)
l_mime = *imime;
} else {
LOGINFO("FileInterner:: " << (m_fn) << " over size limit " << (maxkbs) << " kbs\n" );
LOGINFO("FileInterner:: " << m_fn << " over size limit " <<
maxkbs << " kbs\n");
}
}
}
@ -213,7 +216,7 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
if (l_mime.empty()) {
// No mime type. We let it through as config may warrant that
// we index all file names
LOGDEB0("FileInterner:: no mime: [" << (m_fn) << "]\n" );
LOGDEB0("FileInterner:: no mime: [" << m_fn << "]\n");
}
// Look for appropriate handler (might still return empty)
@ -222,7 +225,8 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
if (!df || df->is_unknown()) {
// No real handler for this type, for now :(
LOGDEB("FileInterner:: unprocessed mime: [" << (l_mime) << "] [" << (f) << "]\n" );
LOGDEB("FileInterner:: unprocessed mime: [" << l_mime << "] [" << f <<
"]\n");
if (!df)
return;
}
@ -242,12 +246,12 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
df->set_docsize(docsize);
if (!df->set_document_file(l_mime, m_fn)) {
delete df;
LOGERR("FileInterner:: error converting " << (m_fn) << "\n" );
LOGERR("FileInterner:: error converting " << m_fn << "\n");
return;
}
m_handlers.push_back(df);
LOGDEB("FileInterner:: init ok " << (l_mime) << " [" << (m_fn) << "]\n" );
LOGDEB("FileInterner:: init ok " << l_mime << " [" << m_fn << "]\n");
m_ok = true;
}
@ -256,7 +260,7 @@ FileInterner::FileInterner(const string &data, RclConfig *cnf,
int flags, const string& imime)
: m_ok(false), m_missingdatap(0), m_uncomp((flags & FIF_forPreview) != 0)
{
LOGDEB0("FileInterner::FileInterner(data)\n" );
LOGDEB0("FileInterner::FileInterner(data)\n");
initcommon(cnf, flags);
init(data, cnf, flags, imime);
}
@ -265,7 +269,7 @@ void FileInterner::init(const string &data, RclConfig *cnf,
int flags, const string& imime)
{
if (imime.empty()) {
LOGERR("FileInterner: inmemory constructor needs input mime type\n" );
LOGERR("FileInterner: inmemory constructor needs input mime type\n");
return;
}
m_mimetype = imime;
@ -276,7 +280,7 @@ void FileInterner::init(const string &data, RclConfig *cnf,
if (!df) {
// No handler for this type, for now :( if indexallfilenames
// is set in the config, this normally wont happen (we get mh_unknown)
LOGDEB("FileInterner:: unprocessed mime [" << (m_mimetype) << "]\n" );
LOGDEB("FileInterner:: unprocessed mime [" << m_mimetype << "]\n");
return;
}
df->set_property(Dijon::Filter::OPERATING_MODE,
@ -297,7 +301,8 @@ void FileInterner::init(const string &data, RclConfig *cnf,
}
}
if (!result) {
LOGINFO("FileInterner:: set_doc failed inside for mtype " << (m_mimetype) << "\n" );
LOGINFO("FileInterner:: set_doc failed inside for mtype " <<
m_mimetype << "\n");
delete df;
return;
}
@ -321,17 +326,17 @@ void FileInterner::initcommon(RclConfig *cnf, int flags)
FileInterner::FileInterner(const Rcl::Doc& idoc, RclConfig *cnf, int flags)
: m_ok(false), m_missingdatap(0), m_uncomp(((flags & FIF_forPreview) != 0))
{
LOGDEB0("FileInterner::FileInterner(idoc)\n" );
LOGDEB0("FileInterner::FileInterner(idoc)\n");
initcommon(cnf, flags);
DocFetcher *fetcher = docFetcherMake(cnf, idoc);
if (fetcher == 0) {
LOGERR("FileInterner:: no backend\n" );
LOGERR("FileInterner:: no backend\n");
return;
}
DocFetcher::RawDoc rawdoc;
if (!fetcher->fetch(cnf, idoc, rawdoc)) {
LOGERR("FileInterner:: fetcher failed\n" );
LOGERR("FileInterner:: fetcher failed\n");
return;
}
switch (rawdoc.kind) {
@ -346,7 +351,7 @@ FileInterner::FileInterner(const Rcl::Doc& idoc, RclConfig *cnf, int flags)
m_direct = true;
break;
default:
LOGERR("FileInterner::FileInterner(idoc): bad rawdoc kind ??\n" );
LOGERR("FileInterner::FileInterner(idoc): bad rawdoc kind ??\n");
}
return;
}
@ -355,7 +360,7 @@ bool FileInterner::makesig(RclConfig *cnf, const Rcl::Doc& idoc, string& sig)
{
DocFetcher *fetcher = docFetcherMake(cnf, idoc);
if (fetcher == 0) {
LOGERR("FileInterner::makesig no backend for doc\n" );
LOGERR("FileInterner::makesig no backend for doc\n");
return false;
}
@ -382,12 +387,13 @@ TempFile FileInterner::dataToTempFile(const string& dt, const string& mt)
// Create temp file with appropriate suffix for mime type
TempFile temp(new TempFileInternal(m_cfg->getSuffixFromMimeType(mt)));
if (!temp->ok()) {
LOGERR("FileInterner::dataToTempFile: cant create tempfile: " << (temp->getreason()) << "\n" );
LOGERR("FileInterner::dataToTempFile: cant create tempfile: " <<
temp->getreason() << "\n");
return TempFile();
}
string reason;
if (!stringtofile(dt, temp->filename(), reason)) {
LOGERR("FileInterner::dataToTempFile: stringtofile: " << (reason) << "\n" );
LOGERR("FileInterner::dataToTempFile: stringtofile: " <<reason << "\n");
return TempFile();
}
return temp;
@ -398,7 +404,7 @@ TempFile FileInterner::dataToTempFile(const string& dt, const string& mt)
// RECFILTERROR HELPERNOTFOUND program1 [program2 ...]
void FileInterner::checkExternalMissing(const string& msg, const string& mt)
{
LOGDEB2("checkExternalMissing: [" << (msg) << "]\n" );
LOGDEB2("checkExternalMissing: [" << msg << "]\n");
if (m_missingdatap && msg.find("RECFILTERROR") == 0) {
vector<string> verr;
stringToStrings(msg, verr);
@ -490,10 +496,10 @@ static inline bool getKeyValue(const map<string, string>& docdata,
it = docdata.find(key);
if (it != docdata.end()) {
value = it->second;
LOGDEB2("getKeyValue: [" << (key) << "]->[" << (value) << "]\n" );
LOGDEB2("getKeyValue: [" << key << "]->[" << value << "]\n");
return true;
}
LOGDEB2("getKeyValue: no value for [" << (key) << "]\n" );
LOGDEB2("getKeyValue: no value for [" << key << "]\n");
return false;
}
@ -502,7 +508,7 @@ bool FileInterner::dijontorcl(Rcl::Doc& doc)
RecollFilter *df = m_handlers.back();
if (df == 0) {
//??
LOGERR("FileInterner::dijontorcl: null top handler ??\n" );
LOGERR("FileInterner::dijontorcl: null top handler ??\n");
return false;
}
const map<string, string>& docdata = df->get_meta_data();
@ -571,7 +577,7 @@ bool FileInterner::dijontorcl(Rcl::Doc& doc)
// actually complicated.
void FileInterner::collectIpathAndMT(Rcl::Doc& doc) const
{
LOGDEB2("FileInterner::collectIpathAndMT\n" );
LOGDEB2("FileInterner::collectIpathAndMT\n");
bool hasipath = false;
if (!m_noxattrs) {
@ -618,7 +624,7 @@ void FileInterner::collectIpathAndMT(Rcl::Doc& doc) const
// Trim empty tail elements in ipath.
if (hasipath) {
LOGDEB2("IPATH [" << (doc.ipath) << "]\n" );
LOGDEB2("IPATH [" << doc.ipath << "]\n");
string::size_type sit = doc.ipath.find_last_not_of(cstr_isep);
if (sit == string::npos)
doc.ipath.erase();
@ -655,7 +661,7 @@ int FileInterner::addHandler()
getKeyValue(docdata, cstr_dj_keymt, mimetype);
LOGDEB("FileInterner::addHandler: next_doc is " << mimetype <<
" target [" << m_targetMType << "]\n");
" target [" << m_targetMType << "]\n");
// If we find a document of the target type (text/plain in
// general), we're done decoding. If we hit text/plain, we're done
@ -689,7 +695,7 @@ int FileInterner::addHandler()
if (!newflt) {
// If we can't find a handler, this doc can't be handled
// but there can be other ones so we go on
LOGINFO("FileInterner::addHandler: no filter for [" << mimetype <<
LOGINFO("FileInterner::addHandler: no filter for [" << mimetype <<
"]\n");
return ADD_CONTINUE;
}
@ -748,19 +754,21 @@ void FileInterner::processNextDocError(Rcl::Doc &doc)
collectIpathAndMT(doc);
m_reason = m_handlers.back()->get_error();
checkExternalMissing(m_reason, doc.mimetype);
LOGERR("FileInterner::internfile: next_document error [" << (m_fn) << "" << (doc.ipath.empty() ? "" : "|") << "" << (doc.ipath) << "] " << (doc.mimetype) << " " << (m_reason) << "\n" );
LOGERR("FileInterner::internfile: next_document error [" << m_fn <<
(doc.ipath.empty() ? "" : "|") << doc.ipath << "] " <<
doc.mimetype << " " << m_reason << "\n");
}
FileInterner::Status FileInterner::internfile(Rcl::Doc& doc, const string& ipath)
FileInterner::Status FileInterner::internfile(Rcl::Doc& doc,const string& ipath)
{
LOGDEB("FileInterner::internfile. ipath [" << (ipath) << "]\n" );
LOGDEB("FileInterner::internfile. ipath [" << ipath << "]\n");
// Get rid of possible image tempfile from older call
m_imgtmp.reset();
if (m_handlers.size() < 1) {
// Just means the constructor failed
LOGDEB("FileInterner::internfile: no handler: constructor failed\n" );
LOGDEB("FileInterner::internfile: no handler: constructor failed\n");
return FIError;
}
@ -779,7 +787,7 @@ FileInterner::Status FileInterner::internfile(Rcl::Doc& doc, const string& ipath
}
vipath.insert(vipath.begin(), lipath.begin(), lipath.end());
if (!m_handlers.back()->skip_to_document(vipath[m_handlers.size()-1])){
LOGERR("FileInterner::internfile: can't skip\n" );
LOGERR("FileInterner::internfile: can't skip\n");
return FIError;
}
}
@ -795,7 +803,7 @@ FileInterner::Status FileInterner::internfile(Rcl::Doc& doc, const string& ipath
while (!m_handlers.empty()) {
CancelCheck::instance().checkCancel();
if (loop++ > 1000) {
LOGERR("FileInterner:: looping!\n" );
LOGERR("FileInterner:: looping!\n");
return FIError;
}
// If there are no more docs at the current top level we pop and
@ -807,7 +815,7 @@ FileInterner::Status FileInterner::internfile(Rcl::Doc& doc, const string& ipath
if (m_forPreview) {
m_reason += "Requested document does not exist. ";
m_reason += m_handlers.back()->get_error();
LOGERR("FileInterner: requested document does not exist\n" );
LOGERR("FileInterner: requested document does not exist\n");
return FIError;
}
popHandler();
@ -822,7 +830,7 @@ FileInterner::Status FileInterner::internfile(Rcl::Doc& doc, const string& ipath
if (m_forPreview) {
m_reason += "Requested document does not exist. ";
m_reason += m_handlers.back()->get_error();
LOGERR("FileInterner: requested document does not exist\n" );
LOGERR("FileInterner: requested document does not exist\n");
return FIError;
}
popHandler();
@ -833,20 +841,20 @@ FileInterner::Status FileInterner::internfile(Rcl::Doc& doc, const string& ipath
// handler to stack.
switch (addHandler()) {
case ADD_OK: // Just go through: handler has been stacked, use it
LOGDEB2("addHandler returned OK\n" );
LOGDEB2("addHandler returned OK\n");
break;
case ADD_CONTINUE:
// forget this doc and retrieve next from current handler
// (ipath stays same)
LOGDEB2("addHandler returned CONTINUE\n" );
LOGDEB2("addHandler returned CONTINUE\n");
continue;
case ADD_BREAK:
// Stop looping: doc type ok, need complete its processing
// and return it
LOGDEB2("addHandler returned BREAK\n" );
LOGDEB2("addHandler returned BREAK\n");
goto breakloop; // when you have to you have to
case ADD_ERROR:
LOGDEB2("addHandler returned ERROR\n" );
LOGDEB2("addHandler returned ERROR\n");
return FIError;
}
@ -860,14 +868,14 @@ FileInterner::Status FileInterner::internfile(Rcl::Doc& doc, const string& ipath
if (!ipath.empty()) {
if (m_handlers.size() <= vipath.size() &&
!m_handlers.back()->skip_to_document(vipath[m_handlers.size()-1])) {
LOGERR("FileInterner::internfile: can't skip\n" );
LOGERR("FileInterner::internfile: can't skip\n");
return FIError;
}
}
}
breakloop:
if (m_handlers.empty()) {
LOGDEB("FileInterner::internfile: conversion ended with no doc\n" );
LOGDEB("FileInterner::internfile: conversion ended with no doc\n");
return FIError;
}
@ -910,7 +918,7 @@ bool FileInterner::tempFileForMT(TempFile& otemp, RclConfig* cnf,
TempFile temp(new TempFileInternal(
cnf->getSuffixFromMimeType(mimetype)));
if (!temp->ok()) {
LOGERR("FileInterner::interntofile: can't create temp file\n" );
LOGERR("FileInterner::interntofile: can't create temp file\n");
return false;
}
otemp = temp;
@ -936,7 +944,7 @@ bool FileInterner::tempFileForMT(TempFile& otemp, RclConfig* cnf,
bool FileInterner::idocToFile(TempFile& otemp, const string& tofile,
RclConfig *cnf, const Rcl::Doc& idoc)
{
LOGDEB("FileInterner::idocToFile\n" );
LOGDEB("FileInterner::idocToFile\n");
if (idoc.ipath.empty()) {
return topdocToFile(otemp, tofile, cnf, idoc);
@ -955,12 +963,12 @@ bool FileInterner::topdocToFile(TempFile& otemp, const string& tofile,
{
DocFetcher *fetcher = docFetcherMake(cnf, idoc);
if (fetcher == 0) {
LOGERR("FileInterner::idocToFile no backend\n" );
LOGERR("FileInterner::idocToFile no backend\n");
return false;
}
DocFetcher::RawDoc rawdoc;
if (!fetcher->fetch(cnf, idoc, rawdoc)) {
LOGERR("FileInterner::idocToFile fetcher failed\n" );
LOGERR("FileInterner::idocToFile fetcher failed\n");
return false;
}
const char *filename = "";
@ -977,18 +985,18 @@ bool FileInterner::topdocToFile(TempFile& otemp, const string& tofile,
switch (rawdoc.kind) {
case DocFetcher::RawDoc::RDK_FILENAME:
if (!copyfile(rawdoc.data.c_str(), filename, reason)) {
LOGERR("FileInterner::idocToFile: copyfile: " << (reason) << "\n" );
LOGERR("FileInterner::idocToFile: copyfile: " << reason << "\n");
return false;
}
break;
case DocFetcher::RawDoc::RDK_DATA:
if (!stringtofile(rawdoc.data, filename, reason)) {
LOGERR("FileInterner::idocToFile: stringtofile: " << (reason) << "\n" );
LOGERR("FileInterner::idocToFile: stringtofile: " << reason <<"\n");
return false;
}
break;
default:
LOGERR("FileInterner::FileInterner(idoc): bad rawdoc kind ??\n" );
LOGERR("FileInterner::FileInterner(idoc): bad rawdoc kind ??\n");
}
if (tofile.empty())
@ -1000,13 +1008,13 @@ bool FileInterner::interntofile(TempFile& otemp, const string& tofile,
const string& ipath, const string& mimetype)
{
if (!ok()) {
LOGERR("FileInterner::interntofile: constructor failed\n" );
LOGERR("FileInterner::interntofile: constructor failed\n");
return false;
}
Rcl::Doc doc;
Status ret = internfile(doc, ipath);
if (ret == FileInterner::FIError) {
LOGERR("FileInterner::interntofile: internfile() failed\n" );
LOGERR("FileInterner::interntofile: internfile() failed\n");
return false;
}
@ -1033,7 +1041,7 @@ bool FileInterner::interntofile(TempFile& otemp, const string& tofile,
}
string reason;
if (!stringtofile(doc.text, filename, reason)) {
LOGERR("FileInterner::interntofile: stringtofile : " << (reason) << "\n" );
LOGERR("FileInterner::interntofile: stringtofile : " << reason << "\n");
return false;
}
@ -1044,15 +1052,16 @@ bool FileInterner::interntofile(TempFile& otemp, const string& tofile,
bool FileInterner::isCompressed(const string& fn, RclConfig *cnf)
{
LOGDEB("FileInterner::isCompressed: [" << (fn) << "]\n" );
LOGDEB("FileInterner::isCompressed: [" << fn << "]\n");
struct stat st;
if (path_fileprops(fn, &st) < 0) {
LOGERR("FileInterner::isCompressed: can't stat [" << (fn) << "]\n" );
LOGERR("FileInterner::isCompressed: can't stat [" << fn << "]\n");
return false;
}
string l_mime = mimetype(fn, &st, cnf, true);
if (l_mime.empty()) {
LOGERR("FileInterner::isUncompressed: can't get mime for [" << (fn) << "]\n" );
LOGERR("FileInterner::isUncompressed: can't get mime for [" << fn <<
"]\n");
return false;
}
@ -1067,15 +1076,16 @@ bool FileInterner::isCompressed(const string& fn, RclConfig *cnf)
bool FileInterner::maybeUncompressToTemp(TempFile& temp, const string& fn,
RclConfig *cnf, const Rcl::Doc& doc)
{
LOGDEB("FileInterner::maybeUncompressToTemp: [" << (fn) << "]\n" );
LOGDEB("FileInterner::maybeUncompressToTemp: [" << fn << "]\n");
struct stat st;
if (path_fileprops(fn.c_str(), &st) < 0) {
LOGERR("FileInterner::maybeUncompressToTemp: can't stat [" << (fn) << "]\n" );
LOGERR("FileInterner::maybeUncompressToTemp: can't stat [" <<fn<<"]\n");
return false;
}
string l_mime = mimetype(fn, &st, cnf, true);
if (l_mime.empty()) {
LOGERR("FileInterner::maybeUncompress.: can't id. mime for [" << (fn) << "]\n" );
LOGERR("FileInterner::maybeUncompress.: can't id. mime for [" <<
fn << "]\n");
return false;
}
@ -1087,13 +1097,14 @@ bool FileInterner::maybeUncompressToTemp(TempFile& temp, const string& fn,
int maxkbs = -1;
if (cnf->getConfParam("compressedfilemaxkbs", &maxkbs) &&
maxkbs >= 0 && int(st.st_size / 1024) > maxkbs) {
LOGINFO("FileInterner:: " << (fn) << " over size limit " << (maxkbs) << " kbs\n" );
LOGINFO("FileInterner:: " << fn << " over size limit " << maxkbs <<
" kbs\n");
return false;
}
temp =
TempFile(new TempFileInternal(cnf->getSuffixFromMimeType(doc.mimetype)));
if (!temp->ok()) {
LOGERR("FileInterner: cant create temporary file" );
LOGERR("FileInterner: cant create temporary file\n");
return false;
}
@ -1108,7 +1119,8 @@ bool FileInterner::maybeUncompressToTemp(TempFile& temp, const string& fn,
// uncompressed file, hopefully staying on the same dev.
string reason;
if (!renameormove(uncomped.c_str(), temp->filename(), reason)) {
LOGERR("FileInterner::maybeUncompress: move [" << (uncomped) << "] -> [" << (temp->filename()) << "] failed: " << (reason) << "\n" );
LOGERR("FileInterner::maybeUncompress: move [" << uncomped <<
"] -> [" << temp->filename() << "] failed: " << reason << "\n");
return false;
}
return true;
@ -1141,7 +1153,7 @@ static string usage =
static void
Usage(void)
{
cerr << thisprog << ": usage:\n" << usage;
cerr << thisprog << ": usage:\n" << usage;
exit(1);
}