tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

internal xslt working for single-sheet (abw). Still leaking memory?

Browse Source
This commit is contained in:
Jean-Francois Dockes
2018-12-25 10:57:26 +01:00
parent 2bd4b5ef4f
commit 00c0c5168b
13 changed files with 9812 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/internfile/mh_mail.h
+1 -3
View File
@@ -38,9 +38,7 @@ public:
MimeHandlerMail(RclConfig *cnf, const std::string &id);
virtual ~MimeHandlerMail();
virtual bool is_data_input_ok(DataInput input) const {
if (input == DOCUMENT_FILE_NAME || input == DOCUMENT_STRING)
return true;
return false;
return (input == DOCUMENT_FILE_NAME || input == DOCUMENT_STRING);
}
virtual bool next_document();
virtual bool skip_to_document(const std::string& ipath);
src/internfile/mh_xslt.cpp
+232
View File
@@ -0,0 +1,232 @@
/* Copyright (C) 2005 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "autoconfig.h"
#include <libxml/parser.h>
#include <libxml/tree.h>
#include <libxslt/transform.h>
#include <libxslt/xsltInternals.h>
#include <libxslt/xsltutils.h>
#include "cstr.h"
#include "mh_xslt.h"
#include "log.h"
#include "smallut.h"
#include "md5ut.h"
#include "rclconfig.h"
#include "readfile.h"
using namespace std;
class FileScanXML : public FileScanDo {
public:
FileScanXML(const string& fn) : m_fn(fn) {}
virtual ~FileScanXML() {
if (ctxt) {
xmlFreeParserCtxt(ctxt);
}
}
xmlDocPtr getDoc() {
int ret;
if ((ret = xmlParseChunk(ctxt, nullptr, 0, 1))) {
xmlError *error = xmlGetLastError();
LOGERR("FileScanXML: final xmlParseChunk failed with error " <<
ret << " error: " <<
(error ? error->message :
" null return from xmlGetLastError()") << "\n");
return nullptr;
}
return ctxt->myDoc;
}
virtual bool init(int64_t size, string *) {
LOGDEB1("FileScanXML: init: size " << size << endl);
ctxt = xmlCreatePushParserCtxt(NULL, NULL, NULL, 0, m_fn.c_str());
if (ctxt == nullptr) {
LOGERR("FileScanXML: xmlCreatePushParserCtxt failed\n");
return false;
} else {
return true;
}
}
virtual bool data(const char *buf, int cnt, string*) {
if (0) {
string dt(buf, cnt);
LOGDEB1("FileScanXML: data: cnt " << cnt << " data " << dt << endl);
} else {
LOGDEB1("FileScanXML: data: cnt " << cnt << endl);
}
int ret;
if ((ret = xmlParseChunk(ctxt, buf, cnt, 0))) {
xmlError *error = xmlGetLastError();
LOGERR("FileScanXML: xmlParseChunk failed with error " <<
ret << " for [" << buf << "] error " <<
(error ? error->message :
" null return from xmlGetLastError()") << "\n");
return false;
} else {
LOGDEB1("xmlParseChunk ok (sent " << cnt << " bytes)\n");
return true;
}
}
private:
xmlParserCtxtPtr ctxt{nullptr};
string m_fn;
};
class MimeHandlerXslt::Internal {
public:
~Internal() {
if (metaOrAllSS) {
xsltFreeStylesheet(metaOrAllSS);
}
if (dataSS) {
xsltFreeStylesheet(dataSS);
}
}
bool ok{false};
xsltStylesheet *metaOrAllSS{nullptr};
xsltStylesheet *dataSS{nullptr};
string result;
};
MimeHandlerXslt::~MimeHandlerXslt()
{
delete m;
}
MimeHandlerXslt::MimeHandlerXslt(RclConfig *cnf, const std::string& id,
const std::vector<std::string>& params)
: RecollFilter(cnf, id), m(new Internal)
{
LOGDEB("MimeHandlerXslt: params: " << stringsToString(params) << endl);
string filtersdir = path_cat(cnf->getDatadir(), "filters");
xmlSubstituteEntitiesDefault(0);
xmlLoadExtDtdDefaultValue = 0;
// params can be "xslt stylesheetall" or
// "xslt metamember stylesheetmeta datamember stylesheetdata"
if (params.size() == 2) {
string ssfn = path_cat(filtersdir, params[1]);
FileScanXML XMLstyle(ssfn);
string reason;
if (!file_scan(ssfn, &XMLstyle, &reason)) {
LOGERR("MimeHandlerXslt: file_scan failed for style sheet " <<
ssfn << " : " << reason << endl);
return;
}
xmlDoc *stl = XMLstyle.getDoc();
if (stl == nullptr) {
LOGERR("MimeHandlerXslt: getDoc failed for style sheet " <<
ssfn << endl);
return;
}
m->metaOrAllSS = xsltParseStylesheetDoc(stl);
if (m->metaOrAllSS) {
m->ok = true;
}
} else if (params.size() == 4) {
} else {
LOGERR("MimeHandlerXslt: constructor with wrong param vector: " <<
stringsToString(params) << endl);
}
}
bool MimeHandlerXslt::set_document_file_impl(const std::string& mt,
const std::string &file_path)
{
LOGDEB0("MimeHandlerXslt::set_document_file_: fn: " << file_path << endl);
if (!m || !m->ok) {
return false;
}
if (nullptr == m->dataSS) {
if (nullptr == m->metaOrAllSS) {
LOGERR("MimeHandlerXslt::set_document_file_impl: both ss empty??\n");
return false;
}
FileScanXML XMLdoc(file_path);
string md5, reason;
if (!file_scan(file_path, &XMLdoc, 0, -1, &reason,
m_forPreview ? nullptr : &md5)) {
LOGERR("MimeHandlerXslt::set_document_file_impl: file_scan failed "
"for " << file_path << " : " << reason << endl);
return false;
}
if (!m_forPreview) {
m_metaData[cstr_dj_keymd5] = md5;
}
xmlDocPtr doc = XMLdoc.getDoc();
if (nullptr == doc) {
LOGERR("MimeHandlerXslt::set_doc_file_impl: no parsed doc\n");
return false;
}
xmlDocPtr transformed = xsltApplyStylesheet(m->metaOrAllSS, doc, NULL);
if (nullptr == transformed) {
LOGERR("MimeHandlerXslt::set_doc_file_: xslt transform failed\n");
xmlFreeDoc(doc);
return false;
}
xmlChar *outstr;
int outlen;
xsltSaveResultToString(&outstr, &outlen, transformed, m->metaOrAllSS);
m->result = string((const char*)outstr, outlen);
xmlFree(outstr);
xmlFreeDoc(transformed);
xmlFreeDoc(doc);
} else {
LOGERR("Not ready for multipart yet\n");
abort();
}
m_havedoc = true;
return true;
}
bool MimeHandlerXslt::set_document_string_impl(const string& mt,
const string& msgtxt)
{
if (!m || !m->ok) {
return false;
}
return true;
}
bool MimeHandlerXslt::next_document()
{
if (!m || !m->ok) {
return false;
}
if (m_havedoc == false)
return false;
m_havedoc = false;
m_metaData[cstr_dj_keymt] = cstr_texthtml;
m_metaData[cstr_dj_keycontent].swap(m->result);
LOGDEB1("MimeHandlerXslt::next_document: result: [" <<
m_metaData[cstr_dj_keycontent] << "]\n");
return true;
}
void MimeHandlerXslt::clear_impl()
{
m_havedoc = false;
m->result.clear();
}
src/internfile/mh_xslt.h
+49
View File
@@ -0,0 +1,49 @@
/* Copyright (C) 2018 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef _MH_XSLT_H_INCLUDED_
#define _MH_XSLT_H_INCLUDED_
#include <string>
#include "mimehandler.h"
class MimeHandlerXslt : public RecollFilter {
public:
MimeHandlerXslt(RclConfig *cnf, const std::string& id,
const std::vector<std::string>& params);
virtual ~MimeHandlerXslt();
virtual bool next_document() override;
virtual void clear_impl() override;
virtual bool is_data_input_ok(DataInput input) const override {
return (input == DOCUMENT_FILE_NAME || input == DOCUMENT_STRING);
}
protected:
virtual bool set_document_file_impl(const std::string& mt,
const std::string& file_path);
virtual bool set_document_string_impl(const std::string& mt,
const std::string& data);
class Internal;
private:
Internal *m{nullptr};
};
#endif /* _MH_XSLT_H_INCLUDED_ */
src/internfile/mimehandler.cpp
+30 -24
View File
@@ -41,6 +41,7 @@ using namespace std;
#include "mh_symlink.h"
#include "mh_unknown.h"
#include "mh_null.h"
#include "mh_xslt.h"
// Performance help: we use a pool of already known and created
// handlers. There can be several instances for a given mime type
@@ -137,11 +138,17 @@ void clearMimeHandlerCache()
/** For mime types set as "internal" in mimeconf:
* create appropriate handler object. */
static RecollFilter *mhFactory(RclConfig *config, const string &mime,
static RecollFilter *mhFactory(RclConfig *config, const string &mimeOrParams,
bool nobuild, string& id)
{
LOGDEB2("mhFactory(" << mime << ")\n");
string lmime(mime);
LOGDEB1("mhFactory(" << mimeOrParams << ")\n");
vector<string> lparams;
stringToStrings(mimeOrParams, lparams);
if (lparams.empty()) {
// ??
return nullptr;
}
string lmime(lparams[0]);
stringtolower(lmime);
if (cstr_textplain == lmime) {
LOGDEB2("mhFactory(" << mime << "): returning MimeHandlerText\n");
@@ -160,11 +167,11 @@ static RecollFilter *mhFactory(RclConfig *config, const string &mime,
MD5String("MimeHandlerMail", id);
return nobuild ? 0 : new MimeHandlerMail(config, id);
} else if ("inode/symlink" == lmime) {
LOGDEB2("mhFactory(" << mime << "): ret MimeHandlerSymlink\n");
LOGDEB2("mhFactory(" << mime << "): returning MimeHandlerSymlink\n");
MD5String("MimeHandlerSymlink", id);
return nobuild ? 0 : new MimeHandlerSymlink(config, id);
} else if ("application/x-zerosize" == lmime) {
LOGDEB("mhFactory(" << mime << "): ret MimeHandlerNull\n");
LOGDEB("mhFactory(" << lmime << "): returning MimeHandlerNull\n");
MD5String("MimeHandlerNull", id);
return nobuild ? 0 : new MimeHandlerNull(config, id);
} else if (lmime.find("text/") == 0) {
@@ -175,7 +182,11 @@ static RecollFilter *mhFactory(RclConfig *config, const string &mime,
// exec) but still opening with a specific editor.
LOGDEB2("mhFactory(" << mime << "): returning MimeHandlerText(x)\n");
MD5String("MimeHandlerText", id);
return nobuild ? 0 : new MimeHandlerText(config, id);
return nobuild ? 0 : new MimeHandlerText(config, id);
} else if ("xsltproc" == lmime) {
// XML Types processed with one or several xslt style sheets.
MD5String(mimeOrParams, id);
return nobuild ? 0 : new MimeHandlerXslt(config, id, lparams);
} else {
// We should not get there. It means that "internal" was set
// as a handler in mimeconf for a mime type we actually can't
@@ -262,7 +273,7 @@ MimeHandlerExec *mhExecFactory(RclConfig *cfg, const string& mtype, string& hs,
/* Get handler/filter object for given mime type: */
RecollFilter *getMimeHandler(const string &mtype, RclConfig *cfg,
bool filtertypes)
bool filtertypes)
{
LOGDEB("getMimeHandler: mtype [" << mtype << "] filtertypes " <<
filtertypes << "\n");
@@ -291,7 +302,7 @@ RecollFilter *getMimeHandler(const string &mtype, RclConfig *cfg,
}
bool internal = !stringlowercmp("internal", handlertype);
if (internal) {
// For internal types let the factory compute the id
// For internal types let the factory compute the cache id
mhFactory(cfg, cmdstr.empty() ? mtype : cmdstr, true, id);
} else {
// exec/execm: use the md5 of the def line
@@ -304,16 +315,15 @@ RecollFilter *getMimeHandler(const string &mtype, RclConfig *cfg,
goto out;
LOGDEB2("getMimeHandler: " << mtype << " not in cache\n");
// Not in cache.
if (internal) {
// If there is a parameter after "internal" it's the mime
// type to use. This is so that we can have bogus mime
// types like text/x-purple-html-log (for ie: specific
// icon) and still use the html filter on them. This is
// partly redundant with the localfields/rclaptg, but
// better and the latter will probably go away at some
// point in the future.
// type to use, or the further qualifier (e.g. style sheet
// name for xslt types). This is so that we can have bogus
// mime types like text/x-purple-html-log (for ie:
// specific icon) and still use the html filter on
// them. This is partly redundant with the
// localfields/rclaptg, but better? (and the latter will
// probably go away at some point in the future?).
LOGDEB2("handlertype internal, cmdstr [" << cmdstr << "]\n");
h = mhFactory(cfg, cmdstr.empty() ? mtype : cmdstr, false, id);
goto out;
@@ -336,14 +346,10 @@ RecollFilter *getMimeHandler(const string &mtype, RclConfig *cfg,
goto out;
}
}
}
// We get here if there was no specific error, but there is no
// identified mime type, or no handler associated.
// Finally, unhandled files are either ignored or their name and
// generic metadata is indexed, depending on configuration
{
} else {
// No identified mime type, or no handler associated.
// Unhandled files are either ignored or their name and
// generic metadata is indexed, depending on configuration
bool indexunknown = false;
cfg->getConfParam("indexallfilenames", &indexunknown);
if (indexunknown) {