tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

execm persistent filters

Browse Source
This commit is contained in:
dockes
2009-10-09 13:58:32 +00:00
parent 3223d1245a
commit c8a88029f7
9 changed files with 389 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/internfile/mh_exec.cpp
+9 -3
View File
@@ -108,7 +108,15 @@ bool MimeHandlerExec::next_document()
return false;
}
// if output is text, we must handle the conversion to utf-8
finaldetails();
return true;
}
void MimeHandlerExec::finaldetails()
{
string& output = m_metaData["content"];
// if output is text/plain (not text/html), we must convert it to utf-8
string charset = cfgCharset.empty() ? "utf-8" : cfgCharset;
string mt = cfgMtype.empty() ? "text/html" : cfgMtype;
if (!mt.compare("text/plain") && charset.compare("utf-8")) {
@@ -139,6 +147,4 @@ bool MimeHandlerExec::next_document()
LOGERR(("MimeHandlerExec: cant compute md5 for [%s]: %s\n",
m_fn.c_str(), reason.c_str()));
}
return true;
}
src/internfile/mh_exec.h
+19 -7
View File
@@ -30,21 +30,31 @@ using std::string;
*
* The command to execute, and its parameters, are stored in the "params"
* which is built in mimehandler.cpp out of data from the mimeconf file.
*
* As any RecollFilter, a MimeHandlerExec object can be reset
* by calling clear(), and will stay initialised for the same mtype
* (cmd, params etc.)
*/
class MimeHandlerExec : public RecollFilter {
public:
// Members not reset by clear(). params, cfgMtype and chgCharset
// actually define what I am. missingHelper is a permanent error
///////////////////////
// Members not reset by clear(). params, cfgMtype and chgCharset
// define what I am. missingHelper is a permanent error
// (no use to try and execute over and over something that's not
// here).
// Parameter list: this has been built by our creator, from config file
// data. We always add the file name at the end before actual execution
list<string> params;
// The defaults for external filters is to output html except if defined
// otherwise in the config.
// Filter output type. The default for ext. filters is to output html,
// but some don't, in which case the type is defined in the config.
string cfgMtype;
// For ext programs which don't output html, the output charset
// has to be known: ie they have a --charset utf-8 like option.
// Output character set if the above type is not text/html. For
// those filters, the output charset has to be known: ie set by a command
// line option.
string cfgCharset;
bool missingHelper;
////////////////
MimeHandlerExec(const string& mt) : RecollFilter(mt), missingHelper(false)
{}
@@ -66,9 +76,11 @@ class MimeHandlerExec : public RecollFilter {
RecollFilter::clear();
}
private:
protected:
string m_fn;
string m_ipath;
virtual void finaldetails();
};
#endif /* _MH_EXEC_H_INCLUDED_ */
src/internfile/mh_execm.cpp
+157
View File
@@ -0,0 +1,157 @@
#ifndef lint
static char rcsid[] = "@(#$Id: mh_exec.cpp,v 1.14 2008-10-09 09:19:37 dockes Exp $ (C) 2005 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include <iostream>
#include <sstream>
#include "mh_execm.h"
#include "mh_html.h"
#include "debuglog.h"
#include "cancelcheck.h"
#include "smallut.h"
#include "transcode.h"
#include "md5.h"
#include <sys/types.h>
#include <sys/wait.h>
#ifndef NO_NAMESPACES
using namespace std;
#endif /* NO_NAMESPACES */
bool MimeHandlerExecMultiple::startCmd()
{
LOGDEB(("MimeHandlerExecMultiple::startCmd\n"));
// Command name
string cmd = params.front();
// Build parameter list: delete cmd name
list<string>::iterator it = params.begin();
list<string>myparams(++it, params.end());
// Start filter
m_cmd.putenv(m_forPreview ? "RECOLL_FILTER_FORPREVIEW=yes" :
"RECOLL_FILTER_FORPREVIEW=no");
if (m_cmd.startExec(cmd, myparams, 1, 1) < 0) {
missingHelper = true;
return false;
}
return true;
}
bool MimeHandlerExecMultiple::readDataElement(string& name)
{
string ibuf;
if (m_cmd.getline(ibuf) <= 0) {
LOGERR(("MHExecMultiple: getline error\n"));
return false;
}
if (!ibuf.compare("\n")) {
LOGDEB(("MHExecMultiple: Got empty line\n"));
name = "";
return true;
}
// We're expecting something like paramname: len\n
list<string> tokens;
stringToTokens(ibuf, tokens);
if (tokens.size() != 2) {
LOGERR(("MHExecMultiple: bad line in filter output: [%s]\n",
ibuf.c_str()));
return false;
}
list<string>::iterator it = tokens.begin();
name = *it++;
string& slen = *it;
int len;
if (sscanf(slen.c_str(), "%d", &len) != 1) {
LOGERR(("MHExecMultiple: bad line in filter output: [%s]\n",
ibuf.c_str()));
return false;
}
LOGDEB(("MHExecMultiple: got paramname [%s] len: %d\n",
name.c_str(), len));
// We only care about the "data:" field for now
string discard;
string *datap;
if (!stringlowercmp("data:", name)) {
datap = &m_metaData["content"];
} else {
datap = &discard;
}
// Then the data.
datap->erase();
if (m_cmd.receive(*datap, len) != len) {
LOGERR(("MHExecMultiple: expected %d bytes of data, got %d\n",
len, datap->length()));
return false;
}
return true;
}
// Execute an external program to translate a file from its native
// format to text or html.
bool MimeHandlerExecMultiple::next_document()
{
if (m_havedoc == false)
return false;
if (missingHelper) {
LOGDEB(("MHExecMultiple::next_document(): helper known missing\n"));
return false;
}
if (params.empty()) {
// Hu ho
LOGERR(("MHExecMultiple::mkDoc: empty params\n"));
m_reason = "RECFILTERROR BADCONFIG";
return false;
}
if (m_cmd.getChildPid() < 0 && !startCmd()) {
return false;
}
// Send request to child process
ostringstream obuf;
obuf << "FileName: " << m_fn.length() << endl << m_fn << endl;
if (m_cmd.send(obuf.str()) < 0) {
LOGERR(("MHExecMultiple: send error\n"));
return false;
}
// Read answer
LOGDEB(("MHExecMultiple: reading answer\n"));
for (int loop=0;;loop++) {
string name;
if (!readDataElement(name)) {
return false;
}
if (name.empty())
break;
if (loop == 10) {
// ??
LOGERR(("MHExecMultiple: filter sent too many parameters\n"));
return false;
}
}
finaldetails();
m_havedoc = false;
return true;
}
src/internfile/mh_execm.h
+79
View File
@@ -0,0 +1,79 @@
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef _MH_EXECM_H_INCLUDED_
#define _MH_EXECM_H_INCLUDED_
/* @(#$Id: mh_exec.h,v 1.8 2008-10-06 06:22:46 dockes Exp $ (C) 2004 J.F.Dockes */
#include "mh_exec.h"
#include "execmd.h"
/**
* Turn external document into internal one by executing an external filter.
*
* The command to execute, and its parameters, are stored in the "params"
* which is built in mimehandler.cpp out of data from the mimeconf file.
*
* This version uses persistent filters which can handle multiple requests
* without exiting, with a simple question/response protocol.
*
* The data is exchanged in TLV fashion, in a way that should be
* usable in most script languages. The basic unit has one line with a
* data type and a count, followed by the data. A 'message' ends with
* one empty line. A possible exchange:
*
* From recollindex (the message begins before 'Filename'):
*
Filename: 24
/my/home/mail/somefolderIpath: 2
22
<Message ends here: because of the empty line after '22'
*
* Example answer:
*
Mimetype: 10
text/plainData: 10
0123456789
<Message ends here because of empty line
*
* Until proven otherwise, this format is both extensible and
* reasonably easy to parse. While it's more destined for python or
* perl on the script side, it should even be sort of usable from the shell
* (ie: use dd to read the counted data). Most alternatives would need data
* encoding in some cases.
*/
class MimeHandlerExecMultiple : public MimeHandlerExec {
/////////
// Things not reset by "clear()", additionally to those in MimeHandlerExec
ExecCmd m_cmd;
/////// End un-cleared stuff.
public:
MimeHandlerExecMultiple(const string& mt)
: MimeHandlerExec(mt)
{}
// No resources to clean up, the ExecCmd destructor does it.
virtual ~MimeHandlerExecMultiple() {}
virtual bool next_document();
virtual void clear() {
MimeHandlerExec::clear();
}
private:
bool startCmd();
bool readDataElement(string& name);
};
#endif /* _MH_EXECM_H_INCLUDED_ */
src/internfile/mimehandler.cpp
+14 -3
View File
@@ -31,6 +31,7 @@ using namespace std;
#include "smallut.h"
#include "mh_exec.h"
#include "mh_execm.h"
#include "mh_html.h"
#include "mh_mail.h"
#include "mh_mbox.h"
@@ -68,7 +69,8 @@ static Dijon::Filter *mhFactory(const string &mime)
* We don't support ';' inside a quoted string for now. Can't see a use
* for it
*/
MimeHandlerExec *mhExecFactory(RclConfig *cfg, const string& mtype, string& hs)
MimeHandlerExec *mhExecFactory(RclConfig *cfg, const string& mtype, string& hs,
bool multiple)
{
list<string>semicolist;
stringToTokens(hs, semicolist, ";");
@@ -86,7 +88,9 @@ MimeHandlerExec *mhExecFactory(RclConfig *cfg, const string& mtype, string& hs)
return 0;
}
MimeHandlerExec *h = new MimeHandlerExec(mtype.c_str());
MimeHandlerExec *h = multiple ?
new MimeHandlerExecMultiple(mtype.c_str()) :
new MimeHandlerExec(mtype.c_str());
list<string>::iterator it;
@@ -181,7 +185,14 @@ Dijon::Filter *getMimeHandler(const string &mtype, RclConfig *cfg,
mtype.c_str(), hs.c_str()));
return 0;
}
return mhExecFactory(cfg, mtype, hs);
return mhExecFactory(cfg, mtype, hs, false);
} else if (!stringlowercmp("execm", *it)) {
if (toks.size() < 2) {
LOGERR(("getMimeHandler: bad line for %s: %s\n",
mtype.c_str(), hs.c_str()));
return 0;
}
return mhExecFactory(cfg, mtype, hs, true);
}
}