tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added external filters and pdf handling

Browse Source
This commit is contained in:
dockes
2005-02-01 17:20:06 +00:00
parent cc512e2ec0
commit d0aaf92220
14 changed files with 373 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/internfile/mh_html.cpp
+13 -6
View File
@@ -32,13 +32,14 @@
#include "mimeparse.h"
#include "myhtmlparse.h"
#include "indextext.h"
#include "html.h"
#include <iostream>
using namespace std;
bool textHtmlToDoc(RclConfig *conf, const string &fn,
const string &mtype, Rcl::Doc &docout)
bool MimeHandlerHtml::worker(RclConfig *conf, const string &fn,
const string &mtype, Rcl::Doc &docout)
{
LOGDEB(("textHtmlToDoc: %s\n", fn.c_str()));
string otext;
@@ -46,7 +47,13 @@ bool textHtmlToDoc(RclConfig *conf, const string &fn,
LOGINFO(("textHtmlToDoc: cant read: %s\n", fn.c_str()));
return false;
}
return worker1(conf, fn, otext, mtype, docout);
}
bool MimeHandlerHtml::worker1(RclConfig *conf, const string &fn,
const string& htext,
const string &mtype, Rcl::Doc &docout)
{
// Character set handling:
// - We first try to convert from the default configured charset
@@ -57,7 +64,7 @@ bool textHtmlToDoc(RclConfig *conf, const string &fn,
// instead of the configuration one.
string charset;
if (conf->guesscharset) {
charset = csguess(otext, conf->defcharset);
charset = csguess(htext, conf->defcharset);
} else
charset = conf->defcharset;
@@ -69,10 +76,10 @@ bool textHtmlToDoc(RclConfig *conf, const string &fn,
MyHtmlParser p;
// Try transcoding. If it fails, use original text.
if (!transcode(otext, transcoded, charset, "UTF-8")) {
if (!transcode(htext, transcoded, charset, "UTF-8")) {
LOGERR(("textHtmlToDoc: transcode failed from cs '%s' to UTF-8\n",
charset.c_str()));
transcoded = otext;
transcoded = htext;
// We don't know the charset, at all
p.ocharset = p.charset = charset = "";
} else {
src/internfile/mh_html.h
+14
View File
@@ -0,0 +1,14 @@
#ifndef _HTML_H_INCLUDED_
#define _HTML_H_INCLUDED_
/* @(#$Id: mh_html.h,v 1.1 2005-02-01 17:20:05 dockes Exp $ (C) 2004 J.F.Dockes */
#include "mimehandler.h"
class MimeHandlerHtml : public MimeHandler {
public:
virtual bool worker(RclConfig *conf, const string &fn,
const string &mtype, Rcl::Doc &docout);
virtual bool worker1(RclConfig *conf, const string &fn,
const string& htext,
const string &mtype, Rcl::Doc &docout);
};
#endif /* _HTML_H_INCLUDED_ */
src/internfile/mimehandler.cpp
+64 -30
View File
@@ -1,5 +1,5 @@
#ifndef lint
static char rcsid[] = "@(#$Id: mimehandler.cpp,v 1.4 2005-01-29 15:41:11 dockes Exp $ (C) 2004 J.F.Dockes";
static char rcsid[] = "@(#$Id: mimehandler.cpp,v 1.5 2005-02-01 17:20:05 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
#include <iostream>
@@ -12,9 +12,19 @@ using namespace std;
#include "transcode.h"
#include "debuglog.h"
#include "smallut.h"
#include "html.h"
#include "execmd.h"
bool textPlainToDoc(RclConfig *conf, const string &fn,
const string &mtype, Rcl::Doc &docout)
class MimeHandlerText : public MimeHandler {
public:
bool worker(RclConfig *conf, const string &fn,
const string &mtype, Rcl::Doc &docout);
};
// Process a plain text file
bool MimeHandlerText::worker(RclConfig *conf, const string &fn,
const string &mtype, Rcl::Doc &docout)
{
string otext;
if (!file_to_string(fn, otext))
@@ -45,25 +55,51 @@ bool textPlainToDoc(RclConfig *conf, const string &fn,
return true;
}
// Map of mime types to internal interner functions. This could just as well
// be an if else if suite inside getMimeHandler(), but this is prettier ?
static map<string, MimeHandlerFunc> ihandlers;
// Static object to get the map to be initialized at program start.
class IHandler_Init {
class MimeHandlerExec : public MimeHandler {
public:
IHandler_Init() {
ihandlers["text/plain"] = textPlainToDoc;
ihandlers["text/html"] = textHtmlToDoc;
// Add new associations here when needed
}
};
static IHandler_Init ihandleriniter;
list<string> params;
virtual ~MimeHandlerExec() {}
virtual bool worker(RclConfig *conf, const string &fn,
const string &mtype, Rcl::Doc &docout);
};
// Execute an external program to translate a file from its native format
// to html. Then call the html parser to do the actual indexing
bool MimeHandlerExec::worker(RclConfig *conf, const string &fn,
const string &mtype, Rcl::Doc &docout)
{
string cmd = params.front();
list<string>::iterator it = params.begin();
list<string>myparams(++it, params.end());
myparams.push_back(fn);
string html;
ExecCmd exec;
int status = exec.doexec(cmd, myparams, 0, &html);
if (status) {
LOGDEB(("MimeHandlerExec: command status 0x%x: %s\n",
status, cmd.c_str()));
return false;
}
MimeHandlerHtml hh;
return hh.worker1(conf, fn, html, mtype, docout);
}
static MimeHandler *mhfact(const string &mime)
{
if (!stringlowercmp("text/plain", mime))
return new MimeHandlerText;
else if (!stringlowercmp("text/html", mime))
return new MimeHandlerHtml;
return 0;
}
/**
* Return handler function for given mime type
*/
MimeHandlerFunc getMimeHandler(const std::string &mtype, ConfTree *mhandlers)
MimeHandler *getMimeHandler(const std::string &mtype, ConfTree *mhandlers)
{
// Return handler definition for mime type
string hs;
@@ -82,25 +118,23 @@ MimeHandlerFunc getMimeHandler(const std::string &mtype, ConfTree *mhandlers)
// Retrieve handler function according to type
if (!stringlowercmp("internal", toks[0])) {
map<string, MimeHandlerFunc>::const_iterator it =
ihandlers.find(mtype);
if (it == ihandlers.end()) {
LOGERR(("getMimeHandler: internal handler not found for %s\n",
mtype.c_str()));
return 0;
}
return it->second;
return mhfact(mtype);
} else if (!stringlowercmp("dll", toks[0])) {
if (toks.size() != 2)
return 0;
return 0;
} else if (!stringlowercmp("exec", toks[0])) {
if (toks.size() != 2)
if (toks.size() < 2) {
LOGERR(("getMimeHandler: bad line for %s: %s\n", mtype.c_str(),
hs.c_str()));
return 0;
return 0;
} else {
return 0;
}
MimeHandlerExec *h = new MimeHandlerExec;
vector<string>::const_iterator it1 = toks.begin();
it1++;
for (;it1 != toks.end();it1++)
h->params.push_back(*it1);
return h;
}
return 0;
}
/**
src/internfile/mimehandler.h
+15 -10
View File
@@ -1,21 +1,29 @@
#ifndef _MIMEHANDLER_H_INCLUDED_
#define _MIMEHANDLER_H_INCLUDED_
/* @(#$Id: mimehandler.h,v 1.3 2005-01-29 15:41:11 dockes Exp $ (C) 2004 J.F.Dockes */
/* @(#$Id: mimehandler.h,v 1.4 2005-02-01 17:20:05 dockes Exp $ (C) 2004 J.F.Dockes */
#include <string>
#include "rclconfig.h"
#include "rcldb.h"
/* Definition for document interner functions */
typedef bool (*MimeHandlerFunc)(RclConfig *, const std::string &,
const std::string &, Rcl::Doc&);
/**
* Return indexing handler function for given mime type
* Document interner class. We sometimes have data to pass to an interner
*/
extern MimeHandlerFunc getMimeHandler(const std::string &mtype,
ConfTree *mhandlers);
class MimeHandler {
public:
virtual ~MimeHandler() {}
virtual bool worker(RclConfig *, const std::string &filename,
const std::string &mimetype, Rcl::Doc& outdoc) = 0;
};
/**
* Return indexing handler class for given mime type
* returned pointer should be deleted by caller
*/
extern MimeHandler *getMimeHandler(const std::string &mtype,
ConfTree *mhandlers);
/**
* Return external viewer exec string for given mime type
@@ -23,7 +31,4 @@ extern MimeHandlerFunc getMimeHandler(const std::string &mtype,
extern string getMimeViewer(const std::string &mtype,
ConfTree *mhandlers);
extern bool textHtmlToDoc(RclConfig *conf, const string &fn,
const string &mtype, Rcl::Doc &docout);
#endif /* _MIMEHANDLER_H_INCLUDED_ */