tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

sort of indexes html

Browse Source
This commit is contained in:
dockes
2005-01-26 13:03:02 +00:00
parent 0b18276947
commit b9bb21f118
4 changed files with 23 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/internfile/mh_html.cpp
+17 -11
View File
@@ -37,7 +37,8 @@ class MyHtmlParser : public HtmlParser {
bool in_script_tag;
bool in_style_tag;
string title, sample, keywords, dump;
string charset; // This is the charset our user thinks the doc is in
string ocharset; // This is the charset our user thinks the doc was
string charset; // This is the charset it was supposedly converted to
string doccharset; // Set this to value of charset parameter in header
bool indexing_allowed;
void process_text(const string &text);
@@ -125,12 +126,18 @@ MyHtmlParser::opening_tag(const string &tag, const map<string,string> &p)
map<string, string>::const_iterator k;
if ((k = p.params.find("charset")) != p.params.end()) {
doccharset = k->second;
if (doccharset != charset)
if (doccharset != ocharset) {
LOGDEB1(("Doc specified charset '%s' "
"differs from announced '%s'\n",
doccharset.c_str(), ocharset.c_str()));
throw true;
}
}
}
}
}
} else if (tag == "p" || tag == "br") {
dump += "\n";
} else if (tag == "script") {
in_script_tag = true;
} else if (tag == "style") {
@@ -179,14 +186,11 @@ bool textHtmlToDoc(RclConfig *conf, const string &fn,
} else
charset = conf->defcharset;
LOGDEB(("textHtmlToDoc: charset before parsing: %s\n",
charset.c_str()));
LOGDEB(("textHtmlToDoc: charset before parsing: %s\n", charset.c_str()));
MyHtmlParser pres;
for (int pass = 0; pass < 2; pass++) {
string transcoded;
LOGDEB(("textHtmlToDoc: transcode from %s to %s\n",
charset.c_str(), "UTF-8"));
MyHtmlParser p;
// Try transcoding. If it fails, use original text.
@@ -195,10 +199,11 @@ bool textHtmlToDoc(RclConfig *conf, const string &fn,
charset.c_str()));
transcoded = otext;
// We don't know the charset, at all
p.charset = charset = "";
p.ocharset = p.charset = charset = "";
} else {
// charset has the putative source charset, transcoded is now
// ocharset has the putative source charset, transcoded is now
// in utf-8
p.ocharset = charset;
p.charset = "utf-8";
}
@@ -206,10 +211,10 @@ bool textHtmlToDoc(RclConfig *conf, const string &fn,
p.parse_html(transcoded);
} catch (bool) {
pres = p;
if (!pres.doccharset.empty() && pres.doccharset != charset) {
if (!pres.doccharset.empty() &&
pres.doccharset != pres.ocharset) {
LOGDEB(("textHtmlToDoc: charset '%s' doc charset '%s',"
"reparse\n", charset.c_str(),
pres.doccharset.c_str()));
"reparse\n", charset.c_str(),pres.doccharset.c_str()));
charset = pres.doccharset;
} else
break;
@@ -219,6 +224,7 @@ bool textHtmlToDoc(RclConfig *conf, const string &fn,
Rcl::Doc out;
out.origcharset = charset;
out.text = pres.dump;
// LOGDEB(("textHtmlToDoc: dump : %s\n", pres.dump.c_str()));
out.title = pres.title;
out.keywords = pres.keywords;
out.abstract = pres.sample;
src/internfile/mimehandler.cpp
+1 -2
View File
@@ -1,5 +1,5 @@
#ifndef lint
static char rcsid[] = "@(#$Id: mimehandler.cpp,v 1.2 2005-01-26 11:47:27 dockes Exp $ (C) 2004 J.F.Dockes";
static char rcsid[] = "@(#$Id: mimehandler.cpp,v 1.3 2005-01-26 13:03:02 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
#include <iostream>
@@ -79,7 +79,6 @@ MimeHandlerFunc getMimeHandler(const std::string &mtype, ConfTree *mhandlers)
// Retrieve handler function according to type
if (!strcasecmp(toks[0].c_str(), "internal")) {
cerr << "Internal Handler" << endl;
map<string, MimeHandlerFunc>::const_iterator it =
ihandlers.find(mtype);
if (it == ihandlers.end()) {