tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

HTML: do not concatenate text found before body tag with the title. Fixes issue #125

Browse Source
This commit is contained in:
Jean-Francois Dockes 2013-01-12 14:06:40 +01:00
parent 8dc1177fd4
commit f897f087aa
3 changed files with 23 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/internfile/myhtmlparse.cpp
View File

@ -177,6 +177,7 @@ MyHtmlParser::MyHtmlParser()
: in_script_tag(false),
in_style_tag(false),
in_pre_tag(false),
in_title_tag(false),
pending_space(false),
indexing_allowed(true)
{
@ -256,12 +257,20 @@ void MyHtmlParser::decode_entities(string &s)
void
MyHtmlParser::process_text(const string &text)
{
LOGDEB2(("process_text: pending_space %d txt [%s]\n", pending_space,
text.c_str()));
LOGDEB2(("process_text: title %d script %d style %d pre %d "
"pending_space %d txt [%s]\n",
in_title_tag,
in_script_tag,
in_style_tag,
in_pre_tag,
pending_space,
text.c_str()));
CancelCheck::instance().checkCancel();
if (!in_script_tag && !in_style_tag) {
if (!in_pre_tag) {
if (in_title_tag) {
titledump += text;
} else if (!in_pre_tag) {
string::size_type b = 0;
bool only_space = true;
while ((b = text.find_first_not_of(WHITESPACE, b)) != string::npos) {
@ -461,7 +470,11 @@ MyHtmlParser::opening_tag(const string &tag)
break;
case 't':
if (tag == "table" || tag == "td" || tag == "textarea" ||
tag == "th") pending_space = true;
tag == "th") {
pending_space = true;
} else if (tag == "title") {
in_title_tag = true;
}
break;
case 'u':
if (tag == "ul") pending_space = true;
@ -542,9 +555,10 @@ MyHtmlParser::closing_tag(const string &tag)
break;
case 't':
if (tag == "title") {
in_title_tag = false;
if (meta.find("title") == meta.end()|| meta["title"].empty()) {
meta["title"] = dump;
dump.clear();
meta["title"] = titledump;
titledump.clear();
}
break;
}

3
src/internfile/myhtmlparse.h
View File

@ -37,9 +37,10 @@ class MyHtmlParser : public HtmlParser {
bool in_script_tag;
bool in_style_tag;
bool in_pre_tag;
bool in_title_tag;
bool pending_space;
map<string,string> meta;
string dump, dmtime;
string dump, dmtime, titledump;
// This is the charset our caller thinks the doc used (initially
// comes from the environment/configuration, used as source for
// conversion to utf-8)

2
tests/html/html.txt
View File

@ -6,7 +6,7 @@ text/html [file:///home/dockes/projets/fulltext/testrecoll/html/qtextedit.html]
text/html [file:///home/dockes/projets/fulltext/testrecoll/html/mysqlmanualSmall.html] [MySQL 3.23, 4.0, 4.1 Reference Manual] 1480354 bytes
text/html [file:///home/dockes/projets/fulltext/testrecoll/html/mysqlmanual.html] [MySQL 3.23, 4.0, 4.1 Reference Manual] 8136414 bytes
1 results
text/html [file:///home/dockes/projets/fulltext/testrecoll/html/badhtml.html] [bla bla "bad quotes] 236 bytes
text/html [file:///home/dockes/projets/fulltext/testrecoll/html/badhtml.html] ["bad quotes] 236 bytes
1 results
text/html [file:///home/dockes/projets/fulltext/testrecoll/html/htmlfield.html] [htmlfield.html] 137 bytes
testfield = testfieldvalue