updated html parser to omega 1.0.1 + moved entity decoder to myhtmlparse to minimize amount of diffs
This commit is contained in:
dockes
parent
0c74bd6e36
commit
750d1c918d
@ -1,11 +1,10 @@
|
||||
/* This file was copied from omega-0.8.5 and modified */
|
||||
/* This file was copied/updated from xapian-omega-1.0.1 and modified */
|
||||
|
||||
/* htmlparse.cc: simple HTML parser for omega indexer
|
||||
*
|
||||
* ----START-LICENCE----
|
||||
* Copyright 1999,2000,2001 BrightStation PLC
|
||||
* Copyright 2001 Ananova Ltd
|
||||
* Copyright 2002 Olly Betts
|
||||
* Copyright 2002,2006 Olly Betts
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License as
|
||||
@ -19,212 +18,99 @@
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
|
||||
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
|
||||
* USA
|
||||
* -----END-LICENCE-----
|
||||
*/
|
||||
|
||||
#ifndef lint
|
||||
static char rcsid[] = "@(#$Id: htmlparse.cpp,v 1.6 2006-01-30 11:15:27 dockes Exp $ ";
|
||||
static char rcsid[] = "@(#$Id: htmlparse.cpp,v 1.7 2007-06-19 10:28:40 dockes Exp $ ";
|
||||
#endif
|
||||
|
||||
//#include <config.h>
|
||||
|
||||
#include <algorithm>
|
||||
#ifndef NO_NAMESPACES
|
||||
using namespace std;
|
||||
//using std::find;
|
||||
//using std::find_if;
|
||||
#endif /* NO_NAMESPACES */
|
||||
using std::find;
|
||||
using std::find_if;
|
||||
#include "htmlparse.h"
|
||||
#include <stdio.h>
|
||||
#include <ctype.h>
|
||||
|
||||
#include "transcode.h"
|
||||
|
||||
map<string, string> HtmlParser::named_ents;
|
||||
|
||||
inline static bool
|
||||
p_alpha(char c)
|
||||
{
|
||||
return isalpha(c);
|
||||
}
|
||||
map<string, unsigned int> HtmlParser::named_ents;
|
||||
|
||||
inline static bool
|
||||
p_notdigit(char c)
|
||||
{
|
||||
return !isdigit(c);
|
||||
return !isdigit(static_cast<unsigned char>(c));
|
||||
}
|
||||
|
||||
inline static bool
|
||||
p_notxdigit(char c)
|
||||
{
|
||||
return !isxdigit(c);
|
||||
return !isxdigit(static_cast<unsigned char>(c));
|
||||
}
|
||||
|
||||
inline static bool
|
||||
p_notalnum(char c)
|
||||
{
|
||||
return !isalnum(c);
|
||||
return !isalnum(static_cast<unsigned char>(c));
|
||||
}
|
||||
|
||||
inline static bool
|
||||
p_notwhitespace(char c)
|
||||
{
|
||||
return !isspace(c);
|
||||
return !isspace(static_cast<unsigned char>(c));
|
||||
}
|
||||
|
||||
inline static bool
|
||||
p_nottag(char c)
|
||||
{
|
||||
return !isalnum(c) && c != '.' && c != '-';
|
||||
return !isalnum(static_cast<unsigned char>(c)) &&
|
||||
c != '.' && c != '-' && c != ':'; // ':' for XML namespaces.
|
||||
}
|
||||
|
||||
inline static bool
|
||||
p_whitespacegt(char c)
|
||||
{
|
||||
return isspace(c) || c == '>';
|
||||
return isspace(static_cast<unsigned char>(c)) || c == '>';
|
||||
}
|
||||
|
||||
inline static bool
|
||||
p_whitespaceeqgt(char c)
|
||||
{
|
||||
return isspace(c) || c == '=' || c == '>';
|
||||
return isspace(static_cast<unsigned char>(c)) || c == '=' || c == '>';
|
||||
}
|
||||
|
||||
/*
|
||||
* The following array was taken from Estraier. Estraier was
|
||||
* written by Mikio Hirabayashi.
|
||||
* Copyright (C) 2003-2004 Mikio Hirabayashi
|
||||
* The version where this comes from
|
||||
* is covered by the GNU licence, as this file.*/
|
||||
static const char *epairs[] = {
|
||||
/* basic symbols */
|
||||
"amp", "&", "lt", "<", "gt", ">", "quot", "\"", "apos", "'",
|
||||
/* ISO-8859-1 */
|
||||
"nbsp", "\xc2\xa0", "iexcl", "\xc2\xa1", "cent", "\xc2\xa2",
|
||||
"pound", "\xc2\xa3", "curren", "\xc2\xa4", "yen", "\xc2\xa5",
|
||||
"brvbar", "\xc2\xa6", "sect", "\xc2\xa7", "uml", "\xc2\xa8",
|
||||
"copy", "\xc2\xa9", "ordf", "\xc2\xaa", "laquo", "\xc2\xab",
|
||||
"not", "\xc2\xac", "shy", "\xc2\xad", "reg", "\xc2\xae",
|
||||
"macr", "\xc2\xaf", "deg", "\xc2\xb0", "plusmn", "\xc2\xb1",
|
||||
"sup2", "\xc2\xb2", "sup3", "\xc2\xb3", "acute", "\xc2\xb4",
|
||||
"micro", "\xc2\xb5", "para", "\xc2\xb6", "middot", "\xc2\xb7",
|
||||
"cedil", "\xc2\xb8", "sup1", "\xc2\xb9", "ordm", "\xc2\xba",
|
||||
"raquo", "\xc2\xbb", "frac14", "\xc2\xbc", "frac12", "\xc2\xbd",
|
||||
"frac34", "\xc2\xbe", "iquest", "\xc2\xbf", "Agrave", "\xc3\x80",
|
||||
"Aacute", "\xc3\x81", "Acirc", "\xc3\x82", "Atilde", "\xc3\x83",
|
||||
"Auml", "\xc3\x84", "Aring", "\xc3\x85", "AElig", "\xc3\x86",
|
||||
"Ccedil", "\xc3\x87", "Egrave", "\xc3\x88", "Eacute", "\xc3\x89",
|
||||
"Ecirc", "\xc3\x8a", "Euml", "\xc3\x8b", "Igrave", "\xc3\x8c",
|
||||
"Iacute", "\xc3\x8d", "Icirc", "\xc3\x8e", "Iuml", "\xc3\x8f",
|
||||
"ETH", "\xc3\x90", "Ntilde", "\xc3\x91", "Ograve", "\xc3\x92",
|
||||
"Oacute", "\xc3\x93", "Ocirc", "\xc3\x94", "Otilde", "\xc3\x95",
|
||||
"Ouml", "\xc3\x96", "times", "\xc3\x97", "Oslash", "\xc3\x98",
|
||||
"Ugrave", "\xc3\x99", "Uacute", "\xc3\x9a", "Ucirc", "\xc3\x9b",
|
||||
"Uuml", "\xc3\x9c", "Yacute", "\xc3\x9d", "THORN", "\xc3\x9e",
|
||||
"szlig", "\xc3\x9f", "agrave", "\xc3\xa0", "aacute", "\xc3\xa1",
|
||||
"acirc", "\xc3\xa2", "atilde", "\xc3\xa3", "auml", "\xc3\xa4",
|
||||
"aring", "\xc3\xa5", "aelig", "\xc3\xa6", "ccedil", "\xc3\xa7",
|
||||
"egrave", "\xc3\xa8", "eacute", "\xc3\xa9", "ecirc", "\xc3\xaa",
|
||||
"euml", "\xc3\xab", "igrave", "\xc3\xac", "iacute", "\xc3\xad",
|
||||
"icirc", "\xc3\xae", "iuml", "\xc3\xaf", "eth", "\xc3\xb0",
|
||||
"ntilde", "\xc3\xb1", "ograve", "\xc3\xb2", "oacute", "\xc3\xb3",
|
||||
"ocirc", "\xc3\xb4", "otilde", "\xc3\xb5", "ouml", "\xc3\xb6",
|
||||
"divide", "\xc3\xb7", "oslash", "\xc3\xb8", "ugrave", "\xc3\xb9",
|
||||
"uacute", "\xc3\xba", "ucirc", "\xc3\xbb", "uuml", "\xc3\xbc",
|
||||
"yacute", "\xc3\xbd", "thorn", "\xc3\xbe", "yuml", "\xc3\xbf",
|
||||
/* ISO-10646 */
|
||||
"fnof", "\xc6\x92", "Alpha", "\xce\x91", "Beta", "\xce\x92",
|
||||
"Gamma", "\xce\x93", "Delta", "\xce\x94", "Epsilon", "\xce\x95",
|
||||
"Zeta", "\xce\x96", "Eta", "\xce\x97", "Theta", "\xce\x98",
|
||||
"Iota", "\xce\x99", "Kappa", "\xce\x9a", "Lambda", "\xce\x9b",
|
||||
"Mu", "\xce\x9c", "Nu", "\xce\x9d", "Xi", "\xce\x9e",
|
||||
"Omicron", "\xce\x9f", "Pi", "\xce\xa0", "Rho", "\xce\xa1",
|
||||
"Sigma", "\xce\xa3", "Tau", "\xce\xa4", "Upsilon", "\xce\xa5",
|
||||
"Phi", "\xce\xa6", "Chi", "\xce\xa7", "Psi", "\xce\xa8",
|
||||
"Omega", "\xce\xa9", "alpha", "\xce\xb1", "beta", "\xce\xb2",
|
||||
"gamma", "\xce\xb3", "delta", "\xce\xb4", "epsilon", "\xce\xb5",
|
||||
"zeta", "\xce\xb6", "eta", "\xce\xb7", "theta", "\xce\xb8",
|
||||
"iota", "\xce\xb9", "kappa", "\xce\xba", "lambda", "\xce\xbb",
|
||||
"mu", "\xce\xbc", "nu", "\xce\xbd", "xi", "\xce\xbe",
|
||||
"omicron", "\xce\xbf", "pi", "\xcf\x80", "rho", "\xcf\x81",
|
||||
"sigmaf", "\xcf\x82", "sigma", "\xcf\x83", "tau", "\xcf\x84",
|
||||
"upsilon", "\xcf\x85", "phi", "\xcf\x86", "chi", "\xcf\x87",
|
||||
"psi", "\xcf\x88", "omega", "\xcf\x89", "thetasym", "\xcf\x91",
|
||||
"upsih", "\xcf\x92", "piv", "\xcf\x96", "bull", "\xe2\x80\xa2",
|
||||
"hellip", "\xe2\x80\xa6", "prime", "\xe2\x80\xb2", "Prime", "\xe2\x80\xb3",
|
||||
"oline", "\xe2\x80\xbe", "frasl", "\xe2\x81\x84", "weierp", "\xe2\x84\x98",
|
||||
"image", "\xe2\x84\x91", "real", "\xe2\x84\x9c", "trade", "\xe2\x84\xa2",
|
||||
"alefsym", "\xe2\x84\xb5", "larr", "\xe2\x86\x90", "uarr", "\xe2\x86\x91",
|
||||
"rarr", "\xe2\x86\x92", "darr", "\xe2\x86\x93", "harr", "\xe2\x86\x94",
|
||||
"crarr", "\xe2\x86\xb5", "lArr", "\xe2\x87\x90", "uArr", "\xe2\x87\x91",
|
||||
"rArr", "\xe2\x87\x92", "dArr", "\xe2\x87\x93", "hArr", "\xe2\x87\x94",
|
||||
"forall", "\xe2\x88\x80", "part", "\xe2\x88\x82", "exist", "\xe2\x88\x83",
|
||||
"empty", "\xe2\x88\x85", "nabla", "\xe2\x88\x87", "isin", "\xe2\x88\x88",
|
||||
"notin", "\xe2\x88\x89", "ni", "\xe2\x88\x8b", "prod", "\xe2\x88\x8f",
|
||||
"sum", "\xe2\x88\x91", "minus", "\xe2\x88\x92", "lowast", "\xe2\x88\x97",
|
||||
"radic", "\xe2\x88\x9a", "prop", "\xe2\x88\x9d", "infin", "\xe2\x88\x9e",
|
||||
"ang", "\xe2\x88\xa0", "and", "\xe2\x88\xa7", "or", "\xe2\x88\xa8",
|
||||
"cap", "\xe2\x88\xa9", "cup", "\xe2\x88\xaa", "int", "\xe2\x88\xab",
|
||||
"there4", "\xe2\x88\xb4", "sim", "\xe2\x88\xbc", "cong", "\xe2\x89\x85",
|
||||
"asymp", "\xe2\x89\x88", "ne", "\xe2\x89\xa0", "equiv", "\xe2\x89\xa1",
|
||||
"le", "\xe2\x89\xa4", "ge", "\xe2\x89\xa5", "sub", "\xe2\x8a\x82",
|
||||
"sup", "\xe2\x8a\x83", "nsub", "\xe2\x8a\x84", "sube", "\xe2\x8a\x86",
|
||||
"supe", "\xe2\x8a\x87", "oplus", "\xe2\x8a\x95", "otimes", "\xe2\x8a\x97",
|
||||
"perp", "\xe2\x8a\xa5", "sdot", "\xe2\x8b\x85", "lceil", "\xe2\x8c\x88",
|
||||
"rceil", "\xe2\x8c\x89", "lfloor", "\xe2\x8c\x8a", "rfloor", "\xe2\x8c\x8b",
|
||||
"lang", "\xe2\x8c\xa9", "rang", "\xe2\x8c\xaa", "loz", "\xe2\x97\x8a",
|
||||
"spades", "\xe2\x99\xa0", "clubs", "\xe2\x99\xa3", "hearts", "\xe2\x99\xa5",
|
||||
"diams", "\xe2\x99\xa6", "OElig", "\xc5\x92", "oelig", "\xc5\x93",
|
||||
"Scaron", "\xc5\xa0", "scaron", "\xc5\xa1", "Yuml", "\xc5\xb8",
|
||||
"circ", "\xcb\x86", "tilde", "\xcb\x9c", "ensp", "\xe2\x80\x82",
|
||||
"emsp", "\xe2\x80\x83", "thinsp", "\xe2\x80\x89", "zwnj", "\xe2\x80\x8c",
|
||||
"zwj", "\xe2\x80\x8d", "lrm", "\xe2\x80\x8e", "rlm", "\xe2\x80\x8f",
|
||||
"ndash", "\xe2\x80\x93", "mdash", "\xe2\x80\x94", "lsquo", "\xe2\x80\x98",
|
||||
"rsquo", "\xe2\x80\x99", "sbquo", "\xe2\x80\x9a", "ldquo", "\xe2\x80\x9c",
|
||||
"rdquo", "\xe2\x80\x9d", "bdquo", "\xe2\x80\x9e", "dagger", "\xe2\x80\xa0",
|
||||
"Dagger", "\xe2\x80\xa1", "permil", "\xe2\x80\xb0", "lsaquo", "\xe2\x80\xb9",
|
||||
"rsaquo", "\xe2\x80\xba", "euro", "\xe2\x82\xac",
|
||||
NULL, NULL
|
||||
};
|
||||
|
||||
HtmlParser::HtmlParser()
|
||||
{
|
||||
// RECOLL: no need to initialize these entities, we use those from
|
||||
// myhtmlparse
|
||||
#if 0
|
||||
static const struct ent { const char *n; unsigned int v; } ents[] = {
|
||||
#include "namedentities.h"
|
||||
{ NULL, 0 }
|
||||
};
|
||||
if (named_ents.empty()) {
|
||||
for (int i = 0;;) {
|
||||
const char *ent;
|
||||
const char *val;
|
||||
ent = epairs[i++];
|
||||
if (ent == 0)
|
||||
break;
|
||||
val = epairs[i++];
|
||||
if (val == 0)
|
||||
break;
|
||||
named_ents[string(ent)] = val;
|
||||
const struct ent *i = ents;
|
||||
while (i->n) {
|
||||
named_ents[string(i->n)] = i->v;
|
||||
++i;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void
|
||||
HtmlParser::decode_entities(string &s)
|
||||
{
|
||||
// This has no meaning whatsoever if the character encoding is unknown,
|
||||
// so don't do it. If charset known, caller has converted text to utf-8,
|
||||
// and this is also how we translate entities
|
||||
// if (charset != "utf-8")
|
||||
// return;
|
||||
|
||||
// Not used for recoll. Kept here to minimize the amount of diffs
|
||||
#if 0
|
||||
// We need a const_iterator version of s.end() - otherwise the
|
||||
// find() and find_if() templates don't work...
|
||||
string::const_iterator amp = s.begin(), s_end = s.end();
|
||||
while ((amp = find(amp, s_end, '&')) != s_end) {
|
||||
unsigned int val = 0;
|
||||
string::const_iterator end, p = amp + 1;
|
||||
string subs;
|
||||
if (p != s_end && *p == '#') {
|
||||
p++;
|
||||
if (p != s_end && tolower(*p) == 'x') {
|
||||
if (p != s_end && (*p == 'x' || *p == 'X')) {
|
||||
// hex
|
||||
p++;
|
||||
end = find_if(p, s_end, p_notxdigit);
|
||||
@ -237,59 +123,95 @@ HtmlParser::decode_entities(string &s)
|
||||
} else {
|
||||
end = find_if(p, s_end, p_notalnum);
|
||||
string code = s.substr(p - s.begin(), end - p);
|
||||
map<string, string>::const_iterator i;
|
||||
map<string, unsigned int>::const_iterator i;
|
||||
i = named_ents.find(code);
|
||||
if (i != named_ents.end())
|
||||
subs = i->second;
|
||||
if (i != named_ents.end()) val = i->second;
|
||||
}
|
||||
|
||||
if (end < s_end && *end == ';')
|
||||
end++;
|
||||
|
||||
if (end < s_end && *end == ';') end++;
|
||||
if (val) {
|
||||
// The code is the code position for a unicode char. We need
|
||||
// to translate it to an utf-8 string.
|
||||
string utf16be;
|
||||
utf16be += char(val / 256);
|
||||
utf16be += char(val % 256);
|
||||
transcode(utf16be, subs, "UTF-16BE", "UTF-8");
|
||||
}
|
||||
|
||||
if (subs.length() > 0) {
|
||||
string::size_type amp_pos = amp - s.begin();
|
||||
s.replace(amp_pos, end - amp, subs);
|
||||
if (val < 0x80) {
|
||||
s.replace(amp_pos, end - amp, 1u, char(val));
|
||||
} else {
|
||||
// Convert unicode value val to UTF-8.
|
||||
char seq[4];
|
||||
unsigned len = Xapian::Unicode::nonascii_to_utf8(val, seq);
|
||||
s.replace(amp_pos, end - amp, seq, len);
|
||||
}
|
||||
s_end = s.end();
|
||||
// We've modified the string, so the iterators are no longer
|
||||
// valid...
|
||||
amp = s.begin() + amp_pos + subs.length();
|
||||
amp = s.begin() + amp_pos + 1;
|
||||
} else {
|
||||
amp = end;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void
|
||||
HtmlParser::parse_html(const string &body)
|
||||
{
|
||||
in_script = false;
|
||||
|
||||
map<string,string> Param;
|
||||
string::const_iterator start = body.begin();
|
||||
|
||||
while (1) {
|
||||
while (true) {
|
||||
// Skip through until we find an HTML tag, a comment, or the end of
|
||||
// document. Ignore isolated occurences of `<' which don't start
|
||||
// a tag or comment.
|
||||
string::const_iterator p = start;
|
||||
|
||||
// Eat text until we find an HTML tag, a comment, or the end
|
||||
// of document. Ignore isolated occurences of `<' which don't
|
||||
// start a tag or comment
|
||||
while (1) {
|
||||
while (true) {
|
||||
p = find(p, body.end(), '<');
|
||||
if (p == body.end()) break;
|
||||
char ch = *(p + 1);
|
||||
// tag, closing tag, comment (or SGML declaration), or PHP
|
||||
if (isalpha(ch) || ch == '/' || ch == '!' || ch == '?') break;
|
||||
unsigned char ch = *(p + 1);
|
||||
|
||||
// Tag, closing tag, or comment (or SGML declaration).
|
||||
if ((!in_script && isalpha(ch)) || ch == '/' || ch == '!') break;
|
||||
if (ch == '?') {
|
||||
// PHP code or XML declaration.
|
||||
// XML declaration is only valid at the start of the first line.
|
||||
// FIXME: need to deal with BOMs...
|
||||
if (p != body.begin() || body.size() < 20) break;
|
||||
|
||||
// XML declaration looks something like this:
|
||||
// <?xml version="1.0" encoding="UTF-8"?>
|
||||
if (p[2] != 'x' || p[3] != 'm' || p[4] != 'l') break;
|
||||
if (strchr(" \t\r\n", p[5]) == NULL) break;
|
||||
|
||||
string::const_iterator decl_end = find(p + 6, body.end(), '?');
|
||||
if (decl_end == body.end()) break;
|
||||
|
||||
// Default charset for XML is UTF-8.
|
||||
charset = "UTF-8";
|
||||
|
||||
string decl(p + 6, decl_end);
|
||||
size_t enc = decl.find("encoding");
|
||||
if (enc == string::npos) break;
|
||||
|
||||
enc = decl.find_first_not_of(" \t\r\n", enc + 8);
|
||||
if (enc == string::npos || enc == decl.size()) break;
|
||||
|
||||
if (decl[enc] != '=') break;
|
||||
|
||||
enc = decl.find_first_not_of(" \t\r\n", enc + 1);
|
||||
if (enc == string::npos || enc == decl.size()) break;
|
||||
|
||||
if (decl[enc] != '"' && decl[enc] != '\'') break;
|
||||
|
||||
char quote = decl[enc++];
|
||||
size_t enc_end = decl.find(quote, enc);
|
||||
|
||||
if (enc != string::npos)
|
||||
charset = decl.substr(enc, enc_end - enc);
|
||||
|
||||
break;
|
||||
}
|
||||
p++;
|
||||
}
|
||||
|
||||
// Process text
|
||||
// Process text up to start of tag.
|
||||
if (p > start || p == body.end()) {
|
||||
string text = body.substr(start - body.begin(), p - start);
|
||||
decode_entities(text);
|
||||
@ -310,19 +232,32 @@ HtmlParser::parse_html(const string &body)
|
||||
if (++start == body.end()) break;
|
||||
// comment or SGML declaration
|
||||
if (*(start - 1) == '-' && *start == '-') {
|
||||
start = find(start + 1, body.end(), '>');
|
||||
// unterminated comment swallows rest of document
|
||||
// (like NS, but unlike MSIE iirc)
|
||||
if (start == body.end()) break;
|
||||
|
||||
p = start;
|
||||
++start;
|
||||
string::const_iterator close = find(start, body.end(), '>');
|
||||
// An unterminated comment swallows rest of document
|
||||
// (like Netscape, but unlike MSIE IIRC)
|
||||
if (close == body.end()) break;
|
||||
|
||||
p = close;
|
||||
// look for -->
|
||||
while (p != body.end() && (*(p - 1) != '-' || *(p - 2) != '-'))
|
||||
p = find(p + 1, body.end(), '>');
|
||||
|
||||
// If we found --> skip to there, otherwise
|
||||
// skip to the first > we found (as Netscape does)
|
||||
if (p != body.end()) start = p;
|
||||
if (p != body.end()) {
|
||||
// Check for htdig's "ignore this bit" comments.
|
||||
if (p - start == 15 && string(start, p - 2) == "htdig_noindex") {
|
||||
string::size_type i;
|
||||
i = body.find("<!--/htdig_noindex-->", p + 1 - body.begin());
|
||||
if (i == string::npos) break;
|
||||
start = body.begin() + i + 21;
|
||||
continue;
|
||||
}
|
||||
// If we found --> skip to there.
|
||||
start = p;
|
||||
} else {
|
||||
// Otherwise skip to the first > we found (as Netscape does).
|
||||
start = close;
|
||||
}
|
||||
} else {
|
||||
// just an SGML declaration, perhaps giving the DTD - ignore it
|
||||
start = find(start - 1, body.end(), '>');
|
||||
@ -354,11 +289,12 @@ HtmlParser::parse_html(const string &body)
|
||||
start = find_if(start, body.end(), p_nottag);
|
||||
string tag = body.substr(p - body.begin(), start - p);
|
||||
// convert tagname to lowercase
|
||||
for (string::iterator i = tag.begin(); i != tag.end(); i++)
|
||||
*i = tolower(*i);
|
||||
for (string::iterator i = tag.begin(); i != tag.end(); ++i)
|
||||
*i = tolower(static_cast<unsigned char>(*i));
|
||||
|
||||
if (closing) {
|
||||
closing_tag(tag);
|
||||
if (in_script && tag == "script") in_script = false;
|
||||
|
||||
/* ignore any bogus parameters on closing tags */
|
||||
p = find(start, body.end(), '>');
|
||||
@ -402,8 +338,8 @@ HtmlParser::parse_html(const string &body)
|
||||
if (name.size()) {
|
||||
// convert parameter name to lowercase
|
||||
string::iterator i;
|
||||
for (i = name.begin(); i != name.end(); i++)
|
||||
*i = tolower(*i);
|
||||
for (i = name.begin(); i != name.end(); ++i)
|
||||
*i = tolower(static_cast<unsigned char>(*i));
|
||||
// in case of multiple entries, use the first
|
||||
// (as Netscape does)
|
||||
if (Param.find(name) == Param.end())
|
||||
@ -414,6 +350,10 @@ HtmlParser::parse_html(const string &body)
|
||||
opening_tag(tag, Param);
|
||||
Param.clear();
|
||||
|
||||
// In <script> tags we ignore opening tags to avoid problems
|
||||
// with "a<b".
|
||||
if (tag == "script") in_script = true;
|
||||
|
||||
if (start != body.end() && *start == '>') ++start;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,10 +1,9 @@
|
||||
/* This file was copied from omega-0.8.5 and modified */
|
||||
/* This file was copied from xapian-omega-1.0.1 and modified */
|
||||
|
||||
/* htmlparse.cc: simple HTML parser for omega indexer
|
||||
/* htmlparse.h: simple HTML parser for omega indexer
|
||||
*
|
||||
* ----START-LICENCE----
|
||||
* Copyright 1999,2000,2001 BrightStation PLC
|
||||
* Copyright 2002 Olly Betts
|
||||
* Copyright 2002,2006 Olly Betts
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License as
|
||||
@ -18,23 +17,25 @@
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
|
||||
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
|
||||
* USA
|
||||
* -----END-LICENCE-----
|
||||
*/
|
||||
|
||||
#ifndef INCLUDED_HTMLPARSE_H
|
||||
#define INCLUDED_HTMLPARSE_H
|
||||
|
||||
#include <string>
|
||||
#include <map>
|
||||
|
||||
#ifndef NO_NAMESPACES
|
||||
using std::string;
|
||||
using std::map;
|
||||
#endif /* NO_NAMESPACES */
|
||||
|
||||
class HtmlParser {
|
||||
protected:
|
||||
void decode_entities(string &s);
|
||||
static map<string, string> named_ents;
|
||||
bool in_script;
|
||||
string charset;
|
||||
static map<string, unsigned int> named_ents;
|
||||
public:
|
||||
virtual void process_text(const string &/*text*/) { }
|
||||
virtual void opening_tag(const string &/*tag*/,
|
||||
@ -45,3 +46,5 @@ class HtmlParser {
|
||||
HtmlParser();
|
||||
virtual ~HtmlParser() { }
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
@ -31,6 +31,209 @@
|
||||
#include "smallut.h"
|
||||
#include "cancelcheck.h"
|
||||
#include "debuglog.h"
|
||||
#include "transcode.h"
|
||||
|
||||
map<string, string> MyHtmlParser::my_named_ents;
|
||||
|
||||
inline static bool
|
||||
p_notdigit(char c)
|
||||
{
|
||||
return !isdigit(static_cast<unsigned char>(c));
|
||||
}
|
||||
|
||||
inline static bool
|
||||
p_notxdigit(char c)
|
||||
{
|
||||
return !isxdigit(static_cast<unsigned char>(c));
|
||||
}
|
||||
|
||||
inline static bool
|
||||
p_notalnum(char c)
|
||||
{
|
||||
return !isalnum(static_cast<unsigned char>(c));
|
||||
}
|
||||
|
||||
/*
|
||||
* The following array was taken from Estraier. Estraier was
|
||||
* written by Mikio Hirabayashi.
|
||||
* Copyright (C) 2003-2004 Mikio Hirabayashi
|
||||
* The version where this comes from
|
||||
* is covered by the GNU licence, as this file.*/
|
||||
static const char *epairs[] = {
|
||||
/* basic symbols */
|
||||
"amp", "&", "lt", "<", "gt", ">", "quot", "\"", "apos", "'",
|
||||
/* ISO-8859-1 */
|
||||
"nbsp", "\xc2\xa0", "iexcl", "\xc2\xa1", "cent", "\xc2\xa2",
|
||||
"pound", "\xc2\xa3", "curren", "\xc2\xa4", "yen", "\xc2\xa5",
|
||||
"brvbar", "\xc2\xa6", "sect", "\xc2\xa7", "uml", "\xc2\xa8",
|
||||
"copy", "\xc2\xa9", "ordf", "\xc2\xaa", "laquo", "\xc2\xab",
|
||||
"not", "\xc2\xac", "shy", "\xc2\xad", "reg", "\xc2\xae",
|
||||
"macr", "\xc2\xaf", "deg", "\xc2\xb0", "plusmn", "\xc2\xb1",
|
||||
"sup2", "\xc2\xb2", "sup3", "\xc2\xb3", "acute", "\xc2\xb4",
|
||||
"micro", "\xc2\xb5", "para", "\xc2\xb6", "middot", "\xc2\xb7",
|
||||
"cedil", "\xc2\xb8", "sup1", "\xc2\xb9", "ordm", "\xc2\xba",
|
||||
"raquo", "\xc2\xbb", "frac14", "\xc2\xbc", "frac12", "\xc2\xbd",
|
||||
"frac34", "\xc2\xbe", "iquest", "\xc2\xbf", "Agrave", "\xc3\x80",
|
||||
"Aacute", "\xc3\x81", "Acirc", "\xc3\x82", "Atilde", "\xc3\x83",
|
||||
"Auml", "\xc3\x84", "Aring", "\xc3\x85", "AElig", "\xc3\x86",
|
||||
"Ccedil", "\xc3\x87", "Egrave", "\xc3\x88", "Eacute", "\xc3\x89",
|
||||
"Ecirc", "\xc3\x8a", "Euml", "\xc3\x8b", "Igrave", "\xc3\x8c",
|
||||
"Iacute", "\xc3\x8d", "Icirc", "\xc3\x8e", "Iuml", "\xc3\x8f",
|
||||
"ETH", "\xc3\x90", "Ntilde", "\xc3\x91", "Ograve", "\xc3\x92",
|
||||
"Oacute", "\xc3\x93", "Ocirc", "\xc3\x94", "Otilde", "\xc3\x95",
|
||||
"Ouml", "\xc3\x96", "times", "\xc3\x97", "Oslash", "\xc3\x98",
|
||||
"Ugrave", "\xc3\x99", "Uacute", "\xc3\x9a", "Ucirc", "\xc3\x9b",
|
||||
"Uuml", "\xc3\x9c", "Yacute", "\xc3\x9d", "THORN", "\xc3\x9e",
|
||||
"szlig", "\xc3\x9f", "agrave", "\xc3\xa0", "aacute", "\xc3\xa1",
|
||||
"acirc", "\xc3\xa2", "atilde", "\xc3\xa3", "auml", "\xc3\xa4",
|
||||
"aring", "\xc3\xa5", "aelig", "\xc3\xa6", "ccedil", "\xc3\xa7",
|
||||
"egrave", "\xc3\xa8", "eacute", "\xc3\xa9", "ecirc", "\xc3\xaa",
|
||||
"euml", "\xc3\xab", "igrave", "\xc3\xac", "iacute", "\xc3\xad",
|
||||
"icirc", "\xc3\xae", "iuml", "\xc3\xaf", "eth", "\xc3\xb0",
|
||||
"ntilde", "\xc3\xb1", "ograve", "\xc3\xb2", "oacute", "\xc3\xb3",
|
||||
"ocirc", "\xc3\xb4", "otilde", "\xc3\xb5", "ouml", "\xc3\xb6",
|
||||
"divide", "\xc3\xb7", "oslash", "\xc3\xb8", "ugrave", "\xc3\xb9",
|
||||
"uacute", "\xc3\xba", "ucirc", "\xc3\xbb", "uuml", "\xc3\xbc",
|
||||
"yacute", "\xc3\xbd", "thorn", "\xc3\xbe", "yuml", "\xc3\xbf",
|
||||
/* ISO-10646 */
|
||||
"fnof", "\xc6\x92", "Alpha", "\xce\x91", "Beta", "\xce\x92",
|
||||
"Gamma", "\xce\x93", "Delta", "\xce\x94", "Epsilon", "\xce\x95",
|
||||
"Zeta", "\xce\x96", "Eta", "\xce\x97", "Theta", "\xce\x98",
|
||||
"Iota", "\xce\x99", "Kappa", "\xce\x9a", "Lambda", "\xce\x9b",
|
||||
"Mu", "\xce\x9c", "Nu", "\xce\x9d", "Xi", "\xce\x9e",
|
||||
"Omicron", "\xce\x9f", "Pi", "\xce\xa0", "Rho", "\xce\xa1",
|
||||
"Sigma", "\xce\xa3", "Tau", "\xce\xa4", "Upsilon", "\xce\xa5",
|
||||
"Phi", "\xce\xa6", "Chi", "\xce\xa7", "Psi", "\xce\xa8",
|
||||
"Omega", "\xce\xa9", "alpha", "\xce\xb1", "beta", "\xce\xb2",
|
||||
"gamma", "\xce\xb3", "delta", "\xce\xb4", "epsilon", "\xce\xb5",
|
||||
"zeta", "\xce\xb6", "eta", "\xce\xb7", "theta", "\xce\xb8",
|
||||
"iota", "\xce\xb9", "kappa", "\xce\xba", "lambda", "\xce\xbb",
|
||||
"mu", "\xce\xbc", "nu", "\xce\xbd", "xi", "\xce\xbe",
|
||||
"omicron", "\xce\xbf", "pi", "\xcf\x80", "rho", "\xcf\x81",
|
||||
"sigmaf", "\xcf\x82", "sigma", "\xcf\x83", "tau", "\xcf\x84",
|
||||
"upsilon", "\xcf\x85", "phi", "\xcf\x86", "chi", "\xcf\x87",
|
||||
"psi", "\xcf\x88", "omega", "\xcf\x89", "thetasym", "\xcf\x91",
|
||||
"upsih", "\xcf\x92", "piv", "\xcf\x96", "bull", "\xe2\x80\xa2",
|
||||
"hellip", "\xe2\x80\xa6", "prime", "\xe2\x80\xb2", "Prime", "\xe2\x80\xb3",
|
||||
"oline", "\xe2\x80\xbe", "frasl", "\xe2\x81\x84", "weierp", "\xe2\x84\x98",
|
||||
"image", "\xe2\x84\x91", "real", "\xe2\x84\x9c", "trade", "\xe2\x84\xa2",
|
||||
"alefsym", "\xe2\x84\xb5", "larr", "\xe2\x86\x90", "uarr", "\xe2\x86\x91",
|
||||
"rarr", "\xe2\x86\x92", "darr", "\xe2\x86\x93", "harr", "\xe2\x86\x94",
|
||||
"crarr", "\xe2\x86\xb5", "lArr", "\xe2\x87\x90", "uArr", "\xe2\x87\x91",
|
||||
"rArr", "\xe2\x87\x92", "dArr", "\xe2\x87\x93", "hArr", "\xe2\x87\x94",
|
||||
"forall", "\xe2\x88\x80", "part", "\xe2\x88\x82", "exist", "\xe2\x88\x83",
|
||||
"empty", "\xe2\x88\x85", "nabla", "\xe2\x88\x87", "isin", "\xe2\x88\x88",
|
||||
"notin", "\xe2\x88\x89", "ni", "\xe2\x88\x8b", "prod", "\xe2\x88\x8f",
|
||||
"sum", "\xe2\x88\x91", "minus", "\xe2\x88\x92", "lowast", "\xe2\x88\x97",
|
||||
"radic", "\xe2\x88\x9a", "prop", "\xe2\x88\x9d", "infin", "\xe2\x88\x9e",
|
||||
"ang", "\xe2\x88\xa0", "and", "\xe2\x88\xa7", "or", "\xe2\x88\xa8",
|
||||
"cap", "\xe2\x88\xa9", "cup", "\xe2\x88\xaa", "int", "\xe2\x88\xab",
|
||||
"there4", "\xe2\x88\xb4", "sim", "\xe2\x88\xbc", "cong", "\xe2\x89\x85",
|
||||
"asymp", "\xe2\x89\x88", "ne", "\xe2\x89\xa0", "equiv", "\xe2\x89\xa1",
|
||||
"le", "\xe2\x89\xa4", "ge", "\xe2\x89\xa5", "sub", "\xe2\x8a\x82",
|
||||
"sup", "\xe2\x8a\x83", "nsub", "\xe2\x8a\x84", "sube", "\xe2\x8a\x86",
|
||||
"supe", "\xe2\x8a\x87", "oplus", "\xe2\x8a\x95", "otimes", "\xe2\x8a\x97",
|
||||
"perp", "\xe2\x8a\xa5", "sdot", "\xe2\x8b\x85", "lceil", "\xe2\x8c\x88",
|
||||
"rceil", "\xe2\x8c\x89", "lfloor", "\xe2\x8c\x8a", "rfloor", "\xe2\x8c\x8b",
|
||||
"lang", "\xe2\x8c\xa9", "rang", "\xe2\x8c\xaa", "loz", "\xe2\x97\x8a",
|
||||
"spades", "\xe2\x99\xa0", "clubs", "\xe2\x99\xa3", "hearts", "\xe2\x99\xa5",
|
||||
"diams", "\xe2\x99\xa6", "OElig", "\xc5\x92", "oelig", "\xc5\x93",
|
||||
"Scaron", "\xc5\xa0", "scaron", "\xc5\xa1", "Yuml", "\xc5\xb8",
|
||||
"circ", "\xcb\x86", "tilde", "\xcb\x9c", "ensp", "\xe2\x80\x82",
|
||||
"emsp", "\xe2\x80\x83", "thinsp", "\xe2\x80\x89", "zwnj", "\xe2\x80\x8c",
|
||||
"zwj", "\xe2\x80\x8d", "lrm", "\xe2\x80\x8e", "rlm", "\xe2\x80\x8f",
|
||||
"ndash", "\xe2\x80\x93", "mdash", "\xe2\x80\x94", "lsquo", "\xe2\x80\x98",
|
||||
"rsquo", "\xe2\x80\x99", "sbquo", "\xe2\x80\x9a", "ldquo", "\xe2\x80\x9c",
|
||||
"rdquo", "\xe2\x80\x9d", "bdquo", "\xe2\x80\x9e", "dagger", "\xe2\x80\xa0",
|
||||
"Dagger", "\xe2\x80\xa1", "permil", "\xe2\x80\xb0", "lsaquo", "\xe2\x80\xb9",
|
||||
"rsaquo", "\xe2\x80\xba", "euro", "\xe2\x82\xac",
|
||||
NULL, NULL
|
||||
};
|
||||
|
||||
MyHtmlParser::MyHtmlParser()
|
||||
: in_script_tag(false),
|
||||
in_style_tag(false),
|
||||
in_body_tag(false),
|
||||
in_pre_tag(false),
|
||||
pending_space(false),
|
||||
indexing_allowed(true)
|
||||
{
|
||||
if (my_named_ents.empty()) {
|
||||
for (int i = 0;;) {
|
||||
const char *ent;
|
||||
const char *val;
|
||||
ent = epairs[i++];
|
||||
if (ent == 0)
|
||||
break;
|
||||
val = epairs[i++];
|
||||
if (val == 0)
|
||||
break;
|
||||
my_named_ents[string(ent)] = val;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void MyHtmlParser::decode_entities(string &s)
|
||||
{
|
||||
LOGDEB(("MyHtmlParser::decode_entities\n"));
|
||||
// This has no meaning whatsoever if the character encoding is unknown,
|
||||
// so don't do it. If charset known, caller has converted text to utf-8,
|
||||
// and this is also how we translate entities
|
||||
// if (charset != "utf-8")
|
||||
// return;
|
||||
|
||||
// We need a const_iterator version of s.end() - otherwise the
|
||||
// find() and find_if() templates don't work...
|
||||
string::const_iterator amp = s.begin(), s_end = s.end();
|
||||
while ((amp = find(amp, s_end, '&')) != s_end) {
|
||||
unsigned int val = 0;
|
||||
string::const_iterator end, p = amp + 1;
|
||||
string subs;
|
||||
if (p != s_end && *p == '#') {
|
||||
p++;
|
||||
if (p != s_end && (*p == 'x' || *p == 'X')) {
|
||||
// hex
|
||||
p++;
|
||||
end = find_if(p, s_end, p_notxdigit);
|
||||
sscanf(s.substr(p - s.begin(), end - p).c_str(), "%x", &val);
|
||||
} else {
|
||||
// number
|
||||
end = find_if(p, s_end, p_notdigit);
|
||||
val = atoi(s.substr(p - s.begin(), end - p).c_str());
|
||||
}
|
||||
} else {
|
||||
end = find_if(p, s_end, p_notalnum);
|
||||
string code = s.substr(p - s.begin(), end - p);
|
||||
map<string, string>::const_iterator i;
|
||||
i = my_named_ents.find(code);
|
||||
if (i != my_named_ents.end())
|
||||
subs = i->second;
|
||||
}
|
||||
|
||||
if (end < s_end && *end == ';')
|
||||
end++;
|
||||
|
||||
if (val) {
|
||||
// The code is the code position for a unicode char. We need
|
||||
// to translate it to an utf-8 string.
|
||||
string utf16be;
|
||||
utf16be += char(val / 256);
|
||||
utf16be += char(val % 256);
|
||||
transcode(utf16be, subs, "UTF-16BE", "UTF-8");
|
||||
}
|
||||
|
||||
if (subs.length() > 0) {
|
||||
string::size_type amp_pos = amp - s.begin();
|
||||
s.replace(amp_pos, end - amp, subs);
|
||||
s_end = s.end();
|
||||
// We've modified the string, so the iterators are no longer
|
||||
// valid...
|
||||
amp = s.begin() + amp_pos + subs.length();
|
||||
} else {
|
||||
amp = end;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compress whitespace and suppress newlines
|
||||
// Note that we independantly add some newlines to the output text in the
|
||||
|
||||
@ -40,20 +40,17 @@ class MyHtmlParser : public HtmlParser {
|
||||
bool in_pre_tag;
|
||||
bool pending_space;
|
||||
map<string,string> meta;
|
||||
static map<string, string> my_named_ents;
|
||||
string dump, dmtime;
|
||||
string ocharset; // This is the charset our user thinks the doc was
|
||||
string charset; // This is the charset it was supposedly converted to
|
||||
// charset is declared by HtmlParser
|
||||
//string charset; // This is the charset it was supposedly converted to
|
||||
string doccharset; // Set this to value of charset parameter in header
|
||||
bool indexing_allowed;
|
||||
void process_text(const string &text);
|
||||
void opening_tag(const string &tag, const map<string,string> &p);
|
||||
void closing_tag(const string &tag);
|
||||
void do_eof();
|
||||
MyHtmlParser() :
|
||||
in_script_tag(false),
|
||||
in_style_tag(false),
|
||||
in_body_tag(false),
|
||||
in_pre_tag(false),
|
||||
pending_space(false),
|
||||
indexing_allowed(true) { }
|
||||
void decode_entities(string &s);
|
||||
MyHtmlParser();
|
||||
};
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user