tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

wen

Browse Source
This commit is contained in:
Jean-Francois Dockes
2017-04-18 14:39:12 +02:00
parent 9d8ce3df62
commit 9661a4431e
12 changed files with 147 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/textsplit.cpp
+12 -1
View File
@@ -197,7 +197,9 @@ static inline int whatcc(unsigned int c)
#define UNICODE_IS_CJK(p) \
((p) > 127 && \
(((p) >= 0x2E80 && (p) <= 0x2EFF) || \
((p) >= 0x3000 && (p) <= 0x9FFF) || \
((p) >= 0x3000 && (p) <= 0x309F) || \
((p) >= 0x3100 && (p) <= 0x31EF) || \
((p) >= 0x3200 && (p) <= 0x9FFF) || \
((p) >= 0xA700 && (p) <= 0xA71F) || \
((p) >= 0xAC00 && (p) <= 0xD7AF) || \
((p) >= 0xF900 && (p) <= 0xFAFF) || \
@@ -206,10 +208,19 @@ static inline int whatcc(unsigned int c)
((p) >= 0x20000 && (p) <= 0x2A6DF) || \
((p) >= 0x2F800 && (p) <= 0x2FA1F)))
#define UNICODE_IS_KATAKANA(p) \
((p) > 127 && \
(((p) >= 0x30A0 && (p) <= 0x30FF) || \
((p) >= 0x31F0 && (p) <= 0x31FF)))
bool TextSplit::isCJK(int c)
{
return UNICODE_IS_CJK(c);
}
bool TextSplit::isKATAKANA(int c)
{
return UNICODE_IS_KATAKANA(c);
}
bool TextSplit::o_processCJK = true;
unsigned int TextSplit::o_CJKNgramLen = 2;
src/common/textsplit.h
+5 -4
View File
@@ -92,8 +92,7 @@ public:
/** Called when we encounter formfeed \f 0x0c. Override to use the event.
* Mostly or exclusively used with pdftoxx output. Other filters mostly
* just don't know about pages. */
virtual void newpage(int /*pos*/)
{
virtual void newpage(int /*pos*/) {
}
// Static utility functions:
@@ -111,10 +110,12 @@ public:
* non-utf-8 input (iso-8859 config files work ok). This hopefully
* handles all Unicode whitespace, but needs correct utf-8 input
*/
static bool stringToStrings(const std::string &s, std::vector<std::string> &tokens);
static bool stringToStrings(const std::string &s,
std::vector<std::string> &tokens);
/** Is char CJK ? */
/** Is char CJK ? (excluding Katakana) */
static bool isCJK(int c);
static bool isKATAKANA(int c);
/** Statistics about word length (average and dispersion) can
* detect bad data like undecoded base64 or other mis-identified