tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

dont insert space in cjk abstracts

Browse Source
This commit is contained in:
dockes
2008-12-12 11:53:45 +00:00
parent d318f506bf
commit 64ef8d0b81
3 changed files with 22 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/textsplit.cpp
+6 -1
View File
@@ -1,5 +1,5 @@
#ifndef lint
static char rcsid[] = "@(#$Id: textsplit.cpp,v 1.37 2008-12-05 11:09:31 dockes Exp $ (C) 2004 J.F.Dockes";
static char rcsid[] = "@(#$Id: textsplit.cpp,v 1.38 2008-12-12 11:53:45 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
@@ -149,6 +149,11 @@ static inline int whatcc(unsigned int c)
|| ((p) >= 0x20000 && (p) <= 0x2A6DF) \
|| ((p) >= 0x2F800 && (p) <= 0x2FA1F))
bool TextSplit::isCJK(int c)
{
return UNICODE_IS_CJK(c);
}
bool TextSplit::o_processCJK = true;
unsigned int TextSplit::o_CJKNgramLen = 2;
src/common/textsplit.h
+5 -2
View File
@@ -16,7 +16,7 @@
*/
#ifndef _TEXTSPLIT_H_INCLUDED_
#define _TEXTSPLIT_H_INCLUDED_
/* @(#$Id: textsplit.h,v 1.21 2008-12-05 11:09:31 dockes Exp $ (C) 2004 J.F.Dockes */
/* @(#$Id: textsplit.h,v 1.22 2008-12-12 11:53:45 dockes Exp $ (C) 2004 J.F.Dockes */
#include <string>
#include <list>
@@ -96,7 +96,10 @@ public:
* handles all Unicode whitespace, but needs correct utf-8 input
*/
static bool stringToStrings(const string &s, list<string> &tokens);
/** Is char CJK ? */
static bool isCJK(int c);
private:
Flags m_flags;
TextSplitCB *m_cb;