textsplit: create isNGRAMMED() method to replace isCJK() and let the latter actually return what it says

src/common/textsplit.cpp

@@ -319,8 +319,7 @@ static inline int whatcc(unsigned int c, char *asciirep = nullptr)
 
 bool TextSplit::isCJK(int c)
 {
-    return UNICODE_IS_CJK(c) && !UNICODE_IS_KATAKANA(c) &&
-        !UNICODE_IS_HANGUL(c);
+    return UNICODE_IS_CJK(c);
 }
 bool TextSplit::isKATAKANA(int c)
 {
@@ -330,6 +329,11 @@ bool TextSplit::isHANGUL(int c)
 {
     return UNICODE_IS_HANGUL(c);
 }
+bool TextSplit::isNGRAMMED(int c)
+{
+    return UNICODE_IS_CJK(c) && !UNICODE_IS_KATAKANA(c) &&
+        !UNICODE_IS_HANGUL(c);
+}
 
 
 // This is used to detect katakana/other transitions, which must

src/common/textsplit.h

@@ -89,11 +89,13 @@ public:
     static bool stringToStrings(const std::string &s,
                                 std::vector<std::string> &tokens);
 
-    /** Is char CJK ? (excluding Katakana) */
+    /** Is char CJK ? */
     static bool isCJK(int c);
     static bool isKATAKANA(int c);
     static bool isHANGUL(int c);
-
+    /* Not split in words */
+    static bool isNGRAMMED(int c);
+    
     /** Statistics about word length (average and dispersion) can
      * detect bad data like undecoded base64 or other mis-identified
      * pieces of data taken as text. In practise, this keeps some junk out