korean textsplit with extern help from konlpy, first step

src/common/textsplit.cpp

@@ -246,6 +246,7 @@ static inline int whatcc(unsigned int c, char *asciirep = nullptr)
 #define UNICODE_IS_KATAKANA(p) false
 #endif
 
+#define HANGUL_AS_WORDS
 #ifdef HANGUL_AS_WORDS
 #define UNICODE_IS_HANGUL(p) (                 \
         ((p) >= 0x1100 && (p) <= 0x11FF) ||    \
@@ -323,7 +324,8 @@ void TextSplit::staticConfInit(RclConfig *config)
             charclasses[int('\\')] = SPACE;
         }
     }
-}    
+    koStaticConfInit(config);
+}
 
 // Final term checkpoint: do some checking (the kind which is simpler
 // to do here than in the main loop), then send term to our client.
@@ -632,20 +634,28 @@ bool TextSplit::text_to_words(const string &in)
             csc = CSC_OTHER;
         }
 
-        if (o_processCJK && csc == CSC_CJK) {
-            // CJK character hit. 
+        if (o_processCJK && (csc == CSC_CJK || csc == CSC_HANGUL)) {
+            // CJK character hit. Hangul processing may be special or
+            // not depending on how we were built.
+
             // Do like at EOF with the current non-cjk data.
             if (m_wordLen || m_span.length()) {
                 if (!doemit(true, it.getBpos()))
                     return false;
             }
 
-            // Hand off situation to the cjk routine.
-            if (!cjk_to_words(&it, &c)) {
-                LOGERR("Textsplit: scan error in cjk handler\n");
-                return false;
+            // Hand off situation to the appropriate routine.
+            if (csc == CSC_HANGUL) {
+                if (!ko_to_words(&it, &c)) {
+                    LOGERR("Textsplit: scan error in korean handler\n");
+                    return false;
+                }
+            } else {
+                if (!cjk_to_words(&it, &c)) {
+                    LOGERR("Textsplit: scan error in cjk handler\n");
+                    return false;
+                }
             }
-
             // Check for eof, else c contains the first non-cjk
             // character after the cjk sequence, just go on.
             if (it.eof())

src/common/textsplit.h

@@ -54,6 +54,7 @@ public:
     /** Call at program initialization to read non default values from the 
         configuration */
     static void staticConfInit(RclConfig *config);
+    static void koStaticConfInit(RclConfig *config);
     
     /** Split text, emit words and positions. */
     virtual bool text_to_words(const std::string &in);
@@ -199,6 +200,9 @@ private:
     // This processes cjk text:
     bool cjk_to_words(Utf8Iter *it, unsigned int *cp);
 
+    // Experimental Korean splitter. This uses an external Python tokenizer
+    bool ko_to_words(Utf8Iter *it, unsigned int *cp);
+    
     bool emitterm(bool isspan, std::string &term, int pos, size_t bs,size_t be);
     bool doemit(bool spanerase, size_t bp);
     void discardspan();

src/common/textsplitko.cpp

@@ -0,0 +1,140 @@
+/* Copyright (C) 2020 J.F.Dockes
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include "autoconfig.h"
+
+#include <iostream>
+#include <string>
+#include <cstring>
+#include <unordered_set>
+#include <mutex>
+
+#include "textsplit.h"
+#include "log.h"
+//#define UTF8ITER_CHECK
+#include "utf8iter.h"
+#include "smallut.h"
+#include "rclconfig.h"
+#include "cmdtalk.h"
+
+using namespace std;
+
+static CmdTalk *o_talker;
+static bool o_starterror{false};
+static string o_cmdpath;
+std::mutex o_mutex;
+
+void TextSplit::koStaticConfInit(RclConfig *config)
+{
+    o_cmdpath = config->findFilter("kosplitter.py");
+}
+
+static bool initCmd()
+{
+    if (o_starterror) {
+        // No use retrying
+        return false;
+    }
+    if (o_talker) {
+        return true;
+    }
+    if (o_cmdpath.empty()) {
+        return false;
+    }
+    if (nullptr == (o_talker = new CmdTalk(300))) {
+        o_starterror = true;
+        return false;
+    }
+    if (!o_talker->startCmd(o_cmdpath)) {
+        delete o_talker;
+        o_talker = nullptr;
+        o_starterror = true;
+        return false;
+    }
+    return true;
+}
+
+bool TextSplit::ko_to_words(Utf8Iter *itp, unsigned int *cp)
+{
+    std::unique_lock<std::mutex> mylock(o_mutex);
+    if (nullptr == o_talker) {
+        if (!initCmd()) {
+            return false;
+        }
+    }
+    LOGDEB1("k_to_words: m_wordpos " << m_wordpos << "\n");
+    Utf8Iter &it = *itp;
+    unsigned int c = 0;
+    unordered_map<string, string> args;
+    args.insert(pair<string,string>{"data", string()});
+    string& inputdata{args.begin()->second};
+    
+    // Gather all Korean characters and send the text to the analyser
+    for (; !it.eof(); it++) {
+        c = *it;
+        if (!isHANGUL(c) && !(isascii(c) && (isspace(c) || ispunct(c)))) {
+            // Done with Korean stretch, process and go back to main routine
+            //std::cerr << "Broke on char " << int(c) << endl;
+            break;
+        } else {
+            it.appendchartostring(inputdata);
+        }
+    }
+    LOGDEB1("TextSplit::k_to_words: sending out " << inputdata.size() <<
+            " bytes " << inputdata << endl);
+    unordered_map<string,string> result;
+    if (!o_talker->talk(args, result)) {
+        LOGERR("Python splitter for Korean failed\n");
+        return false;
+    }
+    auto resit = result.find("data");
+    if (resit == result.end()) {
+        LOGERR("No data in Python splitter for Korean\n");
+        return false;
+    }        
+    string& outdata = resit->second;
+    char sepchar = '^';
+    //std::cerr << "GOT FROM SPLITTER: " << outdata << endl;
+    string::size_type wordstart = 0;
+    string::size_type wordend = outdata.find(sepchar);
+    for (;;) {
+        //cerr << "start " << wordstart << " end " << wordend << endl;        
+        if (wordend != wordstart) {
+            string::size_type len = (wordend == string::npos) ?
+                wordend : wordend-wordstart;
+            string word = outdata.substr(wordstart, len);
+            //cerr << " WORD[" <<  word << "]\n";
+            if (!takeword(word, m_wordpos++, 0, 0)) {
+                return false;
+            }
+        }
+        if (wordend == string::npos)
+            break;
+        wordstart = wordend + 1;
+        wordend = outdata.find(sepchar, wordstart);
+    }
+    
+
+    // Reset state, saving term position, and return the found non-cjk
+    // Unicode character value. The current input byte offset is kept
+    // in the utf8Iter
+    int pos = m_wordpos;
+    clearsplitstate();
+    m_spanpos = m_wordpos = pos;
+    *cp = c;
+    return true;
+}