tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

korean textsplit with extern help from konlpy, first step

Browse Source
This commit is contained in:
Jean-Francois Dockes
2020-03-22 10:09:50 +01:00
parent d83bb8cf69
commit 384e3a1087
8 changed files with 802 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils/cmdtalk.cpp
+243
View File
@@ -0,0 +1,243 @@
/* Copyright (C) 2016 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "cmdtalk.h"
#include <stdio.h>
#include <iostream>
#include <sstream>
#include <mutex>
#include "smallut.h"
#include "execmd.h"
#ifdef MDU_INCLUDE_LOG
#include MDU_INCLUDE_LOG
#else
#include "log.h"
#endif
using namespace std;
class TimeoutExcept {};
class Canceler : public ExecCmdAdvise {
public:
Canceler(int tmsecs)
: m_timeosecs(tmsecs) {}
virtual void newData(int cnt) {
if (m_starttime && (time(0) - m_starttime) > m_timeosecs) {
throw TimeoutExcept();
}
}
void reset() {
m_starttime = time(0);
}
int m_timeosecs;
time_t m_starttime{0};
};
class CmdTalk::Internal {
public:
Internal(int timeosecs)
: m_cancel(timeosecs) {}
~Internal() {
delete cmd;
}
bool readDataElement(string& name, string &data);
bool talk(const pair<string, string>& arg0,
const unordered_map<string, string>& args,
unordered_map<string, string>& rep);
ExecCmd *cmd{0};
Canceler m_cancel;
std::mutex mmutex;
};
CmdTalk::CmdTalk(int timeosecs)
{
m = new Internal(timeosecs);
}
CmdTalk::~CmdTalk()
{
delete m;
}
bool CmdTalk::startCmd(const string& cmdname,
const vector<string>& args,
const vector<string>& env,
const vector<string>& path)
{
LOGDEB("CmdTalk::startCmd\n" );
delete m->cmd;
m->cmd = new ExecCmd;
m->cmd->setAdvise(&m->m_cancel);
for (const auto& it : env) {
m->cmd->putenv(it);
}
string acmdname(cmdname);
if (!path.empty()) {
string colonpath;
for (const auto& it: path) {
colonpath += it + ":";
}
if (!colonpath.empty()) {
colonpath.erase(colonpath.size()-1);
}
LOGDEB("CmdTalk::startCmd: PATH: [" << colonpath << "]\n");
ExecCmd::which(cmdname, acmdname, colonpath.c_str());
}
if (m->cmd->startExec(acmdname, args, 1, 1) < 0) {
return false;
}
return true;
}
// Messages are made of data elements. Each element is like:
// name: len\ndata
// An empty line signals the end of the message, so the whole thing
// would look like:
// Name1: Len1\nData1Name2: Len2\nData2\n
bool CmdTalk::Internal::readDataElement(string& name, string &data)
{
string ibuf;
m_cancel.reset();
try {
// Read name and length
if (cmd->getline(ibuf) <= 0) {
LOGERR("CmdTalk: getline error\n" );
return false;
}
} catch (TimeoutExcept) {
LOGINF("CmdTalk:readDataElement: fatal timeout (" <<
m_cancel.m_timeosecs << " S)\n");
return false;
}
LOGDEB1("CmdTalk:rde: line [" << (ibuf) << "]\n" );
// Empty line (end of message) ?
if (!ibuf.compare("\n")) {
LOGDEB("CmdTalk: Got empty line\n" );
return true;
}
// We're expecting something like Name: len\n
vector<string> tokens;
stringToTokens(ibuf, tokens);
if (tokens.size() != 2) {
LOGERR("CmdTalk: bad line in filter output: [" << (ibuf) << "]\n" );
return false;
}
vector<string>::iterator it = tokens.begin();
name = *it++;
string& slen = *it;
int len;
if (sscanf(slen.c_str(), "%d", &len) != 1) {
LOGERR("CmdTalk: bad line in filter output: [" << (ibuf) << "]\n" );
return false;
}
// Read element data
data.erase();
if (len > 0 && cmd->receive(data, len) != len) {
LOGERR("CmdTalk: expected " << len << " bytes of data, got " <<
data.length() << "\n");
return false;
}
LOGDEB1("CmdTalk:rde: got: name [" << name << "] len " << len <<"value ["<<
(data.size() > 100 ? (data.substr(0, 100) + " ...") : data)<< endl);
return true;
}
bool CmdTalk::Internal::talk(const pair<string, string>& arg0,
const unordered_map<string, string>& args,
unordered_map<string, string>& rep)
{
std::unique_lock<std::mutex> lock(mmutex);
if (cmd->getChildPid() <= 0) {
LOGERR("CmdTalk::talk: no process\n");
return false;
}
ostringstream obuf;
if (!arg0.first.empty()) {
obuf << arg0.first << ": " << arg0.second.size() << "\n" << arg0.second;
}
for (const auto& it : args) {
obuf << it.first << ": " << it.second.size() << "\n" << it.second;
}
obuf << "\n";
if (cmd->send(obuf.str()) < 0) {
cmd->zapChild();
LOGERR("CmdTalk: send error\n" );
return false;
}
// Read answer (multiple elements)
LOGDEB1("CmdTalk: reading answer\n" );
for (;;) {
string name, data;
if (!readDataElement(name, data)) {
cmd->zapChild();
return false;
}
if (name.empty()) {
break;
}
trimstring(name, ":");
LOGDEB1("CmdTalk: got [" << name << "] -> [" << data << "]\n");
rep[name] = data;
}
if (rep.find("cmdtalkstatus") != rep.end()) {
return false;
} else {
return true;
}
}
bool CmdTalk::running()
{
return m && m->cmd && m->cmd->getChildPid() > 0;
}
bool CmdTalk::talk(const unordered_map<string, string>& args,
unordered_map<string, string>& rep)
{
return m->talk({"",""}, args, rep);
}
bool CmdTalk::callproc(
const string& proc,
const unordered_map<std::string, std::string>& args,
unordered_map<std::string, std::string>& rep)
{
return m->talk({"cmdtalk:proc", proc}, args, rep);
}
src/utils/cmdtalk.h
+105
View File
@@ -0,0 +1,105 @@
/* Copyright (C) 2016 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef _CMDTALK_H_INCLUDED_
#define _CMDTALK_H_INCLUDED_
/**
* Execute commands and exchange messages with it.
*
* A simple stream protocol is used for the dialog. HTTP or some kind
* of full-blown RPC could have been used, but there was also good
* reason to keep it simple (yet powerful), given the limited context
* of dialog through a pipe.
*
* The data is exchanged in TLV fashion, in a way that should be
* usable in most script languages. The basic unit of data has one line
* with a data type and a count (both ASCII), followed by the data. A
* 'message' is made of one or several units or tags and ends with one empty
* line.
*
* Example:(the message begins before 'Filename' and has 'Filename' and
* 'Ipath' tags):
*
Filename: 24
/my/home/mail/somefolderIpath: 2
22
<Message ends here: because of the empty line after '22'
*
* Example answer, with 'Mimetype' and 'Data' tags
*
Mimetype: 10
text/plainData: 10
0123456789
<Message ends here because of empty line
*
* This format is both extensible and reasonably easy to parse.
* While it's more fitted for python or perl on the script side, it
* should even be sort of usable from the shell (e.g.: use dd to read
* the counted data). Most alternatives would need data encoding in
* some cases.
*
* Higher level dialog:
* The C++ program is the master and sends request messages to the script.
* Both sides of the communication should be prepared to receive and discard
* unknown tags.
*/
#include <string>
#include <vector>
#include <unordered_map>
class CmdTalk {
public:
CmdTalk(int timeosecs);
virtual ~CmdTalk();
// @param env each entry should be of the form name=value. They
// augment the subprocess environnement.
// @param path replaces the PATH variable when looking for the command.
virtual bool startCmd(const std::string& cmdname,
const std::vector<std::string>& args =
std::vector<std::string>(),
const std::vector<std::string>& env =
std::vector<std::string>(),
const std::vector<std::string>& path =
std::vector<std::string>()
);
virtual bool running();
// Single exchange: send and receive data.
virtual bool talk(const std::unordered_map<std::string, std::string>& args,
std::unordered_map<std::string, std::string>& rep);
// Specialized version with special argument used by dispatcher to call
// designated method
virtual bool callproc(
const std::string& proc,
const std::unordered_map<std::string, std::string>& args,
std::unordered_map<std::string, std::string>& rep);
CmdTalk(const CmdTalk&) = delete;
CmdTalk &operator=(const CmdTalk &) = delete;
private:
class Internal;
Internal *m{0};
};
#endif /* _CMDTALK_H_INCLUDED_ */