modified the time at which we unaccent so that we can do the Capitalized->nostemming test on single words (this had been broken by the change of noac/split order done earlier to get japanese to work)

This commit is contained in:
dockes
2009-01-26 18:30:48 +00:00
parent 30c46709ba
commit 7dcc7c61c8
4 changed files with 96 additions and 108 deletions
+5 -1
View File
@@ -42,6 +42,7 @@ using std::set;
#include "smallut.h"
#include "plaintorich.h"
#include "cancelcheck.h"
#include "unacpp.h"
const string PlainToRich::snull = "";
@@ -84,7 +85,10 @@ class myTextSplitCB : public TextSplitCB {
// Callback called by the text-to-words breaker for each word
virtual bool takeword(const std::string& term, int pos, int bts, int bte) {
string dumb;
Rcl::dumb_string(term, dumb);
if (!unacmaybefold(term, dumb, "UTF-8", true)) {
LOGINFO(("PlainToRich::splitter::takeword: unac failed for [%s]\n", term.c_str()));
return true;
}
//LOGDEB2(("Input dumbbed term: '%s' %d %d %d\n", dumb.c_str(),
// pos, bts, bte));