Added a configuration parameter to set specific unaccenting/lowercasing for some characters to be handled differently than would result from using the Unicode database. Exemple: "a with ring above" could be set to be preserved by a Swedish locutor

This commit is contained in:
Jean-Francois Dockes
2012-04-09 12:42:23 +02:00
parent 29ef5bd143
commit a4c17941b1
14 changed files with 365 additions and 23 deletions
+6
View File
@@ -107,9 +107,15 @@ RclConfig *recollinit(RclInitFlags flags,
// Make sure the locale charset is initialized (so that multiple
// threads don't try to do it at once).
config->getDefCharset();
// Init unac locking
unac_init_mt();
// Init Unac translation exceptions
string unacex;
if (config->getConfParam("unac_except_trans", unacex) && !unacex.empty())
unac_set_except_translations(unacex.c_str());
int flushmb;
if (config->getConfParam("idxflushmb", &flushmb) && flushmb > 0) {
LOGDEB1(("rclinit: idxflushmb=%d, set XAPIAN_FLUSH_THRESHOLD to 10E6\n",
+12 -1
View File
@@ -88,6 +88,7 @@ using namespace std;
#include "unacpp.h"
#include "readfile.h"
#include "rclinit.h"
int main(int argc, char **argv)
{
@@ -98,8 +99,13 @@ int main(int argc, char **argv)
}
const char *encoding = argv[1];
string ifn = argv[2];
if (!ifn.compare("stdin"))
ifn.clear();
const char *ofn = argv[3];
string reason;
(void)recollinit(RCLINIT_NONE, 0, 0, reason, 0);
string odata;
if (!file_to_string(ifn, odata)) {
cerr << "file_to_string: " << odata << endl;
@@ -111,7 +117,12 @@ int main(int argc, char **argv)
exit(1);
}
int fd = open(ofn, O_CREAT|O_EXCL|O_WRONLY, 0666);
int fd;
if (strcmp(ofn, "stdout")) {
fd = open(ofn, O_CREAT|O_EXCL|O_WRONLY, 0666);
} else {
fd = 1;
}
if (fd < 0) {
cerr << "Open/Create " << ofn << " failed: " << strerror(errno)
<< endl;