tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

merged the case/diac sensitivity code back into trunk

Browse Source
This commit is contained in:
"Jean-Francois Dockes ext:(%22)
2012-09-25 19:20:24 +02:00
parent a2be0d36ef 1bab5ae658
commit 52bc9f4aa3
40 changed files with 1155 additions and 595 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/rclconfig.cpp
+14 -9
View File
@@ -15,6 +15,8 @@
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef TEST_RCLCONFIG
#include "autoconfig.h"
#include <unistd.h>
#include <fcntl.h>
#include <stdio.h>
@@ -34,6 +36,7 @@
#include <iostream>
#include <cstdlib>
#include <cstring>
using namespace std;
#include "cstr.h"
#include "pathut.h"
@@ -45,15 +48,8 @@
#include "readfile.h"
#include "fstreewalk.h"
#ifndef NO_NAMESPACES
using namespace std;
#endif /* NO_NAMESPACES */
#ifndef MIN
#define MIN(A,B) (((A)<(B)) ? (A) : (B))
#endif
#ifndef MAX
#define MAX(A,B) (((A)>(B)) ? (A) : (B))
#ifndef RCL_INDEX_STRIPCHARS
bool o_index_stripchars;
#endif
bool ParamStale::needrecompute()
@@ -77,6 +73,7 @@ bool ParamStale::needrecompute()
}
return false;
}
void ParamStale::init(RclConfig *rconf, ConfNull *cnf, const string& nm)
{
parent = rconf;
@@ -239,6 +236,14 @@ bool RclConfig::updateMainConfig()
FsTreeWalker::setNoFnmPathname();
}
#ifndef RCL_INDEX_STRIPCHARS
static int m_index_stripchars_init = 0;
if (!m_index_stripchars_init) {
getConfParam("indexStripChars", &o_index_stripchars);
m_index_stripchars_init = 1;
}
#endif
return true;
}
src/common/rclconfig.h
+9 -1
View File
@@ -303,5 +303,13 @@ class RclConfig {
bool readFieldsConfig(const string& errloc);
};
// This global variable defines if we are running with an index
// stripped of accents and case or a raw one. Ideally, it should be
// constant, but it needs to be initialized from the configuration, so
// there is no way to do this. It never changes after initialization
// of course. When set, it is supposed to get all of recoll to behave like if
// if was compiled with RCL_INDEX_STRIPCHARS
#ifndef RCL_INDEX_STRIPCHARS
extern bool o_index_stripchars;
#endif
#endif /* _RCLCONFIG_H_INCLUDED_ */
src/common/unacpp.cpp
+108 -58
View File
@@ -63,26 +63,57 @@ bool unacmaybefold(const string &in, string &out,
return true;
}
// Functions to determine upper-case or accented status could be implemented
// hugely more efficiently inside the unac c code, but there only used for
// testing user-entered terms, so we don't really care.
bool unaciscapital(const string& in)
{
LOGDEB2(("unaciscapital: [%s]\n", in.c_str()));
if (in.empty())
return false;
Utf8Iter it(in);
string shorter;
it.appendchartostring(shorter);
string noacterm, noaclowterm;
if (!unacmaybefold(shorter, noacterm, "UTF-8", UNACOP_UNAC)) {
LOGINFO(("unaciscapital: unac failed for [%s]\n", in.c_str()));
string lower;
if (!unacmaybefold(shorter, lower, "UTF-8", UNACOP_FOLD)) {
LOGINFO(("unaciscapital: unac/fold failed for [%s]\n", in.c_str()));
return false;
}
if (!unacmaybefold(noacterm, noaclowterm, "UTF-8", UNACOP_UNACFOLD)) {
LOGINFO(("unaciscapital: unacfold failed for [%s]\n", in.c_str()));
Utf8Iter it1(lower);
if (*it != *it1)
return true;
else
return false;
}
Utf8Iter it1(noacterm);
Utf8Iter it2(noaclowterm);
if (*it1 != *it2)
}
bool unachasuppercase(const string& in)
{
LOGDEB2(("unachasuppercase: [%s]\n", in.c_str()));
if (in.empty())
return false;
string lower;
if (!unacmaybefold(in, lower, "UTF-8", UNACOP_FOLD)) {
LOGINFO(("unachasuppercase: unac/fold failed for [%s]\n", in.c_str()));
return false;
}
if (lower != in)
return true;
else
return false;
}
bool unachasaccents(const string& in)
{
LOGDEB2(("unachasaccents: [%s]\n", in.c_str()));
if (in.empty())
return false;
string noac;
if (!unacmaybefold(in, noac, "UTF-8", UNACOP_UNAC)) {
LOGINFO(("unachasaccents: unac/unac failed for [%s]\n", in.c_str()));
return false;
}
if (noac != in)
return true;
else
return false;
@@ -107,12 +138,15 @@ static char *thisprog;
static char usage [] = "\n"
"[-c|-C] <encoding> <infile> <outfile>\n"
" Default : unaccent\n"
" -c : unaccent and casefold\n"
" -C : casefold only\n"
" Default : unaccent\n"
" -c : unaccent and casefold\n"
" -C : casefold only\n"
"-t <string> test string as capitalized, upper-case anywhere, accents\n"
" the parameter is supposedly utf-8 so this can only work in an utf-8\n"
" locale\n"
"\n";
;
static void
Usage(void)
{
@@ -123,6 +157,7 @@ Usage(void)
static int op_flags;
#define OPT_c 0x2
#define OPT_C 0x4
#define OPT_t 0x8
int main(int argc, char **argv)
{
@@ -140,58 +175,73 @@ int main(int argc, char **argv)
switch (*(*argv)++) {
case 'c': op_flags |= OPT_c; break;
case 'C': op_flags |= OPT_C; break;
case 't': op_flags |= OPT_t; break;
default: Usage(); break;
}
argc--; argv++;
}
if (op_flags & OPT_c) {
op = UNACOP_UNACFOLD;
} else if (op_flags & OPT_C) {
op = UNACOP_FOLD;
}
if (argc != 3) {
Usage();
}
const char *encoding = *argv++; argc--;
string ifn = *argv++; argc--;
if (!ifn.compare("stdin"))
ifn.clear();
const char *ofn = *argv++; argc--;
string reason;
(void)recollinit(RCLINIT_NONE, 0, 0, reason, 0);
string odata;
if (!file_to_string(ifn, odata)) {
cerr << "file_to_string " << ifn << " : " << odata << endl;
return 1;
}
string ndata;
if (!unacmaybefold(odata, ndata, encoding, op)) {
cerr << "unac: " << ndata << endl;
return 1;
}
int fd;
if (strcmp(ofn, "stdout")) {
fd = open(ofn, O_CREAT|O_EXCL|O_WRONLY, 0666);
if (op_flags & OPT_t) {
if (argc != 1)
Usage();
string in = *argv++;argc--;
bool capital, upper, accent;
capital = unaciscapital(in);
upper = unachasuppercase(in);
accent = unachasaccents(in);
cout << "[" << in << "] : " <<
"capitalized: " << (capital ? "Yes. " : "No. ") <<
"has uppercase: " << (upper ? "Yes. " : "No. ") <<
"has accents: " << (accent ? "Yes. " : "No. ") <<
endl;
return 0;
} else {
fd = 1;
if (argc != 3)
Usage();
if (op_flags & OPT_c) {
op = UNACOP_UNACFOLD;
} else if (op_flags & OPT_C) {
op = UNACOP_FOLD;
}
const char *encoding = *argv++; argc--;
string ifn = *argv++; argc--;
if (!ifn.compare("stdin"))
ifn.clear();
const char *ofn = *argv++; argc--;
string reason;
(void)recollinit(RCLINIT_NONE, 0, 0, reason, 0);
string odata;
if (!file_to_string(ifn, odata)) {
cerr << "file_to_string " << ifn << " : " << odata << endl;
return 1;
}
string ndata;
if (!unacmaybefold(odata, ndata, encoding, op)) {
cerr << "unac: " << ndata << endl;
return 1;
}
int fd;
if (strcmp(ofn, "stdout")) {
fd = open(ofn, O_CREAT|O_EXCL|O_WRONLY, 0666);
} else {
fd = 1;
}
if (fd < 0) {
cerr << "Open/Create " << ofn << " failed: " << strerror(errno)
<< endl;
return 1;
}
if (write(fd, ndata.c_str(), ndata.length()) != (int)ndata.length()) {
cerr << "Write(2) failed: " << strerror(errno) << endl;
return 1;
}
close(fd);
return 0;
}
if (fd < 0) {
cerr << "Open/Create " << ofn << " failed: " << strerror(errno)
<< endl;
return 1;
}
if (write(fd, ndata.c_str(), ndata.length()) != (int)ndata.length()) {
cerr << "Write(2) failed: " << strerror(errno) << endl;
return 1;
}
close(fd);
return 0;
}
#endif
src/common/unacpp.h
+7 -1
View File
@@ -24,11 +24,17 @@ using std::string;
#endif /* NO_NAMESPACES */
// A small stringified wrapper for unac.c
enum UnacOp {UNACOP_UNAC, UNACOP_UNACFOLD, UNACOP_FOLD};
enum UnacOp {UNACOP_UNAC = 1, UNACOP_FOLD = 2, UNACOP_UNACFOLD = 3};
extern bool unacmaybefold(const string& in, string& out,
const char *encoding, UnacOp what);
// Utility function to determine if string begins with capital
extern bool unaciscapital(const string& in);
// Utility function to determine if string has upper-case anywhere
extern bool unachasuppercase(const string& in);
// Utility function to determine if any character is accented. This
// approprialey ignores the characters from unac_except_chars which
// are really separate letters
extern bool unachasaccents(const string& in);
#endif /* _UNACPP_H_INCLUDED_ */