tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

from 1.18 branch: When creating initial config directory (1st exec), initialize specific unac_except_trans for some languages: de, se/no/dk/fi + fix mixup of language and country codes

Browse Source
This commit is contained in:
Jean-Francois Dockes
2012-11-01 11:27:50 +01:00
parent ee7d0f2ee7
commit 04c19b33d5
5 changed files with 87 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/internfile/txtdcode.cpp
+2 -56
View File
@@ -15,47 +15,12 @@
*/
#include "autoconfig.h"
#include <tr1/unordered_map>
using std::tr1::unordered_map;
#include "cstr.h"
#include "transcode.h"
#include "mimehandler.h"
#include "debuglog.h"
#include "smallut.h"
static const char *vcountry_to_code[] = {
"fr", "windows-1252",
"al", "windows-1252",
"dk", "windows-1252",
"en", "windows-1252",
"de", "windows-1252",
"is", "windows-1252",
"my", "windows-1252",
"ie", "windows-1252",
"gb", "windows-1252",
"it", "windows-1252",
"lu", "windows-1252",
"no", "windows-1252",
"pt", "windows-1252",
"es", "windows-1252",
"se", "windows-1252",
"ba", "iso-8859-2",
"hr", "iso-8859-2",
"cz", "iso-8859-2",
"hu", "iso-8859-2",
"pl", "iso-8859-2",
"rs", "iso-8859-2",
"sk", "iso-8859-2",
"si", "iso-8859-2",
"gr", "iso-8859-7",
"il", "iso-8859-8",
"tr", "iso-8859-9",
"th", "iso-8859-11",
"lv", "iso-8859-13",
"lt", "iso-8859-13",
};
// Called after decoding from utf-8 failed. Handle the common case
// where this is a good old 8bit-encoded text document left-over when
@@ -64,27 +29,8 @@ static const char *vcountry_to_code[] = {
// heuristic, but may be better than discarding the data.
static bool alternate_decode(const string& in, string& out)
{
static unordered_map<string, string> country_to_code;
if (country_to_code.empty()) {
for (unsigned int i = 0;
i < sizeof(vcountry_to_code) / sizeof(char *); i += 2) {
country_to_code[vcountry_to_code[i]] = vcountry_to_code[i+1];
}
}
string locale = setlocale(LC_CTYPE, 0);
LOGDEB(("RecollFilter::alternate_dcde: locale: [%s]\n", locale.c_str()));
string::size_type under = locale.find_first_of("_");
if (under == string::npos)
return false;
string country = locale.substr(0, under);
unordered_map<string,string>::const_iterator it =
country_to_code.find(country);
if (it == country_to_code.end())
return false;
string code = it->second;
string lang = localelang();
string code = langtocode(lang);
LOGDEB(("RecollFilter::txtdcode: trying alternate decode from %s\n",
code.c_str()));
return transcode(in, out, code, cstr_utf8);