Add aspell support on Windows

This commit is contained in:
Jean-Francois Dockes 2019-10-13 10:12:53 +02:00
parent a96ee950b1
commit 6c5440ff7b
10 changed files with 497 additions and 325 deletions

View File

@ -1,5 +1,4 @@
#ifndef TEST_RCLASPELL /* Copyright (C) 2006-2019 J.F.Dockes
/* Copyright (C) 2006 J.F.Dockes
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or * the Free Software Foundation; either version 2 of the License, or
@ -15,49 +14,47 @@
* Free Software Foundation, Inc., * Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/ */
#ifdef HAVE_CONFIG_H
#include "autoconfig.h" #include "autoconfig.h"
#endif
#ifdef RCL_USE_ASPELL #ifdef RCL_USE_ASPELL
#include <mutex>
#include <unistd.h>
#include <dlfcn.h>
#include <stdlib.h>
#include ASPELL_INCLUDE #include ASPELL_INCLUDE
#include <mutex>
#include <unistd.h>
#include <stdlib.h>
#include "dlib.h"
#include "pathut.h" #include "pathut.h"
#include "execmd.h" #include "execmd.h"
#include "rclaspell.h" #include "rclaspell.h"
#include "log.h" #include "log.h"
#include "unacpp.h" #include "unacpp.h"
#include "rclutil.h"
using namespace std; using namespace std;
// Just a place where we keep the Aspell library entry points together // Aspell library entry points
class AspellApi { class AspellApi {
public: public:
struct AspellConfig *(*new_aspell_config)(); struct AspellConfig *(*new_aspell_config)();
int (*aspell_config_replace)(struct AspellConfig *, const char * key, int (*aspell_config_replace)(struct AspellConfig *, const char * key,
const char * value); const char * value);
struct AspellCanHaveError *(*new_aspell_speller)(struct AspellConfig *); struct AspellCanHaveError *(*new_aspell_speller)(struct AspellConfig *);
void (*delete_aspell_config)(struct AspellConfig *); void (*delete_aspell_config)(struct AspellConfig *);
void (*delete_aspell_can_have_error)(struct AspellCanHaveError *); void (*delete_aspell_can_have_error)(struct AspellCanHaveError *);
struct AspellSpeller * (*to_aspell_speller)(struct AspellCanHaveError *); struct AspellSpeller * (*to_aspell_speller)(struct AspellCanHaveError *);
struct AspellConfig * (*aspell_speller_config)(struct AspellSpeller *); struct AspellConfig * (*aspell_speller_config)(struct AspellSpeller *);
const struct AspellWordList * (*aspell_speller_suggest) const struct AspellWordList * (*aspell_speller_suggest)
(struct AspellSpeller *, const char *, int); (struct AspellSpeller *, const char *, int);
int (*aspell_speller_check)(struct AspellSpeller *, const char *, int); int (*aspell_speller_check)(struct AspellSpeller *, const char *, int);
struct AspellStringEnumeration * (*aspell_word_list_elements) struct AspellStringEnumeration * (*aspell_word_list_elements)
(const struct AspellWordList * ths); (const struct AspellWordList * ths);
const char * (*aspell_string_enumeration_next) const char * (*aspell_string_enumeration_next)
(struct AspellStringEnumeration * ths); (struct AspellStringEnumeration * ths);
void (*delete_aspell_string_enumeration)(struct AspellStringEnumeration *); void (*delete_aspell_string_enumeration)(struct AspellStringEnumeration *);
const struct AspellError *(*aspell_error) const struct AspellError *(*aspell_error)
(const struct AspellCanHaveError *); (const struct AspellCanHaveError *);
const char *(*aspell_error_message)(const struct AspellCanHaveError *); const char *(*aspell_error_message)(const struct AspellCanHaveError *);
const char *(*aspell_speller_error_message)(const struct AspellSpeller *); const char *(*aspell_speller_error_message)(const struct AspellSpeller *);
void (*delete_aspell_speller)(struct AspellSpeller *); void (*delete_aspell_speller)(struct AspellSpeller *);
@ -66,49 +63,51 @@ public:
static AspellApi aapi; static AspellApi aapi;
static std::mutex o_aapi_mutex; static std::mutex o_aapi_mutex;
#define NMTOPTR(NM, TP) \ #define NMTOPTR(NM, TP) \
if ((aapi.NM = TP dlsym(m_data->m_handle, #NM)) == 0) { \ if ((aapi.NM = TP dlib_sym(m_data->m_handle, #NM)) == 0) { \
badnames += #NM + string(" "); \ badnames += #NM + string(" "); \
} }
static const vector<string> aspell_lib_suffixes { static const vector<string> aspell_lib_suffixes {
#if defined(__APPLE__) #if defined(__APPLE__)
".15.dylib", ".15.dylib",
".dylib", ".dylib",
#elif defined(_WIN32)
"-15.dll",
#else #else
".so", ".so",
".so.15", ".so.15",
".so.16",
#endif #endif
}; };
// Stuff that we don't wish to see in the .h (possible sysdeps, etc.) // Private rclaspell data
class AspellData { class AspellData {
public: public:
AspellData()
: m_handle(0), m_speller(0)
{}
~AspellData() { ~AspellData() {
LOGDEB2("~AspellData\n" ); LOGDEB2("~AspellData\n" );
if (m_handle) { if (m_handle) {
dlclose(m_handle); dlib_close(m_handle);
m_handle = 0; m_handle = nullptr;
} }
if (m_speller) { if (m_speller) {
// Dumps core if I do this?? // Dumps core if I do this??
//aapi.delete_aspell_speller(m_speller); //aapi.delete_aspell_speller(m_speller);
m_speller = 0; m_speller = nullptr;
LOGDEB2("~AspellData: speller done\n" ); LOGDEB2("~AspellData: speller done\n" );
} }
} }
void *m_handle; void *m_handle{nullptr};
string m_exec; string m_exec;
AspellSpeller *m_speller; AspellSpeller *m_speller{nullptr};
#ifdef _WIN32
string m_datadir;
#endif
string m_addCreateParam;
}; };
Aspell::Aspell(const RclConfig *cnf) Aspell::Aspell(const RclConfig *cnf)
: m_config(cnf), m_data(0) : m_config(cnf)
{ {
} }
@ -126,15 +125,15 @@ bool Aspell::init(string &reason)
// environment. The aspell language names used for selecting language // environment. The aspell language names used for selecting language
// definition files (used to create dictionaries) are like en, fr // definition files (used to create dictionaries) are like en, fr
if (!m_config->getConfParam("aspellLanguage", m_lang) || m_lang.empty()) { if (!m_config->getConfParam("aspellLanguage", m_lang) || m_lang.empty()) {
string lang = "en"; string lang = "en";
const char *cp; const char *cp;
if ((cp = getenv("LC_ALL"))) if ((cp = getenv("LC_ALL")))
lang = cp; lang = cp;
else if ((cp = getenv("LANG"))) else if ((cp = getenv("LANG")))
lang = cp; lang = cp;
if (!lang.compare("C")) if (!lang.compare("C"))
lang = "en"; lang = "en";
m_lang = lang.substr(0, lang.find_first_of("_")); m_lang = lang.substr(0, lang.find_first_of("_"));
if (!m_lang.compare("ja")) { if (!m_lang.compare("ja")) {
// Aspell has no support for Japanese. We substitute // Aspell has no support for Japanese. We substitute
// english, as Japanese users often have texts with // english, as Japanese users often have texts with
@ -147,23 +146,38 @@ bool Aspell::init(string &reason)
m_data = new AspellData; m_data = new AspellData;
m_config->getConfParam("aspellAddCreateParam", m_data->m_addCreateParam);
#ifdef _WIN32
m_data->m_datadir = path_cat(
path_pkgdatadir(),
"filters/aspell-installed/mingw32/lib/aspell-0.60");
if (m_data->m_addCreateParam.empty()) {
m_data->m_addCreateParam = string("--local-data-dir=") +
path_cat(m_config->getConfDir(), "aspell");
}
#endif // WIN32
const char *aspell_prog_from_env = getenv("ASPELL_PROG"); const char *aspell_prog_from_env = getenv("ASPELL_PROG");
if (aspell_prog_from_env && access(aspell_prog_from_env, X_OK) == 0) { if (aspell_prog_from_env && access(aspell_prog_from_env, X_OK) == 0) {
m_data->m_exec = aspell_prog_from_env; m_data->m_exec = aspell_prog_from_env;
}
#ifdef ASPELL_PROG #ifdef ASPELL_PROG
} else if (access(ASPELL_PROG, X_OK) == 0) {
m_data->m_exec = ASPELL_PROG;
#endif // ASPELL_PROG
} else {
ExecCmd::which("aspell", m_data->m_exec);
}
if (m_data->m_exec.empty()) { if (m_data->m_exec.empty()) {
reason = "aspell program not found or not executable"; string cmd = m_config->findFilter(ASPELL_PROG);
deleteZ(m_data); LOGDEB("rclaspell::init: findFilter returns " << cmd << endl);
return false; if (path_isabsolute(cmd)) {
m_data->m_exec.swap(cmd);
}
}
#endif // ASPELL_PROG
if (m_data->m_exec.empty()) {
ExecCmd::which("aspell", m_data->m_exec);
}
if (m_data->m_exec.empty()) {
reason = "aspell program not found or not executable";
deleteZ(m_data);
return false;
} }
// Don't know what with Apple and (DY)LD_LIBRARY_PATH. Does not work // Don't know what with Apple and (DY)LD_LIBRARY_PATH. Does not work
// So we look in all ../lib in the PATH... // So we look in all ../lib in the PATH...
@ -181,16 +195,28 @@ bool Aspell::init(string &reason)
for (const auto& suff : aspell_lib_suffixes) { for (const auto& suff : aspell_lib_suffixes) {
lib = libbase + suff; lib = libbase + suff;
reason += string("[") + lib + "] "; reason += string("[") + lib + "] ";
if ((m_data->m_handle = dlopen(lib.c_str(), RTLD_LAZY)) != 0) { if ((m_data->m_handle = dlib_open(lib)) != 0) {
reason.erase(); reason.erase();
goto found; goto found;
} }
#if defined(__APPLE__)
// Above was the normal lookup: let dlopen search the directories. // Above was the normal lookup: let dlopen search the directories.
// Here is for Apple. Also look at all ../lib along the PATH // Also look in other places for Apple and Windows.
#if defined(__APPLE__)
for (const auto& dir : path) { for (const auto& dir : path) {
string lib1 = path_canon(dir + "/../lib/" + lib); string lib1 = path_canon(dir + "/../lib/" + lib);
if ((m_data->m_handle = dlopen(lib1.c_str(), RTLD_LAZY)) != 0) { if ((m_data->m_handle = dlib_open(lib1)) != 0) {
reason.erase();
lib=lib1;
goto found;
}
}
#endif
#if defined(_WIN32)
// Look in the directory of the aspell binary
{
string bindir = path_getfather(m_data->m_exec);
string lib1 = path_cat(bindir, lib);
if ((m_data->m_handle = dlib_open(lib1)) != 0) {
reason.erase(); reason.erase();
lib=lib1; lib=lib1;
goto found; goto found;
@ -199,9 +225,9 @@ bool Aspell::init(string &reason)
#endif #endif
} }
found: found:
if (m_data->m_handle == 0) { if (m_data->m_handle == 0) {
reason += string(" : ") + dlerror(); reason += string(" : ") + dlib_error();
deleteZ(m_data); deleteZ(m_data);
return false; return false;
} }
@ -209,41 +235,41 @@ bool Aspell::init(string &reason)
string badnames; string badnames;
NMTOPTR(new_aspell_config, (struct AspellConfig *(*)())); NMTOPTR(new_aspell_config, (struct AspellConfig *(*)()));
NMTOPTR(aspell_config_replace, (int (*)(struct AspellConfig *, NMTOPTR(aspell_config_replace, (int (*)(struct AspellConfig *,
const char *, const char *))); const char *, const char *)));
NMTOPTR(new_aspell_speller, NMTOPTR(new_aspell_speller,
(struct AspellCanHaveError *(*)(struct AspellConfig *))); (struct AspellCanHaveError *(*)(struct AspellConfig *)));
NMTOPTR(delete_aspell_config, NMTOPTR(delete_aspell_config,
(void (*)(struct AspellConfig *))); (void (*)(struct AspellConfig *)));
NMTOPTR(delete_aspell_can_have_error, NMTOPTR(delete_aspell_can_have_error,
(void (*)(struct AspellCanHaveError *))); (void (*)(struct AspellCanHaveError *)));
NMTOPTR(to_aspell_speller, NMTOPTR(to_aspell_speller,
(struct AspellSpeller *(*)(struct AspellCanHaveError *))); (struct AspellSpeller *(*)(struct AspellCanHaveError *)));
NMTOPTR(aspell_speller_config, NMTOPTR(aspell_speller_config,
(struct AspellConfig *(*)(struct AspellSpeller *))); (struct AspellConfig *(*)(struct AspellSpeller *)));
NMTOPTR(aspell_speller_suggest, NMTOPTR(aspell_speller_suggest,
(const struct AspellWordList *(*)(struct AspellSpeller *, (const struct AspellWordList *(*)(struct AspellSpeller *,
const char *, int))); const char *, int)));
NMTOPTR(aspell_speller_check, NMTOPTR(aspell_speller_check,
(int (*)(struct AspellSpeller *, const char *, int))); (int (*)(struct AspellSpeller *, const char *, int)));
NMTOPTR(aspell_word_list_elements, NMTOPTR(aspell_word_list_elements,
(struct AspellStringEnumeration *(*) (struct AspellStringEnumeration *(*)
(const struct AspellWordList *))); (const struct AspellWordList *)));
NMTOPTR(aspell_string_enumeration_next, NMTOPTR(aspell_string_enumeration_next,
(const char * (*)(struct AspellStringEnumeration *))); (const char * (*)(struct AspellStringEnumeration *)));
NMTOPTR(delete_aspell_string_enumeration, NMTOPTR(delete_aspell_string_enumeration,
(void (*)(struct AspellStringEnumeration *))); (void (*)(struct AspellStringEnumeration *)));
NMTOPTR(aspell_error, NMTOPTR(aspell_error,
(const struct AspellError*(*)(const struct AspellCanHaveError *))); (const struct AspellError*(*)(const struct AspellCanHaveError *)));
NMTOPTR(aspell_error_message, NMTOPTR(aspell_error_message,
(const char *(*)(const struct AspellCanHaveError *))); (const char *(*)(const struct AspellCanHaveError *)));
NMTOPTR(aspell_speller_error_message, NMTOPTR(aspell_speller_error_message,
(const char *(*)(const struct AspellSpeller *))); (const char *(*)(const struct AspellSpeller *)));
NMTOPTR(delete_aspell_speller, (void (*)(struct AspellSpeller *))); NMTOPTR(delete_aspell_speller, (void (*)(struct AspellSpeller *)));
if (!badnames.empty()) { if (!badnames.empty()) {
reason = string("Aspell::init: symbols not found:") + badnames; reason = string("Aspell::init: symbols not found:") + badnames;
deleteZ(m_data); deleteZ(m_data);
return false; return false;
} }
return true; return true;
@ -274,29 +300,29 @@ public:
Rcl::TermIter *m_tit; Rcl::TermIter *m_tit;
Rcl::Db &m_db; Rcl::Db &m_db;
AspExecPv(string *i, Rcl::TermIter *tit, Rcl::Db &db) AspExecPv(string *i, Rcl::TermIter *tit, Rcl::Db &db)
: m_input(i), m_tit(tit), m_db(db) : m_input(i), m_tit(tit), m_db(db)
{} {}
void newData() { void newData() {
while (m_db.termWalkNext(m_tit, *m_input)) { while (m_db.termWalkNext(m_tit, *m_input)) {
LOGDEB2("Aspell::buildDict: term: [" << (m_input) << "]\n" ); LOGDEB2("Aspell::buildDict: term: [" << (m_input) << "]\n" );
if (!Rcl::Db::isSpellingCandidate(*m_input)) { if (!Rcl::Db::isSpellingCandidate(*m_input)) {
LOGDEB2("Aspell::buildDict: SKIP\n" ); LOGDEB2("Aspell::buildDict: SKIP\n" );
continue; continue;
} }
if (!o_index_stripchars) { if (!o_index_stripchars) {
string lower; string lower;
if (!unacmaybefold(*m_input, lower, "UTF-8", UNACOP_FOLD)) if (!unacmaybefold(*m_input, lower, "UTF-8", UNACOP_FOLD))
continue; continue;
m_input->swap(lower); m_input->swap(lower);
} }
// Got a non-empty sort-of appropriate term, let's send it to // Got a non-empty sort-of appropriate term, let's send it to
// aspell // aspell
LOGDEB2("Apell::buildDict: SEND\n" ); LOGDEB2("Apell::buildDict: SEND\n" );
m_input->append("\n"); m_input->append("\n");
return; return;
} }
// End of data. Tell so. Exec will close cmd. // End of data. Tell so. Exec will close cmd.
m_input->erase(); m_input->erase();
} }
}; };
@ -304,10 +330,7 @@ public:
bool Aspell::buildDict(Rcl::Db &db, string &reason) bool Aspell::buildDict(Rcl::Db &db, string &reason)
{ {
if (!ok()) if (!ok())
return false; return false;
string addCreateParam;
m_config->getConfParam("aspellAddCreateParam", addCreateParam);
// We create the dictionary by executing the aspell command: // We create the dictionary by executing the aspell command:
// aspell --lang=[lang] create master [dictApath] // aspell --lang=[lang] create master [dictApath]
@ -318,9 +341,12 @@ bool Aspell::buildDict(Rcl::Db &db, string &reason)
cmdstring += string(" ") + string("--lang=") + m_lang; cmdstring += string(" ") + string("--lang=") + m_lang;
args.push_back("--encoding=utf-8"); args.push_back("--encoding=utf-8");
cmdstring += string(" ") + "--encoding=utf-8"; cmdstring += string(" ") + "--encoding=utf-8";
if (!addCreateParam.empty()) { #ifdef _WIN32
args.push_back(addCreateParam); args.push_back(string("--data-dir=") + m_data->m_datadir);
cmdstring += string(" ") + addCreateParam; #endif
if (!m_data->m_addCreateParam.empty()) {
args.push_back(m_data->m_addCreateParam);
cmdstring += string(" ") + m_data->m_addCreateParam;
} }
args.push_back("create"); args.push_back("create");
cmdstring += string(" ") + "create"; cmdstring += string(" ") + "create";
@ -335,54 +361,58 @@ bool Aspell::buildDict(Rcl::Db &db, string &reason)
bool keepStderr = false; bool keepStderr = false;
m_config->getConfParam("aspellKeepStderr", &keepStderr); m_config->getConfParam("aspellKeepStderr", &keepStderr);
if (!keepStderr) if (!keepStderr)
aspell.setStderr("/dev/null"); aspell.setStderr("/dev/null");
Rcl::TermIter *tit = db.termWalkOpen(); Rcl::TermIter *tit = db.termWalkOpen();
if (tit == 0) { if (tit == 0) {
reason = "termWalkOpen failed\n"; reason = "termWalkOpen failed\n";
return false; return false;
} }
string termbuf; string termbuf;
AspExecPv pv(&termbuf, tit, db); AspExecPv pv(&termbuf, tit, db);
aspell.setProvide(&pv); aspell.setProvide(&pv);
if (aspell.doexec(m_data->m_exec, args, &termbuf)) { if (aspell.doexec(m_data->m_exec, args, &termbuf)) {
ExecCmd cmd; ExecCmd cmd;
args.clear(); args.clear();
args.push_back("dicts"); args.push_back("dicts");
string dicts; string dicts;
bool hasdict = false; bool hasdict = false;
if (cmd.doexec(m_data->m_exec, args, 0, &dicts)) { if (cmd.doexec(m_data->m_exec, args, 0, &dicts)) {
vector<string> vdicts; vector<string> vdicts;
stringToTokens(dicts, vdicts, "\n\r\t "); stringToTokens(dicts, vdicts, "\n\r\t ");
if (find(vdicts.begin(), vdicts.end(), m_lang) != vdicts.end()) { if (find(vdicts.begin(), vdicts.end(), m_lang) != vdicts.end()) {
hasdict = true; hasdict = true;
} }
} }
if (hasdict) if (hasdict) {
reason = string( reason = string("\naspell dictionary creation command [") +
"\naspell dictionary creation command [") + cmdstring;
cmdstring + string("] failed. Reason unknown.\n" reason += string(
"Try to set aspellKeepStderr = 1 in recoll.conf, and execute \n" "] failed. Reason unknown.\n"
"the indexing command in a terminal to see the aspell " "Try to set aspellKeepStderr = 1 in recoll.conf, and execute \n"
"diagnostic output.\n"); "the indexing command in a terminal to see the aspell "
else "diagnostic output.\n");
reason = string("aspell dictionary creation command failed:\n") + } else {
reason = string("aspell dictionary creation command failed:\n") +
cmdstring + "\n" cmdstring + "\n"
"One possible reason might be missing language " "One possible reason might be missing language "
"data files for lang = " + m_lang + "data files for lang = " + m_lang +
". Maybe try to execute the command by hand for a better diag."; ". Maybe try to execute the command by hand for a better diag.";
return false; }
return false;
} }
db.termWalkClose(tit); db.termWalkClose(tit);
return true; return true;
} }
static const unsigned int ldatadiroptsz =
string("--local-data-dir=").size();
bool Aspell::make_speller(string& reason) bool Aspell::make_speller(string& reason)
{ {
if (!ok()) if (!ok())
return false; return false;
if (m_data->m_speller != 0) if (m_data->m_speller != 0)
return true; return true;
@ -393,14 +423,18 @@ bool Aspell::make_speller(string& reason)
aapi.aspell_config_replace(config, "encoding", "utf-8"); aapi.aspell_config_replace(config, "encoding", "utf-8");
aapi.aspell_config_replace(config, "master", dicPath().c_str()); aapi.aspell_config_replace(config, "master", dicPath().c_str());
aapi.aspell_config_replace(config, "sug-mode", "fast"); aapi.aspell_config_replace(config, "sug-mode", "fast");
aapi.aspell_config_replace(config, "data-dir", m_data->m_datadir.c_str());
aapi.aspell_config_replace(
config, "local-data-dir",
m_data->m_addCreateParam.substr(ldatadiroptsz).c_str());
// aapi.aspell_config_replace(config, "sug-edit-dist", "2"); // aapi.aspell_config_replace(config, "sug-edit-dist", "2");
ret = aapi.new_aspell_speller(config); ret = aapi.new_aspell_speller(config);
aapi.delete_aspell_config(config); aapi.delete_aspell_config(config);
if (aapi.aspell_error(ret) != 0) { if (aapi.aspell_error(ret) != 0) {
reason = aapi.aspell_error_message(ret); reason = aapi.aspell_error_message(ret);
aapi.delete_aspell_can_have_error(ret); aapi.delete_aspell_can_have_error(ret);
return false; return false;
} }
m_data->m_speller = aapi.to_aspell_speller(ret); m_data->m_speller = aapi.to_aspell_speller(ret);
return true; return true;
@ -417,17 +451,17 @@ bool Aspell::check(const string &iterm, string& reason)
return true; return true;
} }
if (!ok() || !make_speller(reason)) if (!ok() || !make_speller(reason))
return false; return false;
if (iterm.empty()) if (iterm.empty())
return true; //?? return true; //??
if (!o_index_stripchars) { if (!o_index_stripchars) {
string lower; string lower;
if (!unacmaybefold(mterm, lower, "UTF-8", UNACOP_FOLD)) { if (!unacmaybefold(mterm, lower, "UTF-8", UNACOP_FOLD)) {
LOGERR("Aspell::check: cant lowercase input\n"); LOGERR("Aspell::check: cant lowercase input\n");
return false; return false;
} }
mterm.swap(lower); mterm.swap(lower);
} }
int ret = aapi.aspell_speller_check(m_data->m_speller, int ret = aapi.aspell_speller_check(m_data->m_speller,
@ -449,7 +483,7 @@ bool Aspell::suggest(Rcl::Db &db, const string &_term,
{ {
LOGDEB("Aspell::suggest: term [" << _term << "]\n"); LOGDEB("Aspell::suggest: term [" << _term << "]\n");
if (!ok() || !make_speller(reason)) if (!ok() || !make_speller(reason))
return false; return false;
string mterm(_term); string mterm(_term);
if (mterm.empty()) if (mterm.empty())
return true; //?? return true; //??
@ -461,178 +495,36 @@ bool Aspell::suggest(Rcl::Db &db, const string &_term,
} }
if (!o_index_stripchars) { if (!o_index_stripchars) {
string lower; string lower;
if (!unacmaybefold(mterm, lower, "UTF-8", UNACOP_FOLD)) { if (!unacmaybefold(mterm, lower, "UTF-8", UNACOP_FOLD)) {
LOGERR("Aspell::check : cant lowercase input\n"); LOGERR("Aspell::check : cant lowercase input\n");
return false; return false;
} }
mterm.swap(lower); mterm.swap(lower);
} }
AspellCanHaveError *ret;
const AspellWordList *wl = const AspellWordList *wl =
aapi.aspell_speller_suggest(m_data->m_speller, aapi.aspell_speller_suggest(m_data->m_speller,
mterm.c_str(), mterm.length()); mterm.c_str(), mterm.length());
if (wl == 0) { if (wl == 0) {
reason = aapi.aspell_speller_error_message(m_data->m_speller); reason = aapi.aspell_speller_error_message(m_data->m_speller);
return false; return false;
} }
AspellStringEnumeration *els = aapi.aspell_word_list_elements(wl); AspellStringEnumeration *els = aapi.aspell_word_list_elements(wl);
const char *word; const char *word;
while ((word = aapi.aspell_string_enumeration_next(els)) != 0) { while ((word = aapi.aspell_string_enumeration_next(els)) != 0) {
LOGDEB0("Aspell::suggest: got [" << word << "]\n"); LOGDEB0("Aspell::suggest: got [" << word << "]\n");
// Check that the word exists in the index (we don't want // Check that the word exists in the index (we don't want
// aspell computed stuff, only exact terms from the // aspell computed stuff, only exact terms from the
// dictionary). We used to also check that it stems // dictionary). We used to also check that it stems
// differently from the base word but this is complicated // differently from the base word but this is complicated
// (stemming on/off + language), so we now leave this to the // (stemming on/off + language), so we now leave this to the
// caller. // caller.
if (db.termExists(word)) if (db.termExists(word))
suggestions.push_back(word); suggestions.push_back(word);
} }
aapi.delete_aspell_string_enumeration(els); aapi.delete_aspell_string_enumeration(els);
return true; return true;
} }
#endif // RCL_USE_ASPELL #endif // RCL_USE_ASPELL
#else // TEST_RCLASPELL test driver ->
#ifdef HAVE_CONFIG_H
#include "autoconfig.h"
#endif
#ifdef RCL_USE_ASPELL
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <iostream>
using namespace std;
#include "rclinit.h"
#include "rclconfig.h"
#include "rcldb.h"
#include "rclaspell.h"
static char *thisprog;
RclConfig *rclconfig;
static char usage [] =
" -b : build dictionary\n"
" -s <term>: suggestions for term\n"
" -c <term>: check term\n"
"\n"
;
static void
Usage(void)
{
fprintf(stderr, "%s: usage:\n%s", thisprog, usage);
exit(1);
}
static int op_flags;
#define OPT_MOINS 0x1
#define OPT_s 0x2
#define OPT_b 0x4
#define OPT_c 0x8
int main(int argc, char **argv)
{
string word;
thisprog = argv[0];
argc--; argv++;
while (argc > 0 && **argv == '-') {
(*argv)++;
if (!(**argv))
/* Cas du "adb - core" */
Usage();
while (**argv)
switch (*(*argv)++) {
case 'b': op_flags |= OPT_b; break;
case 'c': op_flags |= OPT_c; if (argc < 2) Usage();
word = *(++argv);
argc--;
goto b1;
case 's': op_flags |= OPT_s; if (argc < 2) Usage();
word = *(++argv);
argc--;
goto b1;
default: Usage(); break;
}
b1: argc--; argv++;
}
if (argc != 0 || op_flags == 0)
Usage();
string reason;
rclconfig = recollinit(0, 0, 0, reason);
if (!rclconfig || !rclconfig->ok()) {
fprintf(stderr, "Configuration problem: %s\n", reason.c_str());
exit(1);
}
string dbdir = rclconfig->getDbDir();
if (dbdir.empty()) {
fprintf(stderr, "No db directory in configuration");
exit(1);
}
Rcl::Db rcldb(rclconfig);
if (!rcldb.open(Rcl::Db::DbRO, 0)) {
fprintf(stderr, "Could not open database in %s\n", dbdir.c_str());
exit(1);
}
Aspell aspell(rclconfig);
if (!aspell.init(reason)) {
cerr << "Init failed: " << reason << endl;
exit(1);
}
if (op_flags & OPT_b) {
if (!aspell.buildDict(rcldb, reason)) {
cerr << "buildDict failed: " << reason << endl;
exit(1);
}
} else if (op_flags & OPT_c) {
bool ret = aspell.check(word, reason);
if (!ret && reason.size()) {
cerr << "Aspell error: " << reason << endl;
return 1;
}
cout << word;
if (ret) {
cout << " is in dictionary" << endl;
} else {
cout << " not in dictionary" << endl;
}
} else {
list<string> suggs;
if (!aspell.suggest(rcldb, word, suggs, reason)) {
cerr << "suggest failed: " << reason << endl;
exit(1);
}
cout << "Suggestions for " << word << ":" << endl;
for (list<string>::iterator it = suggs.begin();
it != suggs.end(); it++) {
cout << *it << endl;
}
}
exit(0);
}
#else
int main(int argc, char **argv)
{return 1;}
#endif // RCL_USE_ASPELL
#endif // TEST_RCLASPELL test driver

View File

@ -70,7 +70,7 @@ class Aspell {
std::string dicPath(); std::string dicPath();
const RclConfig *m_config; const RclConfig *m_config;
std::string m_lang; std::string m_lang;
AspellData *m_data; AspellData *m_data{nullptr};
bool make_speller(std::string& reason); bool make_speller(std::string& reason);
}; };

View File

@ -6,10 +6,10 @@ overriden in the c++ code by ifdefs _WIN32 anyway */
/* #undef AC_APPLE_UNIVERSAL_BUILD */ /* #undef AC_APPLE_UNIVERSAL_BUILD */
/* Path to the aspell api include file */ /* Path to the aspell api include file */
/* #undef ASPELL_INCLUDE "aspell-local.h" */ #define ASPELL_INCLUDE "aspell-local.h"
/* Path to the aspell program */ /* Aspell program parameter to findFilter(). */
/* #define ASPELL_PROG "/usr/bin/aspell" */ #define ASPELL_PROG "aspell-installed/mingw32/bin/aspell"
/* No X11 session monitoring support */ /* No X11 session monitoring support */
#define DISABLE_X11MON #define DISABLE_X11MON
@ -24,14 +24,16 @@ overriden in the c++ code by ifdefs _WIN32 anyway */
#define HAVE_CXX0X_UNORDERED 1 #define HAVE_CXX0X_UNORDERED 1
/* Define to 1 if you have the <dlfcn.h> header file. */ /* Define to 1 if you have the <dlfcn.h> header file. */
#define HAVE_DLFCN_H 1 #undef HAVE_DLFCN_H
#undef HAVE_DLOPEN
/* Define if you have the iconv() function and it works. */
#define HAVE_ICONV 1
/* Define to 1 if you have the <inttypes.h> header file. */ /* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1 #define HAVE_INTTYPES_H 1
/* Define to 1 if you have the `dl' library (-ldl). */
#define HAVE_LIBDL 1
/* Define to 1 if you have the `pthread' library (-lpthread). */ /* Define to 1 if you have the `pthread' library (-lpthread). */
#define HAVE_LIBPTHREAD 1 #define HAVE_LIBPTHREAD 1
@ -98,11 +100,16 @@ overriden in the c++ code by ifdefs _WIN32 anyway */
/* Define to 1 if you have the <unistd.h> header file. */ /* Define to 1 if you have the <unistd.h> header file. */
/* #undef HAVE_UNISTD_H */ /* #undef HAVE_UNISTD_H */
/* Define to 1 if you have the `vsnprintf' function. */
#undef HAVE_VSNPRINTF
/* Define as const if the declaration of iconv() needs const. */
#define ICONV_CONST
/* Use multiple threads for indexing */ /* Use multiple threads for indexing */
#define IDX_THREADS 1 #define IDX_THREADS 1
/* Define to the sub-directory in which libtool stores uninstalled libraries. /* Define to the sub-directory where libtool stores uninstalled libraries. */
*/
#define LT_OBJDIR ".libs/" #define LT_OBJDIR ".libs/"
/* Define to the address where bug reports for this package should be sent. */ /* Define to the address where bug reports for this package should be sent. */
@ -112,10 +119,10 @@ overriden in the c++ code by ifdefs _WIN32 anyway */
#define PACKAGE_NAME "Recoll" #define PACKAGE_NAME "Recoll"
/* Define to the full name and version of this package. */ /* Define to the full name and version of this package. */
#define PACKAGE_STRING "Recoll 1.25.21" #define PACKAGE_STRING "Recoll 1.26.0~pre4"
/* Define to the version of this package. */ /* Define to the version of this package. */
#define PACKAGE_VERSION "1.25.21" #define PACKAGE_VERSION "1.26.0~pre4"
/* Define to the one symbol short name of this package. */ /* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME "recoll" #define PACKAGE_TARNAME "recoll"
@ -126,9 +133,6 @@ overriden in the c++ code by ifdefs _WIN32 anyway */
/* putenv parameter is const */ /* putenv parameter is const */
/* #undef PUTENV_ARG_CONST */ /* #undef PUTENV_ARG_CONST */
/* Define as const if the declaration of iconv() needs const. */
#define ICONV_CONST
/* Real time monitoring option */ /* Real time monitoring option */
#undef RCL_MONITOR #undef RCL_MONITOR
@ -136,7 +140,7 @@ overriden in the c++ code by ifdefs _WIN32 anyway */
/* #undef RCL_SPLIT_CAMELCASE */ /* #undef RCL_SPLIT_CAMELCASE */
/* Compile the aspell interface */ /* Compile the aspell interface */
/* #undef RCL_USE_ASPELL */ #define RCL_USE_ASPELL 1
/* Compile the fam interface */ /* Compile the fam interface */
/* #undef RCL_USE_FAM */ /* #undef RCL_USE_FAM */

View File

@ -14,9 +14,7 @@
* Free Software Foundation, Inc., * Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/ */
#ifdef HAVE_CONFIG_H
#include "autoconfig.h" #include "autoconfig.h"
#endif
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>

146
src/testmains/traspell.cpp Normal file
View File

@ -0,0 +1,146 @@
/* Copyright (C) 2006-2019 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "autoconfig.h"
#ifdef RCL_USE_ASPELL
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <iostream>
using namespace std;
#include "rclinit.h"
#include "rclconfig.h"
#include "rcldb.h"
#include "rclaspell.h"
static char *thisprog;
RclConfig *rclconfig;
static char usage [] =
" -b : build dictionary\n"
" -s <term>: suggestions for term\n"
" -c <term>: check term\n"
"\n"
;
static void
Usage(void)
{
fprintf(stderr, "%s: usage:\n%s", thisprog, usage);
exit(1);
}
static int op_flags;
#define OPT_MOINS 0x1
#define OPT_s 0x2
#define OPT_b 0x4
#define OPT_c 0x8
int main(int argc, char **argv)
{
string word;
thisprog = argv[0];
argc--; argv++;
while (argc > 0 && **argv == '-') {
(*argv)++;
if (!(**argv))
/* Cas du "adb - core" */
Usage();
while (**argv)
switch (*(*argv)++) {
case 'b': op_flags |= OPT_b; break;
case 'c': op_flags |= OPT_c; if (argc < 2) Usage();
word = *(++argv);
argc--;
goto b1;
case 's': op_flags |= OPT_s; if (argc < 2) Usage();
word = *(++argv);
argc--;
goto b1;
default: Usage(); break;
}
b1: argc--; argv++;
}
if (argc != 0 || op_flags == 0)
Usage();
string reason;
rclconfig = recollinit(0, 0, 0, reason);
if (!rclconfig || !rclconfig->ok()) {
fprintf(stderr, "Configuration problem: %s\n", reason.c_str());
exit(1);
}
string dbdir = rclconfig->getDbDir();
if (dbdir.empty()) {
fprintf(stderr, "No db directory in configuration");
exit(1);
}
Rcl::Db rcldb(rclconfig);
if (!rcldb.open(Rcl::Db::DbRO, 0)) {
fprintf(stderr, "Could not open database in %s\n", dbdir.c_str());
exit(1);
}
Aspell aspell(rclconfig);
if (!aspell.init(reason)) {
cerr << "Init failed: " << reason << endl;
exit(1);
}
if (op_flags & OPT_b) {
if (!aspell.buildDict(rcldb, reason)) {
cerr << "buildDict failed: " << reason << endl;
exit(1);
}
} else if (op_flags & OPT_c) {
bool ret = aspell.check(word, reason);
if (!ret && reason.size()) {
cerr << "Aspell error: " << reason << endl;
return 1;
}
cout << word;
if (ret) {
cout << " is in dictionary" << endl;
} else {
cout << " not in dictionary" << endl;
}
} else {
list<string> suggs;
if (!aspell.suggest(rcldb, word, suggs, reason)) {
cerr << "suggest failed: " << reason << endl;
exit(1);
}
cout << "Suggestions for " << word << ":" << endl;
for (list<string>::iterator it = suggs.begin();
it != suggs.end(); it++) {
cout << *it << endl;
}
}
exit(0);
}
#endif // RCL_USE_ASPELL

82
src/utils/dlib.cpp Normal file
View File

@ -0,0 +1,82 @@
/* Copyright (C) 2017-2019 J.F.Dockes
*
* License: GPL 2.1
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifdef BUILDING_RECOLL
#include "autoconfig.h"
#else
#include "config.h"
#endif
#include "dlib.h"
#include "pathut.h"
#include "smallut.h"
#ifdef _WIN32
#include "safewindows.h"
#elif defined(HAVE_DLOPEN)
#include <dlfcn.h>
#else
#error dlib.cpp not ported on this system
#endif
void *dlib_open(const std::string& libname, int flags)
{
#ifdef _WIN32
return LoadLibraryA(libname.c_str());
#elif defined(HAVE_DLOPEN)
return dlopen(libname.c_str(), RTLD_LAZY);
#else
return nullptr;
#endif
}
void *dlib_sym(void *handle, const char *name)
{
#ifdef _WIN32
return (void *)::GetProcAddress((HMODULE)handle, name);
#elif defined(HAVE_DLOPEN)
return dlsym(handle, name);
#else
return nullptr;
#endif
}
void dlib_close(void *handle)
{
#ifdef _WIN32
::FreeLibrary((HMODULE)handle);
#elif defined(HAVE_DLOPEN)
dlclose(handle);
#endif
}
const char *dlib_error()
{
#ifdef _WIN32
int error = GetLastError();
static std::string errorstring;
errorstring = std::string("dlopen/dlsym error: ") + lltodecstr(error);
return errorstring.c_str();
#elif defined(HAVE_DLOPEN)
return dlerror();
#else
return "??? dlib not ported";
#endif
}

32
src/utils/dlib.h Normal file
View File

@ -0,0 +1,32 @@
/* Copyright (C) 2017-2019 J.F.Dockes
*
* License: GPL 2.1
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef _DLIB_H_INCLUDED_
#define _DLIB_H_INCLUDED_
/** Dynamic library functions */
#include <string>
extern void *dlib_open(const std::string& libname, int flags = 0);
extern void *dlib_sym(void *handle, const char *name);
extern void dlib_close(void *handle);
extern const char *dlib_error();
#endif /* _DLIB_H_INCLUDED_ */

View File

@ -46,12 +46,17 @@ LIBREVENGE=${RCLDEPS}libwpd/librevenge-0.0.1.jfd/
CHM=${RCLDEPS}pychm CHM=${RCLDEPS}pychm
MISC=${RCLDEPS}misc MISC=${RCLDEPS}misc
LIBPFF=${RCLDEPS}pffinstall LIBPFF=${RCLDEPS}pffinstall
ASPELL=${RCLDEPS}/aspell-0.60.7/aspell-installed
# Where to copy the Qt Dlls from: # Where to copy the Qt Dlls from:
QTBIN=C:/Qt/Qt5.8.0/5.8/mingw53_32/bin QTBIN=C:/Qt/Qt5.8.0/5.8/mingw53_32/bin
QTGCCBIN=C:/qt/Qt5.8.0/Tools/mingw530_32/bin/ QTGCCBIN=C:/qt/Qt5.8.0/Tools/mingw530_32/bin/
# Where to find libgcc_s_dw2-1.dll for progs which need it copied
MINGWBIN=$QTBIN # Where to find libgcc_s_dw2-1.dll et all for progs compiled with c:/MinGW
# (as opposed to the mingw bundled with qt
MINGWBIN=C:/MinGW/bin
PATH=$MINGWBIN:$QTGCCBIN:$PATH PATH=$MINGWBIN:$QTGCCBIN:$PATH
export PATH export PATH
@ -158,7 +163,7 @@ copyrecoll()
chkcp $RCL/python/recoll/recoll/rclconfig.py $FILTERS chkcp $RCL/python/recoll/recoll/rclconfig.py $FILTERS
chkcp $RCL/python/recoll/recoll/conftree.py $FILTERS chkcp $RCL/python/recoll/recoll/conftree.py $FILTERS
rm -f $FILTERS/rclimg rm -f $FILTERS/rclimg*
chkcp $RCL/filters/* $FILTERS chkcp $RCL/filters/* $FILTERS
rm -f $FILTERS/rclimg $FILTERS/rclimg.py rm -f $FILTERS/rclimg $FILTERS/rclimg.py
chkcp $RCLDEPS/rclimg/rclimg.exe $FILTERS chkcp $RCLDEPS/rclimg/rclimg.exe $FILTERS
@ -273,6 +278,17 @@ copypff()
chkcp $QTBIN/libwinpthread-1.dll $DEST chkcp $QTBIN/libwinpthread-1.dll $DEST
} }
copyaspell()
{
DEST=$FILTERS
cp -rp $ASPELL $DEST || fatal "can't copy $ASPELL"
DEST=$DEST/aspell-installed/mingw32/bin
# Check that we do have an aspell.exe.
chkcp $ASPELL/mingw32/bin/aspell.exe $DEST
chkcp $MINGWBIN/libgcc_s_dw2-1.dll $DEST
chkcp $MINGWBIN/libstdc++-6.dll $DEST
}
for d in doc examples filters images translations; do for d in doc examples filters images translations; do
test -d $DESTDIR/Share/$d || mkdir -p $DESTDIR/Share/$d || \ test -d $DESTDIR/Share/$d || mkdir -p $DESTDIR/Share/$d || \
fatal mkdir $d failed fatal mkdir $d failed
@ -290,6 +306,7 @@ test "$VERSION" = "$CFVERS" ||
echo Packaging version $CFVERS echo Packaging version $CFVERS
copyaspell
# copyrecoll must stay before copyqt so that windeployqt can do its thing # copyrecoll must stay before copyqt so that windeployqt can do its thing
copyrecoll copyrecoll
copyqt copyqt

View File

@ -96,6 +96,7 @@ SOURCES += \
../../utils/conftree.cpp \ ../../utils/conftree.cpp \
../../utils/copyfile.cpp \ ../../utils/copyfile.cpp \
../../utils/cpuconf.cpp \ ../../utils/cpuconf.cpp \
../../utils/dlib.cpp \
../../utils/ecrontab.cpp \ ../../utils/ecrontab.cpp \
../../utils/utf8iter.cpp \ ../../utils/utf8iter.cpp \
../../utils/zlibut.cpp \ ../../utils/zlibut.cpp \

View File

@ -2,7 +2,7 @@
; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES! ; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES!
#define MyAppName "Recoll" #define MyAppName "Recoll"
#define MyAppVersion "1.25.0-20190125-540140bd" #define MyAppVersion "1.26.0-pre4-20191011-2491388e"
#define MyAppPublisher "Recoll.org" #define MyAppPublisher "Recoll.org"
#define MyAppURL "http://www.recoll.org" #define MyAppURL "http://www.recoll.org"
#define MyAppExeName "recoll.exe" #define MyAppExeName "recoll.exe"