Windows: deal with non-ASCII user login, non-ascii paths in confdir etc.

src/common/rclconfig.cpp

@@ -36,6 +36,7 @@
 #include <list>
 #include <iostream>
 #include <sstream>
+#include <fstream>
 #include <cstdlib>
 #include <cstring>
 #include <unordered_map>
@@ -936,12 +937,9 @@ bool RclConfig::getMissingHelperDesc(string& out) const
 void RclConfig::storeMissingHelperDesc(const string &s)
 {
     string fmiss = path_cat(getCacheDir(), "missing");
-    FILE *fp = fopen(fmiss.c_str(), "w");
+    fstream fp = path_open(fmiss, ios::trunc | ios::out);
-    if (fp) {
+    if (fp.is_open()) {
-        if (s.size() > 0 && fwrite(s.c_str(), s.size(), 1, fp) != 1) {
+        fp << s;
-            LOGERR("storeMissingHelperDesc: fwrite failed\n");
-        }
-        fclose(fp);
     }
 }
 
@@ -1340,9 +1338,23 @@ string RclConfig::getCachedirPath(const char *varname, const char *dflt) const
     return path_canon(result);
 }
 
+// On Windows, try to translate a possibly non-ascii path into the
+// shortpath alias. We first create the target, as this only works if
+// it exists. Used for xapiandb and aspell as these can't handle
+// Unicode paths.
+static string maybeshortpath(const std::string& in)
+{
+#ifdef _WIN32
+    path_makepath(in, 0700);
+    return path_shortpath(in);
+#else
+    return in;
+#endif
+}
+
 string RclConfig::getDbDir() const
 {
-    return getCachedirPath("dbdir", "xapiandb");
+    return maybeshortpath(getCachedirPath("dbdir", "xapiandb"));
 }
 string RclConfig::getWebcacheDir() const
 {
@@ -1354,7 +1366,7 @@ string RclConfig::getMboxcacheDir() const
 }
 string RclConfig::getAspellcacheDir() const
 {
-    return getCachedirPath("aspellDicDir", "");
+    return maybeshortpath(getCachedirPath("aspellDicDir", ""));
 }
 
 string RclConfig::getStopfile() const
@@ -1730,31 +1742,28 @@ bool RclConfig::initUserConfig()
 
     // Use protective 700 mode to create the top configuration
     // directory: documents can be reconstructed from index data.
-    if (!path_exists(m_confdir) && 
+    if (!path_exists(m_confdir) && !path_makepath(m_confdir, 0700)) {
-        mkdir(m_confdir.c_str(), 0700) < 0) {
+        m_reason += string("mkdir(") + m_confdir + ") failed: "+strerror(errno);
-        m_reason += string("mkdir(") + m_confdir + ") failed: " + 
-            strerror(errno);
         return false;
     }
     string lang = localelang();
     for (int i = 0; i < ncffiles; i++) {
         string dst = path_cat(m_confdir, string(configfiles[i])); 
         if (!path_exists(dst)) {
-            FILE *fp = fopen(dst.c_str(), "w");
+            fstream output = path_open(dst, ios::out);
-            if (fp) {
+            if (output.is_open()) {
-                fprintf(fp, "%s\n", blurb);
+                output << blurb << "\n";
                 if (!strcmp(configfiles[i], "recoll.conf")) {
                     // Add improved unac_except_trans for some languages
                     if (lang == "se" || lang == "dk" || lang == "no" || 
                         lang == "fi") {
-                        fprintf(fp, "%s\n", swedish_ex);
+                        output << swedish_ex << "\n";
                     } else if (lang == "de") {
-                        fprintf(fp, "%s\n", german_ex);
+                        output << german_ex << "\n";
                     }
                 }
-                fclose(fp);
             } else {
-                m_reason += string("fopen ") + dst + ": " + strerror(errno);
+                m_reason += string("open ") + dst + ": " + strerror(errno);
                 return false;
             }
         }

src/qtgui/rclmain_w.cpp

@@ -18,6 +18,7 @@
 
 #include <utility>
 #include <memory>
+#include <fstream>
 #include <stdlib.h>
 
 #include <qapplication.h>
@@ -1071,16 +1072,15 @@ void RclMain::exportSimpleSearchHistory()
     }
     string path = qs2utf8s(dialog.selectedFiles().value(0));
     LOGDEB("Chosen path: " << path << "\n");
-    FILE *fp = fopen(path.c_str(), "wb");
+    std::fstream fp = path_open(path, std::ios::out | std::ios::trunc);
-    if (fp == 0) {
+    if (!fp.is_open()) {
-        QMessageBox::warning(0, "Recoll", 
+        QMessageBox::warning(0, "Recoll", tr("Could not open/create file"));
-                             tr("Could not open/create file"));
         return;
     }
     for (int i = 0; i < prefs.ssearchHistory.count(); i++) {
-        fprintf(fp, "%s\n", qs2utf8s(prefs.ssearchHistory[i]).c_str());
+        fp << qs2utf8s(prefs.ssearchHistory[i]) << "\n";
     }
-    fclose(fp);
+    fp.close();
 }
 
 // Called when the uiprefs dialog is ok'd

src/qtgui/restable.cpp

@@ -24,6 +24,7 @@
 
 #include <algorithm>
 #include <memory>
+#include <fstream>
 
 #include <Qt>
 #include <QShortcut>
@@ -418,7 +419,7 @@ QVariant RecollModel::data(const QModelIndex& index, int role) const
     return QString::fromUtf8(lr.front().c_str());
 }
 
-void RecollModel::saveAsCSV(FILE *fp)
+void RecollModel::saveAsCSV(std::fstream& fp)
 {
     if (!m_source)
         return;
@@ -433,7 +434,7 @@ void RecollModel::saveAsCSV(FILE *fp)
     }
     string csv;
     stringsToCSV(tokens, csv);
-    fprintf(fp, "%s\n", csv.c_str());
+    fp << csv << "\n";
     tokens.clear();
 
     for (int row = 0; row < rows; row++) {
@@ -445,7 +446,7 @@ void RecollModel::saveAsCSV(FILE *fp)
             tokens.push_back(m_getters[col](m_fields[col], doc));
         }
         stringsToCSV(tokens, csv);
-        fprintf(fp, "%s\n", csv.c_str());
+        fp << csv << "\n";
         tokens.clear();
     }
 }
@@ -785,22 +786,19 @@ void ResTable::saveAsCSV()
     LOGDEB("ResTable::saveAsCSV\n");
     if (!m_model)
         return;
-    QString s = 
+    QString s = QFileDialog::getSaveFileName(
-        QFileDialog::getSaveFileName(this, //parent
+        this, tr("Save table to CSV file"), path2qs(path_home()));
-                                     tr("Save table to CSV file"),
-                                     QString::fromLocal8Bit(path_home().c_str())
-            );
     if (s.isEmpty())
         return;
-    const char *tofile = s.toLocal8Bit();
+    std::string tofile = qs2path(s);
-    FILE *fp = fopen(tofile, "w");
+    std::fstream fp = path_open(tofile, std::ios::out|std::ios::trunc);
-    if (fp == 0) {
+    if (!fp.is_open()) {
         QMessageBox::warning(0, "Recoll", 
                              tr("Can't open/create file: ") + s);
         return;
     }
     m_model->saveAsCSV(fp);
-    fclose(fp);
+    fp.close();
 }
 
 // This is called when the sort order is changed from another widget

src/qtgui/restable.h

@@ -24,6 +24,7 @@
 #include <map>
 #include <vector>
 #include <memory>
+#include <fstream>
 
 #include "ui_restable.h"
 #include "docseq.h"
@@ -47,7 +48,7 @@ public:
 				 int role = Qt::DisplayRole ) const;
     virtual QVariant data(const QModelIndex& index, 
 			   int role = Qt::DisplayRole ) const;
-    virtual void saveAsCSV(FILE *fp);
+    virtual void saveAsCSV(std::fstream& fp);
     virtual void sort(int column, Qt::SortOrder order = Qt::AscendingOrder);
     // Specific methods
     virtual void readDocSource();

src/qtgui/winschedtool.cpp

@@ -20,6 +20,7 @@
 
 #include <stdio.h>
 #include <string>
+#include <fstream>
 
 #include <QPushButton>
 #include <QMessageBox>
@@ -42,18 +43,21 @@ void WinSchedToolW::init()
 
     connect(startPB, SIGNAL(clicked()), this, SLOT(startWinScheduler()));
 
-    // thisexecpath returns the directory
+    // Use a short path on Windows if possible to avoid issues with
+    // accented characters
+    string confdir = path_shortpath(theconfig->getConfDir());
+    
+    // path_thisexecpath() returns the directory
     string recollindex = path_cat(path_thisexecpath(), "recollindex.exe");
     LOGDEB("WinSchedTool: recollindex: " << recollindex << endl);
 
-    string batchfile = path_cat(theconfig->getConfDir(), "winsched.bat");
+    string batchfile = path_cat(confdir, "winsched.bat");
     LOGDEB("WinSchedTool: batch file " << batchfile << endl);
 
     if (!path_exists(batchfile)) {
-        FILE *fp = fopen(batchfile.c_str(), "w");
+        std::fstream fp = path_open(batchfile, ios::out|ios::trunc);
-        fprintf(fp, "\"%s\" -c \"%s\"\n", recollindex.c_str(),
+        fp << "\"" << recollindex << "\" -c \"" << confdir << "\"\n";
-                theconfig->getConfDir().c_str());
+        fp.close();
-        fclose(fp);
     }
     QString blurb = tr(
         "<h3>Recoll indexing batch scheduling</h3>"

src/rcldb/rcldb.cpp

@@ -28,6 +28,7 @@
 #include <algorithm>
 #include <sstream>
 #include <iostream>
+#include <fstream>
 
 using namespace std;
 
@@ -283,12 +284,12 @@ void Db::Native::openWrite(const string& dir, Db::OpenMode mode)
             // Force Chert format, don't store the text.
             string stub = path_cat(m_rcldb->m_config->getConfDir(),
                                    "xapian.stub");
-            FILE *fp = fopen(stub.c_str(), "w");
+            std::fstream fp = path_open(stub, std::ios::out|std::ios::trunc);
-            if (nullptr == fp) {
+            if (!fp.is_open()) {
                 throw(string("Can't create ") + stub);
             }
-            fprintf(fp, "chert %s\n", dir.c_str());
+            fp << "chert " << dir << "\n";
-            fclose(fp);
+            fp.close();
             xwdb = Xapian::WritableDatabase(stub, action);
             m_storetext = false;
         }

src/utils/conftree.cpp

@@ -235,31 +235,27 @@ ConfSimple::ConfSimple(const char *fname, int readonly, bool tildexp,
       m_fmtime(0), m_holdWrites(false)
 {
     status = readonly ? STATUS_RO : STATUS_RW;
+    int mode = readonly ? ios::in : ios::in | ios::out;
+    if (!readonly && !path_exists(fname)) {
+        mode |= ios::trunc;
+    }
+    fstream input = path_open(fname, mode);
+    if (!input.is_open()) {
+        LOGERR("ConfSimple::ConfSimple: fstream(w)("<<fname<<", "<< mode <<
+               ") errno" << errno << "\n");
+    }
 
-    ifstream input;
+    if (!readonly && !input.is_open()) {
-    if (readonly) {
+        // reset errors
-        input.open(fname, ios::in);
+        input.clear();
-    } else {
+        status = STATUS_RO;
-        ios::openmode mode = ios::in | ios::out;
+        // open readonly
-        // It seems that there is no separate 'create if not exists'
+        input = path_open(fname, ios::in);
-        // open flag. Have to truncate to create, but dont want to do
-        // this to an existing file !
-        if (!path_exists(fname)) {
-            mode |= ios::trunc;
-        }
-        input.open(fname, mode);
-        if (input.is_open()) {
-            status = STATUS_RW;
-        } else {
-            input.clear();
-            input.open(fname, ios::in);
-            if (input.is_open()) {
-                status = STATUS_RO;
-            }
-        }
     }
 
     if (!input.is_open()) {
+        LOGERR("ConfSimple::ConfSimple: fstream("<<fname<<", "<<ios::in<<
+               ") errno" << errno << "\n");
         status = STATUS_ERROR;
         return;
     }
@@ -581,7 +577,16 @@ bool ConfSimple::write()
         return true;
     }
     if (m_filename.length()) {
-        ofstream output(m_filename.c_str(), ios::out | ios::trunc);
+        fstream output;
+#if (defined(BUILDING_RECOLL) && defined(_MSC_VER))
+        // msc has support for using wide chars for opening files. We may
+        // need this if, e.g. the user name/home directory is not ascii
+        wchar_t wfname[MAX_PATH + 1];
+        utf8towchar(m_filename, wfname, MAX_PATH);
+        output = fstream(wfname, ios::out | ios::trunc);
+#else
+        output = fstream(m_filename, ios::out | ios::trunc);
+#endif
         if (!output.is_open()) {
             return 0;
         }

src/utils/pathut.cpp

@@ -49,13 +49,16 @@
 #include <math.h>
 #include <errno.h>
 #include <dirent.h>
+#include <fstream>
 
 #ifdef _WIN32
+
 #include "safefcntl.h"
 #include "safeunistd.h"
 #include "safewindows.h"
 #include "safesysstat.h"
 #include "transcode.h"
+#include "log.h"
 
 #define STAT _wstati64
 #define LSTAT _wstati64
@@ -66,8 +69,12 @@
 #define READDIR _wreaddir
 #define DIRENT _wdirent
 #define DIRHDL _WDIR
+#define MKDIR(a,b) _wmkdir(a)
+#define OPEN ::_wopen
+#define UNLINK _wunlink
+
+#else /* !_WIN32 -> */
 
-#else // Not windows ->
 #include <fcntl.h>
 #include <unistd.h>
 #include <sys/param.h>
@@ -86,7 +93,12 @@
 #define READDIR readdir
 #define DIRENT dirent
 #define DIRHDL DIR
-#endif
+#define MKDIR(a,b) mkdir(a,b)
+#define O_BINARY 0
+#define OPEN ::open
+#define UNLINK unlink
+
+#endif /* !_WIN32 */
 
 #include <cstdlib>
 #include <cstring>
@@ -100,9 +112,11 @@
 #include "pathut.h"
 #include "smallut.h"
 
+
 using namespace std;
 
 #ifdef _WIN32
+#include <shlobj_core.h>
 /// Convert \ separators to /
 void path_slashize(string& s)
 {
@@ -276,7 +290,25 @@ flock (int fd, int operation)
     return 0;
 }
 
-#endif // Win32 only section
+std::string path_shortpath(const std::string& path)
+{
+    SYSPATH(path, syspath);
+    wchar_t wspath[MAX_PATH];
+    int ret = GetShortPathNameW(syspath, wspath, MAX_PATH);
+    if (ret == 0) {
+        LOGERR("GetShortPathNameW failed for [" << path << "]\n");
+        return path;
+    } else if (ret >= MAX_PATH) {
+        LOGERR("GetShortPathNameW [" << path << "] too long " <<
+               path.size() << " MAX_PATH " << MAX_PATH << "\n");
+        return path;
+    }
+    string shortpath;
+    wchartoutf8(wspath, shortpath);
+    return shortpath;
+}
+
+#endif /* _WIN32 */
 
 bool fsocc(const string& path, int *pc, long long *avmbs)
 {
@@ -284,8 +316,8 @@ bool fsocc(const string& path, int *pc, long long *avmbs)
 #ifdef _WIN32
     ULARGE_INTEGER freebytesavail;
     ULARGE_INTEGER totalbytes;
-    if (!GetDiskFreeSpaceExA(path.c_str(), &freebytesavail,
+    SYSPATH(path, syspath);
-                             &totalbytes, NULL)) {
+    if (!GetDiskFreeSpaceExW(syspath, &freebytesavail, &totalbytes, NULL)) {
         return false;
     }
     if (pc) {
@@ -295,7 +327,7 @@ bool fsocc(const string& path, int *pc, long long *avmbs)
         *avmbs = int(totalbytes.QuadPart / FSOCC_MB);
     }
     return true;
-#else // not windows ->
+#else /* !_WIN32 */
 
     struct statvfs buf;
     if (statvfs(path.c_str(), &buf) != 0) {
@@ -323,7 +355,7 @@ bool fsocc(const string& path, int *pc, long long *avmbs)
         }
     }
     return true;
-#endif
+#endif /* !_WIN32 */
 }
 
 
@@ -433,21 +465,30 @@ string path_home()
 {
 #ifdef _WIN32
     string dir;
-    const char *cp = getenv("USERPROFILE");
+    // Using wgetenv does not work well, depending on the
+    // environment I get wrong values for the accented chars (works
+    // with recollindex started from msys command window, does not
+    // work when started from recoll. SHGet... fixes this
+    //const wchar_t *cp = _wgetenv(L"USERPROFILE");
+    wchar_t *cp;
+    SHGetKnownFolderPath(FOLDERID_Profile, 0, nullptr, &cp);
     if (cp != 0) {
-        dir = cp;
+        wchartoutf8(cp, dir);
     }
     if (dir.empty()) {
-        cp = getenv("HOMEDRIVE");
+        cp = _wgetenv(L"HOMEDRIVE");
+        wchartoutf8(cp, dir);
         if (cp != 0) {
-            const char *cp1 = getenv("HOMEPATH");
+            string dir1;
+            const wchar_t *cp1 = _wgetenv(L"HOMEPATH");
+            wchartoutf8(cp1, dir1);
             if (cp1 != 0) {
-                dir = string(cp) + string(cp1);
+                dir = path_cat(dir, dir1);
             }
         }
     }
     if (dir.empty()) {
-        dir = "C:\\";
+        dir = "C:/";
     }
     dir = path_canon(dir);
     path_catslash(dir);
@@ -475,12 +516,15 @@ string path_home()
 string path_homedata()
 {
 #ifdef _WIN32
-    const char *cp = getenv("LOCALAPPDATA");
+    wchar_t *cp;
+    SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, nullptr, &cp);
     string dir;
     if (cp != 0) {
-        dir = path_canon(cp);
+        wchartoutf8(cp, dir);
     }
-    if (dir.empty()) {
+    if (!dir.empty()) {
+        dir = path_canon(dir);
+    } else {
         dir = path_cat(path_home(), "AppData/Local/");
     }
     return dir;
@@ -673,8 +717,11 @@ bool path_makepath(const string& ipath, int mode)
         path += elem;
         // Not using path_isdir() here, because this cant grok symlinks
         // If we hit an existing file, no worry, mkdir will just fail.
-        if (access(path.c_str(), 0) != 0) {
+        LOGDEB1("path_makepath: testing existence: ["  << path << "]\n");
-            if (mkdir(path.c_str(), mode) != 0)  {
+        if (!path_exists(path)) {
+            LOGDEB1("path_makepath: creating directory ["  << path << "]\n");
+            SYSPATH(path, syspath);
+            if (MKDIR(syspath, mode) != 0)  {
                 //cerr << "mkdir " << path << " failed, errno " << errno << endl;
                 return false;
             }
@@ -684,6 +731,24 @@ bool path_makepath(const string& ipath, int mode)
     return true;
 }
 
+std::fstream path_open(const std::string& path, int mode)
+{
+#if defined(_WIN32) && defined (_MSC_VER)
+    // MSC STL has support for using wide chars in fstream
+    // constructor. We need this if, e.g. the user name/home directory
+    // is not ASCII. Actually don't know how to do this with gcc
+    wchar_t wpath[MAX_PATH + 1];
+    utf8towchar(path, wpath, MAX_PATH);
+    std::fstream ret(wpath, mode);
+    if (!ret.is_open()) {
+        LOGERR("path_open("<< path << ", "<< mode <<") errno " << errno <<"\n");
+    }
+    return ret;
+#else
+    return std::fstream(path, mode);
+#endif
+}
+
 bool path_isdir(const string& path, bool follow)
 {
     struct STATBUF st;
@@ -1098,7 +1163,8 @@ Pidfile::~Pidfile()
 
 int Pidfile::read_pid()
 {
-    int fd = ::open(m_path.c_str(), O_RDONLY);
+    SYSPATH(m_path, syspath);
+    int fd = OPEN(syspath, O_RDONLY);
     if (fd == -1) {
         return -1;
     }
@@ -1120,8 +1186,8 @@ int Pidfile::read_pid()
 
 int Pidfile::flopen()
 {
-    const char *path = m_path.c_str();
+    SYSPATH(m_path, syspath);
-    if ((m_fd = ::open(path, O_RDWR | O_CREAT, 0644)) == -1) {
+    if ((m_fd = OPEN(syspath, O_RDWR | O_CREAT, 0644)) == -1) {
         m_reason = "Open failed: [" + m_path + "]: " + strerror(errno);
         return -1;
     }
@@ -1198,7 +1264,8 @@ int Pidfile::close()
 
 int Pidfile::remove()
 {
-    return unlink(m_path.c_str());
+    SYSPATH(m_path, syspath);
+    return UNLINK(syspath);
 }
 
 // Call funcs that need static init (not initially reentrant)

src/utils/pathut.h

@@ -21,6 +21,7 @@
 #include <vector>
 #include <set>
 #include <cstdint>
+#include <fstream>
 
 // Must be called in main thread before starting other threads
 extern void pathut_init_mt();
@@ -129,6 +130,22 @@ bool fsocc(const std::string& path, int *pc, long long *avmbs = 0);
 /// mkdir -p
 extern bool path_makepath(const std::string& path, int mode);
 
+/* Open file, trying to do the right thing with non-ASCII paths on
+ * Windows, where it only works with MSVC at the moment if the path is
+ * not ASCII, because it uses fstream(wchar_t*), which is an MSVC
+ * extension. On other OSes, just builds the fstream.  We'd need to
+ * find a way to make this work with g++. It would be easier in this
+ * case to use a FILE (_openw(), then fdopen()), but conftree really
+ * depends on std::iostream. One possible workaround for g++ would be
+ * to use shortpaths (which we already use to pass file names to
+ * xapian and aspell). Most of the problems are caused by the home
+ * directory name being non-ASCII, so returning a short path in
+ * path_home() would probably solve everything (but not pretty).
+ *
+ * @param path an utf-8 file path.
+ * @param mode is an std::fstream mode (ios::in etc.) */
+extern std::fstream path_open(const std::string& path, int mode);
+
 /// Where we create the user data subdirs
 extern std::string path_homedata();
 /// Test if path is absolute
@@ -164,6 +181,9 @@ public:
 /// Convert \ separators to /
 void path_slashize(std::string& s);
 void path_backslashize(std::string& s);
+extern std::string path_shortpath(const std::string& path);
+#else
+#define path_shortpath(path) (path)
 #endif
 
 /// Lock/pid file class. This is quite close to the pidfile_xxx

src/utils/rclutil.cpp

@@ -118,6 +118,10 @@ static const string& path_wingetrcltmpdir()
                 LOGSYSERR("path_wingettempfilename", "path_makepath", tdir);
             }
         }
+        // Try to use a short path to avoid issues in case the user
+        // login name is not ascii, especially with command line
+        // parameter passing
+        tdir = path_shortpath(tdir);
     }
     return tdir;
 }