Windows: deal with non-ASCII user login, non-ascii paths in confdir etc.

src/utils/conftree.cpp

@@ -235,31 +235,27 @@ ConfSimple::ConfSimple(const char *fname, int readonly, bool tildexp,
       m_fmtime(0), m_holdWrites(false)
 {
     status = readonly ? STATUS_RO : STATUS_RW;
+    int mode = readonly ? ios::in : ios::in | ios::out;
+    if (!readonly && !path_exists(fname)) {
+        mode |= ios::trunc;
+    }
+    fstream input = path_open(fname, mode);
+    if (!input.is_open()) {
+        LOGERR("ConfSimple::ConfSimple: fstream(w)("<<fname<<", "<< mode <<
+               ") errno" << errno << "\n");
+    }
 
-    ifstream input;
-    if (readonly) {
-        input.open(fname, ios::in);
-    } else {
-        ios::openmode mode = ios::in | ios::out;
-        // It seems that there is no separate 'create if not exists'
-        // open flag. Have to truncate to create, but dont want to do
-        // this to an existing file !
-        if (!path_exists(fname)) {
-            mode |= ios::trunc;
-        }
-        input.open(fname, mode);
-        if (input.is_open()) {
-            status = STATUS_RW;
-        } else {
-            input.clear();
-            input.open(fname, ios::in);
-            if (input.is_open()) {
-                status = STATUS_RO;
-            }
-        }
+    if (!readonly && !input.is_open()) {
+        // reset errors
+        input.clear();
+        status = STATUS_RO;
+        // open readonly
+        input = path_open(fname, ios::in);
     }
 
     if (!input.is_open()) {
+        LOGERR("ConfSimple::ConfSimple: fstream("<<fname<<", "<<ios::in<<
+               ") errno" << errno << "\n");
         status = STATUS_ERROR;
         return;
     }
@@ -581,7 +577,16 @@ bool ConfSimple::write()
         return true;
     }
     if (m_filename.length()) {
-        ofstream output(m_filename.c_str(), ios::out | ios::trunc);
+        fstream output;
+#if (defined(BUILDING_RECOLL) && defined(_MSC_VER))
+        // msc has support for using wide chars for opening files. We may
+        // need this if, e.g. the user name/home directory is not ascii
+        wchar_t wfname[MAX_PATH + 1];
+        utf8towchar(m_filename, wfname, MAX_PATH);
+        output = fstream(wfname, ios::out | ios::trunc);
+#else
+        output = fstream(m_filename, ios::out | ios::trunc);
+#endif
         if (!output.is_open()) {
             return 0;
         }

src/utils/pathut.cpp

@@ -49,13 +49,16 @@
 #include <math.h>
 #include <errno.h>
 #include <dirent.h>
+#include <fstream>
 
 #ifdef _WIN32
+
 #include "safefcntl.h"
 #include "safeunistd.h"
 #include "safewindows.h"
 #include "safesysstat.h"
 #include "transcode.h"
+#include "log.h"
 
 #define STAT _wstati64
 #define LSTAT _wstati64
@@ -66,8 +69,12 @@
 #define READDIR _wreaddir
 #define DIRENT _wdirent
 #define DIRHDL _WDIR
+#define MKDIR(a,b) _wmkdir(a)
+#define OPEN ::_wopen
+#define UNLINK _wunlink
+
+#else /* !_WIN32 -> */
 
-#else // Not windows ->
 #include <fcntl.h>
 #include <unistd.h>
 #include <sys/param.h>
@@ -86,7 +93,12 @@
 #define READDIR readdir
 #define DIRENT dirent
 #define DIRHDL DIR
-#endif
+#define MKDIR(a,b) mkdir(a,b)
+#define O_BINARY 0
+#define OPEN ::open
+#define UNLINK unlink
+
+#endif /* !_WIN32 */
 
 #include <cstdlib>
 #include <cstring>
@@ -100,9 +112,11 @@
 #include "pathut.h"
 #include "smallut.h"
 
+
 using namespace std;
 
 #ifdef _WIN32
+#include <shlobj_core.h>
 /// Convert \ separators to /
 void path_slashize(string& s)
 {
@@ -276,7 +290,25 @@ flock (int fd, int operation)
     return 0;
 }
 
-#endif // Win32 only section
+std::string path_shortpath(const std::string& path)
+{
+    SYSPATH(path, syspath);
+    wchar_t wspath[MAX_PATH];
+    int ret = GetShortPathNameW(syspath, wspath, MAX_PATH);
+    if (ret == 0) {
+        LOGERR("GetShortPathNameW failed for [" << path << "]\n");
+        return path;
+    } else if (ret >= MAX_PATH) {
+        LOGERR("GetShortPathNameW [" << path << "] too long " <<
+               path.size() << " MAX_PATH " << MAX_PATH << "\n");
+        return path;
+    }
+    string shortpath;
+    wchartoutf8(wspath, shortpath);
+    return shortpath;
+}
+
+#endif /* _WIN32 */
 
 bool fsocc(const string& path, int *pc, long long *avmbs)
 {
@@ -284,8 +316,8 @@ bool fsocc(const string& path, int *pc, long long *avmbs)
 #ifdef _WIN32
     ULARGE_INTEGER freebytesavail;
     ULARGE_INTEGER totalbytes;
-    if (!GetDiskFreeSpaceExA(path.c_str(), &freebytesavail,
-                             &totalbytes, NULL)) {
+    SYSPATH(path, syspath);
+    if (!GetDiskFreeSpaceExW(syspath, &freebytesavail, &totalbytes, NULL)) {
         return false;
     }
     if (pc) {
@@ -295,7 +327,7 @@ bool fsocc(const string& path, int *pc, long long *avmbs)
         *avmbs = int(totalbytes.QuadPart / FSOCC_MB);
     }
     return true;
-#else // not windows ->
+#else /* !_WIN32 */
 
     struct statvfs buf;
     if (statvfs(path.c_str(), &buf) != 0) {
@@ -323,7 +355,7 @@ bool fsocc(const string& path, int *pc, long long *avmbs)
         }
     }
     return true;
-#endif
+#endif /* !_WIN32 */
 }
 
 
@@ -433,21 +465,30 @@ string path_home()
 {
 #ifdef _WIN32
     string dir;
-    const char *cp = getenv("USERPROFILE");
+    // Using wgetenv does not work well, depending on the
+    // environment I get wrong values for the accented chars (works
+    // with recollindex started from msys command window, does not
+    // work when started from recoll. SHGet... fixes this
+    //const wchar_t *cp = _wgetenv(L"USERPROFILE");
+    wchar_t *cp;
+    SHGetKnownFolderPath(FOLDERID_Profile, 0, nullptr, &cp);
     if (cp != 0) {
-        dir = cp;
+        wchartoutf8(cp, dir);
     }
     if (dir.empty()) {
-        cp = getenv("HOMEDRIVE");
+        cp = _wgetenv(L"HOMEDRIVE");
+        wchartoutf8(cp, dir);
         if (cp != 0) {
-            const char *cp1 = getenv("HOMEPATH");
+            string dir1;
+            const wchar_t *cp1 = _wgetenv(L"HOMEPATH");
+            wchartoutf8(cp1, dir1);
             if (cp1 != 0) {
-                dir = string(cp) + string(cp1);
+                dir = path_cat(dir, dir1);
             }
         }
     }
     if (dir.empty()) {
-        dir = "C:\\";
+        dir = "C:/";
     }
     dir = path_canon(dir);
     path_catslash(dir);
@@ -475,12 +516,15 @@ string path_home()
 string path_homedata()
 {
 #ifdef _WIN32
-    const char *cp = getenv("LOCALAPPDATA");
+    wchar_t *cp;
+    SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, nullptr, &cp);
     string dir;
     if (cp != 0) {
-        dir = path_canon(cp);
+        wchartoutf8(cp, dir);
     }
-    if (dir.empty()) {
+    if (!dir.empty()) {
+        dir = path_canon(dir);
+    } else {
         dir = path_cat(path_home(), "AppData/Local/");
     }
     return dir;
@@ -673,8 +717,11 @@ bool path_makepath(const string& ipath, int mode)
         path += elem;
         // Not using path_isdir() here, because this cant grok symlinks
         // If we hit an existing file, no worry, mkdir will just fail.
-        if (access(path.c_str(), 0) != 0) {
-            if (mkdir(path.c_str(), mode) != 0)  {
+        LOGDEB1("path_makepath: testing existence: ["  << path << "]\n");
+        if (!path_exists(path)) {
+            LOGDEB1("path_makepath: creating directory ["  << path << "]\n");
+            SYSPATH(path, syspath);
+            if (MKDIR(syspath, mode) != 0)  {
                 //cerr << "mkdir " << path << " failed, errno " << errno << endl;
                 return false;
             }
@@ -684,6 +731,24 @@ bool path_makepath(const string& ipath, int mode)
     return true;
 }
 
+std::fstream path_open(const std::string& path, int mode)
+{
+#if defined(_WIN32) && defined (_MSC_VER)
+    // MSC STL has support for using wide chars in fstream
+    // constructor. We need this if, e.g. the user name/home directory
+    // is not ASCII. Actually don't know how to do this with gcc
+    wchar_t wpath[MAX_PATH + 1];
+    utf8towchar(path, wpath, MAX_PATH);
+    std::fstream ret(wpath, mode);
+    if (!ret.is_open()) {
+        LOGERR("path_open("<< path << ", "<< mode <<") errno " << errno <<"\n");
+    }
+    return ret;
+#else
+    return std::fstream(path, mode);
+#endif
+}
+
 bool path_isdir(const string& path, bool follow)
 {
     struct STATBUF st;
@@ -1098,7 +1163,8 @@ Pidfile::~Pidfile()
 
 int Pidfile::read_pid()
 {
-    int fd = ::open(m_path.c_str(), O_RDONLY);
+    SYSPATH(m_path, syspath);
+    int fd = OPEN(syspath, O_RDONLY);
     if (fd == -1) {
         return -1;
     }
@@ -1120,8 +1186,8 @@ int Pidfile::read_pid()
 
 int Pidfile::flopen()
 {
-    const char *path = m_path.c_str();
-    if ((m_fd = ::open(path, O_RDWR | O_CREAT, 0644)) == -1) {
+    SYSPATH(m_path, syspath);
+    if ((m_fd = OPEN(syspath, O_RDWR | O_CREAT, 0644)) == -1) {
         m_reason = "Open failed: [" + m_path + "]: " + strerror(errno);
         return -1;
     }
@@ -1198,7 +1264,8 @@ int Pidfile::close()
 
 int Pidfile::remove()
 {
-    return unlink(m_path.c_str());
+    SYSPATH(m_path, syspath);
+    return UNLINK(syspath);
 }
 
 // Call funcs that need static init (not initially reentrant)

src/utils/pathut.h

@@ -21,6 +21,7 @@
 #include <vector>
 #include <set>
 #include <cstdint>
+#include <fstream>
 
 // Must be called in main thread before starting other threads
 extern void pathut_init_mt();
@@ -129,6 +130,22 @@ bool fsocc(const std::string& path, int *pc, long long *avmbs = 0);
 /// mkdir -p
 extern bool path_makepath(const std::string& path, int mode);
 
+/* Open file, trying to do the right thing with non-ASCII paths on
+ * Windows, where it only works with MSVC at the moment if the path is
+ * not ASCII, because it uses fstream(wchar_t*), which is an MSVC
+ * extension. On other OSes, just builds the fstream.  We'd need to
+ * find a way to make this work with g++. It would be easier in this
+ * case to use a FILE (_openw(), then fdopen()), but conftree really
+ * depends on std::iostream. One possible workaround for g++ would be
+ * to use shortpaths (which we already use to pass file names to
+ * xapian and aspell). Most of the problems are caused by the home
+ * directory name being non-ASCII, so returning a short path in
+ * path_home() would probably solve everything (but not pretty).
+ *
+ * @param path an utf-8 file path.
+ * @param mode is an std::fstream mode (ios::in etc.) */
+extern std::fstream path_open(const std::string& path, int mode);
+
 /// Where we create the user data subdirs
 extern std::string path_homedata();
 /// Test if path is absolute
@@ -164,6 +181,9 @@ public:
 /// Convert \ separators to /
 void path_slashize(std::string& s);
 void path_backslashize(std::string& s);
+extern std::string path_shortpath(const std::string& path);
+#else
+#define path_shortpath(path) (path)
 #endif
 
 /// Lock/pid file class. This is quite close to the pidfile_xxx

src/utils/rclutil.cpp

@@ -118,6 +118,10 @@ static const string& path_wingetrcltmpdir()
                 LOGSYSERR("path_wingettempfilename", "path_makepath", tdir);
             }
         }
+        // Try to use a short path to avoid issues in case the user
+        // login name is not ascii, especially with command line
+        // parameter passing
+        tdir = path_shortpath(tdir);
     }
     return tdir;
 }