diff --git a/src/filters/rclexecm.py b/src/filters/rclexecm.py index 15591155..0637c6b2 100644 --- a/src/filters/rclexecm.py +++ b/src/filters/rclexecm.py @@ -58,12 +58,34 @@ class RclExecM: import msvcrt msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) - + self.debugfile = None + if self.debugfile: + self.errfout = open(self.debugfile, "ab") + else: + self.errfout = sys.stderr + def rclog(self, s, doexit = 0, exitvalue = 1): - print("RCLMFILT: %s: %s" % (self.myname, s), file=sys.stderr) + print("RCLMFILT: %s: %s" % (self.myname, s), file=self.errfout) if doexit: sys.exit(exitvalue) + def breakwrite(self, outfile, data): + if sys.platform != "win32": + outfile.write(data) + else: + total = len(data) + bs = 4*1024 + offset = 0 + while total > 0: + if total < bs: + tow = total + else: + tow = bs + #self.rclog("Total %d Writing %d to stdout: %s" % (total,tow,data[offset:offset+tow])) + outfile.write(data[offset:offset+tow]) + offset += tow + total -= tow + # Note: tried replacing this with a multiple replacer according to # http://stackoverflow.com/a/15221068, which was **10 times** slower def htmlescape(self, txt): @@ -119,7 +141,7 @@ class RclExecM: docdata = docdata.encode("UTF-8") print("Document: %d" % len(docdata)) - sys.stdout.write(docdata) + self.breakwrite(sys.stdout, docdata) if len(ipath): print("Ipath: %d" % len(ipath)) @@ -332,7 +354,7 @@ def main(proto, extract): else: bdata = data if debugDumpData or actAsSingle: - sys.stdout.write(bdata) + proto.breakwrite(sys.stdout, bdata) print() sys.exit(0) else: @@ -351,7 +373,7 @@ def main(proto, extract): else: bdata = data if debugDumpData: - sys.stdout.write(bdata) + proto.breakwrite(sys.stdout, bdata) print() if eof != RclExecM.noteof: sys.exit(0) diff --git a/src/librecoll/librecoll.pro b/src/librecoll/librecoll.pro index 5423b607..95eebbba 100644 --- a/src/librecoll/librecoll.pro +++ b/src/librecoll/librecoll.pro @@ -13,6 +13,7 @@ DEFINES += LIBRECOLL_LIBRARY BUILDING_RECOLL DEFINES -= UNICODE DEFINES -= _UNICODE DEFINES += _MBCS +DEFINES += PSAPI_VERSION=1 SOURCES += \ ../aspell/rclaspell.cpp \ @@ -119,7 +120,7 @@ windows{ # Visual Studio } LIBS += c:/recolldeps/xapian/xapian-core-1.2.21/.libs/libxapian-22.dll \ - c:/recolldeps/zlib-1.2.8/zlib1.dll -liconv -lshlwapi -lkernel32 + c:/recolldeps/zlib-1.2.8/zlib1.dll -liconv -lshlwapi -lpsapi -lkernel32 } unix { diff --git a/src/windows/execmd_w.cpp b/src/windows/execmd_w.cpp index 07f84569..3c2d10fc 100644 --- a/src/windows/execmd_w.cpp +++ b/src/windows/execmd_w.cpp @@ -11,6 +11,7 @@ #include "safesysstat.h" #include "safeunistd.h" #include "safewindows.h" +#include #include "smallut.h" #include "pathut.h" @@ -89,7 +90,7 @@ static string argvToCmdLine(const string& cmd, const vector& args) } // Merge the father environment with the variable specified in m_env -char *mergeEnvironment(const STD_UNORDERED_MAP& addenv) +static char *mergeEnvironment(const STD_UNORDERED_MAP& addenv) { // Parse existing environment. char *envir = GetEnvironmentStrings(); @@ -235,13 +236,27 @@ static WaitResult Wait(HANDLE hdl, int timeout) return Timeout; } +static int getVMMBytes(HANDLE hProcess) +{ + PROCESS_MEMORY_COUNTERS pmc; + const int MB = 1024 * 1024; + if (GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc))) { + LOGDEB2(("ExecCmd: getVMMBytes paged Kbs %d non paged %d Kbs\n", + int(pmc.QuotaPagedPoolUsage/1024), + int(pmc.QuotaNonPagedPoolUsage/1024))); + return int(pmc.QuotaPagedPoolUsage /MB + + pmc.QuotaNonPagedPoolUsage / MB); + } + return -1; +} + //////////////////////////////////////////////////////// // ExecCmd: class ExecCmd::Internal { public: Internal() - : m_advise(0), m_provide(0), m_timeoutMs(1000) { + : m_advise(0), m_provide(0), m_timeoutMs(1000), m_rlimit_as_mbytes(0) { reset(); } @@ -249,6 +264,7 @@ public: ExecCmdAdvise *m_advise; ExecCmdProvide *m_provide; int m_timeoutMs; + int m_rlimit_as_mbytes; // We need buffered I/O for getline. The Unix version uses netcon's string m_buf; // Buffer. Only used when doing getline()s @@ -278,6 +294,7 @@ public: bool preparePipes(bool has_input, HANDLE *hChildInput, bool has_output, HANDLE *hChildOutput, HANDLE *hChildError); + bool tooBig(); }; // ExecCmd resource releaser class. Using a separate object makes it @@ -455,7 +472,21 @@ void ExecCmd::putenv(const string &name, const string& value) void ExecCmd::setrlimit_as(int mbytes) { - // Later maybe + m->m_rlimit_as_mbytes = mbytes; +} + +bool ExecCmd::Internal::tooBig() +{ + if (m_rlimit_as_mbytes <= 0) + return false; + int mbytes = getVMMBytes(m_piProcInfo.hProcess); + if (mbytes > m_rlimit_as_mbytes) { + LOGINFO(("ExecCmd:: process mbytes %d > set limit %d\n", + mbytes, m_rlimit_as_mbytes)); + m_killRequest = true; + return true; + } + return false; } bool ExecCmd::Internal::preparePipes(bool has_input,HANDLE *hChildInput, @@ -781,11 +812,11 @@ int ExecCmd::receive(string& data, int cnt) } } while (true) { - const int BUFSIZE = 8192; + const int BUFSIZE = 4096; CHAR chBuf[BUFSIZE]; int toread = cnt > 0 ? MIN(cnt - totread, BUFSIZE) : BUFSIZE; BOOL bSuccess = ReadFile(m->m_hOutputRead, chBuf, toread, - NULL, &m->m_oOutputRead); + NULL, &m->m_oOutputRead); DWORD err = GetLastError(); LOGDEB1(("receive: ReadFile: success %d err %d\n", int(bSuccess), int(err))); @@ -795,7 +826,8 @@ int ExecCmd::receive(string& data, int cnt) break; } - WaitResult waitRes = Wait(m->m_oOutputRead.hEvent, 1000); + waitagain: + WaitResult waitRes = Wait(m->m_oOutputRead.hEvent, m->m_timeoutMs); if (waitRes == Ok) { DWORD dwRead; if (!GetOverlappedResult(m->m_hOutputRead, &m->m_oOutputRead, @@ -820,6 +852,13 @@ int ExecCmd::receive(string& data, int cnt) } break; } else if (waitRes == Timeout) { + LOGDEB0(("ExecCmd::receive: timeout (%d mS)\n", m->m_timeoutMs)); + if (m->tooBig()) { + if (!CancelIo(m->m_hOutputRead)) { + printError("CancelIo"); + } + return -1; + } // We only want to cancel if m_advise says so here. if (m->m_advise) { try { @@ -838,6 +877,7 @@ int ExecCmd::receive(string& data, int cnt) } break; } + goto waitagain; } if ((cnt == 0 && totread > 0) || (cnt > 0 && totread == cnt)) break; @@ -901,7 +941,19 @@ int ExecCmd::wait() DWORD exit_code = -1; if (!m->m_killRequest && m->m_piProcInfo.hProcess) { // Wait until child process exits. - WaitForSingleObject(m->m_piProcInfo.hProcess, INFINITE); + while (WaitForSingleObject(m->m_piProcInfo.hProcess, m->m_timeoutMs) + == WAIT_TIMEOUT) { + LOGDEB(("ExecCmd::wait: timeout (ok)\n")); + if (m->m_advise) { + m->m_advise->newData(0); + } + if (m->tooBig()) { + // Let cleaner work to kill the child + m->m_killRequest = true; + return -1; + } + } + exit_code = 0; GetExitCodeProcess(m->m_piProcInfo.hProcess, &exit_code); // Clean up, here to avoid cleaner trying to kill the now diff --git a/src/windows/mkinstdir.sh b/src/windows/mkinstdir.sh index 9ba720a1..ab2f2335 100644 --- a/src/windows/mkinstdir.sh +++ b/src/windows/mkinstdir.sh @@ -17,6 +17,9 @@ ANTIWORD=c:/recolldeps/antiword CONFIGURATION=Debug PLATFORM=x64 +GUIBIN=c:/recoll/src/build-librecoll-Desktop_Qt_5_5_0_MinGW_32bit-Debug/debug/librecoll.dll +GUILIB=c:/Users/Bill/recoll/src/build-recoll-Desktop_Qt_5_5_0_MinGW_32bit-Debug/debug/recoll.exe + ################ # Script: @@ -57,7 +60,10 @@ copyrecoll() cp $RECOLL/qtgui/mtpics/* $DESTDIR/Share/images - cp $RECOLL/qtgui/i18n/*.qm $DESTDIR/Share/translations + cp $RECOLL/qtgui/i18n/*.qm $DESTDIR/Share/translations + + cp $GUIBIN $DESTDIR || fatal copy recoll.exe + cp $GUILIB $DESTDIR || fatal copy Gui librecoll } copyantiword()