Impose memory usage limit on external filters. Fixes issue #259

src/utils/execmd.cpp

@@ -28,6 +28,8 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/select.h>
+#include <sys/time.h>
+#include <sys/resource.h>
 #include <fcntl.h>
 #include <errno.h>
 #include <signal.h>
@@ -300,6 +302,39 @@ inline void ExecCmd::dochild(const string &cmd, const char **argv,
     pthread_sigmask(SIG_UNBLOCK, &sset, 0);
     sigprocmask(SIG_UNBLOCK, &sset, 0);
 
+#ifdef HAVE_SETRLIMIT
+#if defined RLIMIT_AS || defined RLIMIT_VMEM || defined RLIMIT_DATA
+    if (m_rlimit_as_mbytes > 2000 && sizeof(rlim_t) < 8) {
+	// Impossible limit, don't use it
+	m_rlimit_as_mbytes = 0;
+    }
+    if (m_rlimit_as_mbytes > 0) {
+	struct rlimit ram_limit = {
+	    static_cast<rlim_t>(m_rlimit_as_mbytes * 1024 * 1024),
+	    RLIM_INFINITY
+	};
+	int resource;
+
+	// RLIMIT_AS and RLIMIT_VMEM are usually synonyms when VMEM is
+	// defined. RLIMIT_AS is Posix. Both don't really do what we
+	// want, because they count e.g. shared lib mappings, which we
+	// don't really care about.
+	// RLIMIT_DATA only limits the data segment. Modern mallocs
+	// use mmap and will not be bound. (Otoh if we only have this,
+	// we're probably not modern).
+	// So we're unsatisfied either way.
+#ifdef RLIMIT_AS
+	resource = RLIMIT_AS;
+#elif defined RLIMIT_VMEM
+	resource = RLIMIT_VMEM;
+#else
+	resource = RLIMIT_DATA;
+#endif
+	setrlimit(resource, &ram_limit);
+    }
+#endif
+#endif // have_setrlimit
+
     if (has_input) {
 	close(m_pipein[1]);
 	if (m_pipein[0] != 0) {
@@ -347,6 +382,11 @@ inline void ExecCmd::dochild(const string &cmd, const char **argv,
     _exit(127);
 }
 
+void ExecCmd::setrlimit_as(int mbytes)
+{
+    m_rlimit_as_mbytes = mbytes;
+}
+
 int ExecCmd::startExec(const string &cmd, const vector<string>& args,
 		       bool has_input, bool has_output)
 {
@@ -427,6 +467,7 @@ int ExecCmd::startExec(const string &cmd, const vector<string>& args,
 //////////////////////////////// End vfork child prepare section.
 
 #if HAVE_POSIX_SPAWN && USE_POSIX_SPAWN
+    // Note that posix_spawn provides no way to setrlimit() the child.
     {
         posix_spawnattr_t attrs;
         posix_spawnattr_init (&attrs);

src/utils/execmd.h

@@ -83,6 +83,14 @@ class ExecCmd {
     void putenv(const std::string &envassign);
     void putenv(const std::string &name, const std::string& value);
 
+    /** 
+     * Try to set a limit on child process vm size. This will use
+     * setrlimit() and RLIMIT_AS/VMEM if available. Parameter is in
+     * units of 2**10. Must be called before starting the command, default 
+     * is inherit from parent.
+     */
+    void setrlimit_as(int mbytes);
+
     /** 
      * Set function objects to call whenever new data is available or on
      * select timeout / whenever new data is needed to send. Must be called
@@ -158,7 +166,7 @@ class ExecCmd {
     void zapChild() {setKill(); (void)wait();}
 
     ExecCmd()
-	: m_advise(0), m_provide(0), m_timeoutMs(1000)
+	: m_advise(0), m_provide(0), m_timeoutMs(1000), m_rlimit_as_mbytes(0)
     {
 	reset();
     }
@@ -191,6 +199,7 @@ class ExecCmd {
     ExecCmdProvide  *m_provide;
     bool             m_killRequest;
     int              m_timeoutMs;
+    int              m_rlimit_as_mbytes;
     std::string           m_stderrFile;
     // Pipe for data going to the command
     int              m_pipein[2];