- (dtucker) Bug #1470: Disable OOM-killing of the listening sshd on Linux,

based on a patch from Vaclav Ovsik and Colin Watson.  ok djm.

ChangeLog

@@ -1,3 +1,7 @@
+20091208
+ - (dtucker) Bug #1470: Disable OOM-killing of the listening sshd on Linux,
+   based on a patch from Vaclav Ovsik and Colin Watson.  ok djm.
+
 20091207
  - (dtucker) Bug #1160: use pkg-config for opensc config if it's available.
    Tested by Martin Paljak.

configure.ac

@@ -1,4 +1,4 @@
-# $Id: configure.ac,v 1.431 2009/12/07 00:15:43 dtucker Exp $
+# $Id: configure.ac,v 1.432 2009/12/08 02:39:48 dtucker Exp $
 #
 # Copyright (c) 1999-2004 Damien Miller
 #
@@ -15,7 +15,7 @@
 # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 AC_INIT(OpenSSH, Portable, openssh-unix-dev@mindrot.org)
-AC_REVISION($Revision: 1.431 $)
+AC_REVISION($Revision: 1.432 $)
 AC_CONFIG_SRCDIR([ssh.c])
 
 AC_CONFIG_HEADER(config.h)
@@ -589,6 +589,7 @@ main() { if (NSVersionOfRunTimeLibrary("System") >= (60 << 16))
 		if it doesn't return EOPNOTSUPP.])
 	AC_DEFINE(_PATH_BTMP, "/var/log/btmp", [log for bad login attempts])
 	AC_DEFINE(USE_BTMP)
+	AC_DEFINE(LINUX_OOM_ADJUST, 1, [Adjust Linux out-of-memory killer])
 	inet6_default_4in6=yes
 	case `uname -r` in
 	1.*|2.0.*)

openbsd-compat/port-linux.c

@@ -1,4 +1,4 @@
-/* $Id: port-linux.c,v 1.6 2009/10/24 04:04:13 dtucker Exp $ */
+/* $Id: port-linux.c,v 1.7 2009/12/08 02:39:48 dtucker Exp $ */
 
 /*
  * Copyright (c) 2005 Daniel Walsh <dwalsh@redhat.com>
@@ -23,15 +23,17 @@
 
 #include "includes.h"
 
+#if defined(WITH_SELINUX) || defined(LINUX_OOM_ADJUST)
 #include <errno.h>
 #include <stdarg.h>
 #include <string.h>
+#include <stdio.h>
 
-#ifdef WITH_SELINUX
 #include "log.h"
 #include "xmalloc.h"
 #include "port-linux.h"
 
+#ifdef WITH_SELINUX
 #include <selinux/selinux.h>
 #include <selinux/flask.h>
 #include <selinux/get_context_list.h>
@@ -204,3 +206,60 @@ ssh_selinux_change_context(const char *newname)
 	xfree(newctx);
 }
 #endif /* WITH_SELINUX */
+
+#ifdef LINUX_OOM_ADJUST
+#define OOM_ADJ_PATH	"/proc/self/oom_adj"
+/*
+ * The magic "don't kill me", as documented in eg:
+ * http://lxr.linux.no/#linux+v2.6.32/Documentation/filesystems/proc.txt
+ */
+#define OOM_ADJ_NOKILL	-17
+
+static int oom_adj_save = INT_MIN;
+
+/*
+ * Tell the kernel's out-of-memory killer to avoid sshd.
+ * Returns the previous oom_adj value or zero.
+ */
+void
+oom_adjust_setup(void)
+{
+	FILE *fp;
+
+	debug3("%s", __func__);
+	if ((fp = fopen(OOM_ADJ_PATH, "r+")) != NULL) {
+		if (fscanf(fp, "%d", &oom_adj_save) != 1)
+			logit("error reading %s: %s", OOM_ADJ_PATH, strerror(errno));
+		else {
+			rewind(fp);
+			if (fprintf(fp, "%d\n", OOM_ADJ_NOKILL) <= 0)
+				logit("error writing %s: %s",
+				    OOM_ADJ_PATH, strerror(errno));
+			else
+				verbose("Set %s from %d to %d",
+				    OOM_ADJ_PATH, oom_adj_save, OOM_ADJ_NOKILL);
+		}
+		fclose(fp);
+	}
+}
+
+/* Restore the saved OOM adjustment */
+void
+oom_adjust_restore(void)
+{
+	FILE *fp;
+
+	debug3("%s", __func__);
+	if (oom_adj_save == INT_MIN || (fp = fopen(OOM_ADJ_PATH, "w")) == NULL)
+		return;
+
+	if (fprintf(fp, "%d\n", oom_adj_save) <= 0)
+		logit("error writing %s: %s", OOM_ADJ_PATH, strerror(errno));
+	else
+		verbose("Set %s to %d", OOM_ADJ_PATH, oom_adj_save);
+
+	fclose(fp);
+	return;
+}
+#endif /* LINUX_OOM_ADJUST */
+#endif /* WITH_SELINUX || LINUX_OOM_ADJUST */

openbsd-compat/port-linux.h

@@ -1,4 +1,4 @@
-/* $Id: port-linux.h,v 1.3 2009/10/24 04:04:13 dtucker Exp $ */
+/* $Id: port-linux.h,v 1.4 2009/12/08 02:39:48 dtucker Exp $ */
 
 /*
  * Copyright (c) 2006 Damien Miller <djm@openbsd.org>
@@ -26,4 +26,9 @@ void ssh_selinux_setup_exec_context(char *);
 void ssh_selinux_change_context(const char *);
 #endif
 
+#ifdef LINUX_OOM_ADJUST
+void oom_adjust_restore(void);
+void oom_adjust_setup(void);
+#endif
+
 #endif /* ! _PORT_LINUX_H */

platform.c

@@ -1,4 +1,4 @@
-/* $Id: platform.c,v 1.1 2006/08/30 17:24:41 djm Exp $ */
+/* $Id: platform.c,v 1.2 2009/12/08 02:39:48 dtucker Exp $ */
 
 /*
  * Copyright (c) 2006 Darren Tucker.  All rights reserved.
@@ -21,6 +21,15 @@
 
 #include "openbsd-compat/openbsd-compat.h"
 
+void
+platform_pre_listen(void)
+{
+#ifdef LINUX_OOM_ADJUST
+	/* Adjust out-of-memory killer so listening process is not killed */
+	oom_adjust_setup();
+#endif
+}
+
 void
 platform_pre_fork(void)
 {
@@ -43,4 +52,7 @@ platform_post_fork_child(void)
 #ifdef USE_SOLARIS_PROCESS_CONTRACTS
 	solaris_contract_post_fork_child();
 #endif
+#ifdef LINUX_OOM_ADJUST
+	oom_adjust_restore();
+#endif
 }

platform.h

@@ -1,4 +1,4 @@
-/* $Id: platform.h,v 1.1 2006/08/30 17:24:41 djm Exp $ */
+/* $Id: platform.h,v 1.2 2009/12/08 02:39:48 dtucker Exp $ */
 
 /*
  * Copyright (c) 2006 Darren Tucker.  All rights reserved.
@@ -18,6 +18,7 @@
 
 #include <sys/types.h>
 
+void platform_pre_listen(void);
 void platform_pre_fork(void);
 void platform_post_fork_parent(pid_t child_pid);
 void platform_post_fork_child(void);

sshd.c

@@ -1656,6 +1656,7 @@ main(int ac, char **av)
 	if (inetd_flag) {
 		server_accept_inetd(&sock_in, &sock_out);
 	} else {
+		platform_pre_listen();
 		server_listen();
 
 		if (options.protocol & SSH_PROTO_1)