0001-Allow-multiple-judgedaemons-on-a-single-machine.patch

From 2622a5aedf455abf1e6cbfffd2d7704065923186 Mon Sep 17 00:00:00 2001
From: Keith Johnson <kj@ubergeek42.com>
Date: Tue, 18 Jun 2013 12:26:42 -0400
Subject: Allow multiple judgedaemons on a single machine

Add support for running multiple judgedaemons at once on a single
machine.  This allows for better use of multiple cpu cores.  To provide
for better isolation between the judegdaemons, the cpuset cgroup support
is used to limit processes to a single cpu core.
---
 etc/cgroup-domjudge.conf.in |    7 +++++++
 etc/runguard-config.h.in    |    3 ++-
 judge/compile.sh            |   28 ++++++++++++++++++++++++--
 judge/judgedaemon.main.php  |   46 ++++++++++++++++++++++++++++++++-----------
 judge/runguard.c            |   24 ++++++++++++++++++++--
 judge/testcase_run.sh       |   29 +++++++++++++++++++++++++--
 6 files changed, 119 insertions(+), 18 deletions(-)

diff --git a/etc/cgroup-domjudge.conf.in b/etc/cgroup-domjudge.conf.in
index 1d6cb42..b7f3415 100644
--- a/etc/cgroup-domjudge.conf.in
+++ b/etc/cgroup-domjudge.conf.in
@@ -18,4 +18,11 @@ group domjudge {
 	memory {
 # This section is an empty stub: the limits are set by runguard.
 	}
+# Change the cpuset.cpus line to a range specifying the cores you are
+# going to use.  e.g. if you have a quad core machine, set cpuset.cpus = 0-3
+# and then run 4 judgedaemons.
+	cpuset {
+		cpuset.mems = 0;
+		cpuset.cpus = 0;
+	}
 }
diff --git a/etc/runguard-config.h.in b/etc/runguard-config.h.in
index bea6eca..e0e620b 100644
--- a/etc/runguard-config.h.in
+++ b/etc/runguard-config.h.in
@@ -3,7 +3,8 @@
 #ifndef _RUNGUARD_CONFIG_
 #define _RUNGUARD_CONFIG_
 
-#define VALID_USERS "@RUNUSER@"
+/* Lots of suffixed names because we want to support multiple judgedaemons per host.  This allows 8 of them.*/
+#define VALID_USERS "@RUNUSER@,@RUNUSER@-0,@RUNUSER@-1,@RUNUSER@-2,@RUNUSER@-3,@RUNUSER@-4,@RUNUSER@-5,@RUNUSER@-6,@RUNUSER@-7"
 
 #define CHROOT_PREFIX "@judgehost_judgedir@"
 
diff --git a/judge/compile.sh b/judge/compile.sh
index 3604705..2c520c3 100755
--- a/judge/compile.sh
+++ b/judge/compile.sh
@@ -47,8 +47,32 @@ cleanexit ()
 # Error and logging functions
 . "$DJ_LIBDIR/lib.error.sh"
 
+CPUSET=""
+CPUSET_OPT=""
+# Do argument parsing
+OPTIND=1 # reset if necessary
+while getopts "n:" opt; do
+	case $opt in
+		n)
+			CPUSET="$OPTARG"
+			;;
+		:)
+			echo "Option -$OPTARG requires an argument." >&2
+			;;
+	esac
+done
+# Shift any of the arguments out of the way
+shift $((OPTIND-1))
+[ "$1" = "--" ] && shift
+
+if [ -n "$CPUSET" ]; then
+	CPUSET_OPT="-P $CPUSET"
+	LOGFILE="$DJ_LOGDIR/judge.`hostname | cut -d . -f 1`-$CPUSET.log"
+else
+	LOGFILE="$DJ_LOGDIR/judge.`hostname | cut -d . -f 1`.log"
+fi
+
 # Logging:
-LOGFILE="$DJ_LOGDIR/judge.`hostname | cut -d . -f 1`.log"
 LOGLEVEL=$LOG_DEBUG
 PROGNAME="`basename $0`"
 
@@ -99,7 +123,7 @@ logmsg $LOG_INFO "starting compile"
 # First compile to 'source' then rename to 'program' to avoid problems with
 # the compiler writing to different filenames and deleting intermediate files.
 exitcode=0
-"$RUNGUARD" ${DEBUG:+-v} -t $COMPILETIME -c -f 65536 -T "$WORKDIR/compile.time" -- \
+"$RUNGUARD" ${DEBUG:+-v} $CPUSET_OPT -t $COMPILETIME -c -f 65536 -T "$WORKDIR/compile.time" -- \
 	"$COMPILE_SCRIPT" program "$MEMLIMIT" "$@" >"$WORKDIR/compile.tmp" 2>&1 || \
 	exitcode=$?
 
diff --git a/judge/judgedaemon.main.php b/judge/judgedaemon.main.php
index c6634f4..4dc663c 100644
--- a/judge/judgedaemon.main.php
+++ b/judge/judgedaemon.main.php
@@ -12,26 +12,22 @@ require(ETCDIR . '/judgehost-config.php');
 
 $waittime = 5;
 
-$myhost = trim(`hostname | cut -d . -f 1`);
-
 define ('SCRIPT_ID', 'judgedaemon');
-define ('LOGFILE', LOGDIR.'/judge.'.$myhost.'.log');
 define ('PIDFILE', RUNDIR.'/judgedaemon.pid');
 
-require(LIBDIR . '/init.php');
-
 function usage()
 {
 	echo "Usage: " . SCRIPT_ID . " [OPTION]...\n" .
 	    "Start the judgedaemon.\n\n" .
 	    "  -d       daemonize after startup\n" .
+	    "  -n <id>  daemon number\n" .
 	    "  -v       set verbosity to LEVEL (syslog levels)\n" .
 	    "  -h       display this help and exit\n" .
 	    "  -V       output version information and exit\n\n";
 	exit;
 }
 
-$options = getopt("dv:hV");
+$options = getopt("dv:n:hV");
 // With PHP version >= 5.3 we can also use long options.
 // FIXME: getopt doesn't return FALSE on parse failure as documented!
 if ( $options===FALSE ) {
@@ -40,14 +36,35 @@ if ( $options===FALSE ) {
 }
 if ( isset($options['d']) ) $options['daemon']  = $options['d'];
 if ( isset($options['v']) ) $options['verbose'] = $options['v'];
+if ( isset($options['n']) ) $options['daemonid'] = $options['n'];
 
 if ( isset($options['V']) ) version();
 if ( isset($options['h']) ) usage();
 
+$myhost = trim(`hostname | cut -d . -f 1`);
+if ( isset($options['daemonid']) ) {
+	if ( preg_match('/^\d+$/', $options['daemonid'] ) ) {
+		$myhost = $myhost . "-" . $options['daemonid'];	
+	} else {
+		echo "Invalid value for daemonid, must be positive integer\n";
+		exit(1);
+	}
+}
+
+define ('LOGFILE', LOGDIR.'/judge.'.$myhost.'.log');
+require(LIBDIR . '/init.php');
+
 setup_database_connection();
 
 $verbose = LOG_INFO;
-if ( isset($options['verbose']) ) $verbose = $options['verbose'];
+if ( isset($options['verbose']) ) {
+	if ( preg_match('/^\d+$/', $options['verbose'] ) ) {
+		$verbose = $options['verbose'];	
+	} else {
+		echo "Invalid value for verbose, must be positive integer\n";
+		exit(1);
+	}
+}
 
 if ( DEBUG & DEBUG_JUDGE ) {
 	$verbose = LOG_DEBUG;
@@ -62,7 +79,11 @@ putenv('DJ_JUDGEDIR='    . JUDGEDIR);
 putenv('DJ_LIBDIR='      . LIBDIR);
 putenv('DJ_LIBJUDGEDIR=' . LIBJUDGEDIR);
 putenv('DJ_LOGDIR='      . LOGDIR);
-putenv('RUNUSER='        . RUNUSER);
+if ( isset($options['daemonid']) ) {
+	putenv('RUNUSER='    . RUNUSER . '-' . $options['daemonid']);
+} else {
+	putenv('RUNUSER='    . RUNUSER);
+}
 
 foreach ( $EXITCODES as $code => $name ) {
 	$var = 'E_' . strtoupper(str_replace('-','_',$name));
@@ -281,7 +302,7 @@ while ( TRUE ) {
 
 function judge($mark, $row, $judgingid)
 {
-	global $EXITCODES, $DB, $cid, $myhost, $workdirpath;
+	global $EXITCODES, $DB, $cid, $myhost, $options, $workdirpath;
 
 	// Set configuration variables for called programs
 	// Call dbconfig_init() to prevent using cached values.
@@ -292,6 +313,9 @@ function judge($mark, $row, $judgingid)
 	putenv('FILELIMIT='     . dbconfig_get('filesize_limit'));
 	putenv('PROCLIMIT='     . dbconfig_get('process_limit'));
 
+	$cpuset_opt = "";
+	if ( isset($options['daemonid']) ) $cpuset_opt = "-n ${options['daemonid']}";
+
 	// create workdir for judging
 	$workdir = "$workdirpath/c$cid-s$row[submitid]-j$judgingid";
 
@@ -326,7 +350,7 @@ function judge($mark, $row, $judgingid)
 	}
 
 	// Compile the program.
-	system(LIBJUDGEDIR . "/compile.sh $row[langid] '$workdir' " .
+	system(LIBJUDGEDIR . "/compile.sh $cpuset_opt $row[langid] '$workdir' " .
 	       implode(' ', $files), $retval);
 
 	// what does the exitcode mean?
@@ -412,7 +436,7 @@ function judge($mark, $row, $judgingid)
 	if ( $retval!=0 ) error("Could not copy program to '$programdir'");
 
 	// do the actual test-run
-	system(LIBJUDGEDIR . "/testcase_run.sh $tcfile[input] $tcfile[output] " .
+	system(LIBJUDGEDIR . "/testcase_run.sh $cpuset_opt $tcfile[input] $tcfile[output] " .
 	       "$row[maxruntime] '$testcasedir' " .
 	       "'$row[special_run]' '$row[special_compare]'", $retval);
 
diff --git a/judge/runguard.c b/judge/runguard.c
index 44e494d..e9b9fb5 100644
--- a/judge/runguard.c
+++ b/judge/runguard.c
@@ -108,6 +108,7 @@ char  *exitfilename;
 char  *timefilename;
 #ifdef USE_CGROUPS
 char  *cgroupname;
+const char *cpuset;
 #endif
 
 int runuid;
@@ -159,6 +160,7 @@ struct option const long_opts[] = {
 	{"memsize",    required_argument, NULL,         'm'},
 	{"filesize",   required_argument, NULL,         'f'},
 	{"nproc",      required_argument, NULL,         'p'},
+	{"cpuset",     required_argument, NULL,         'P'},
 	{"no-core",    no_argument,       NULL,         'c'},
 	{"stdout",     required_argument, NULL,         'o'},
 	{"stderr",     required_argument, NULL,         'e'},
@@ -255,6 +257,7 @@ Run COMMAND with restrictions.\n\
   -f, --filesize=SIZE    set maximum created filesize to SIZE kB;\n");
 	printf("\
   -p, --nproc=N          set maximum no. processes to N\n\
+  -P, --cpuset=ID        use only processor number ID\n\
   -c, --no-core          disable core dumps\n\
   -o, --stdout=FILE      redirect COMMAND stdout output to FILE\n\
   -e, --stderr=FILE      redirect COMMAND stderr output to FILE\n\
@@ -363,9 +366,23 @@ void cgroup_create()
 	cgroup_add_value_int64(cg_controller, "memory.limit_in_bytes", memsize);
 	cgroup_add_value_int64(cg_controller, "memory.memsw.limit_in_bytes", memsize);
 
+	/* Set up cpu restrictions; we pin the task to a specific set of cpus,
+	   based on the environment variable CPUSET. We also give it exclusive
+	   access to those cores, and set no limits on memory nodes */
+	if ( cpuset!=NULL && strlen(cpuset)>0 ) {
+		cg_controller = cgroup_add_controller(cg, "cpuset");
+		/* To make a cpuset exclusive, some additional setup outside of domjudge is
+		   required, so for now, we will leave this commented out. */
+		/* cgroup_add_value_int64(cg_controller, "cpuset.cpu_exclusive", 1); */
+		cgroup_add_value_string(cg_controller, "cpuset.mems", "0");
+		cgroup_add_value_string(cg_controller, "cpuset.cpus", cpuset);
+	} else {
+		fprintf(stderr, "CPUSET undefined\n");
+	}
+
 	/* Perform the actual creation of the cgroup */
 	ret = cgroup_create_cgroup(cg, 1);
-	if ( ret!=0) {
+	if ( ret!=0 ) {
 		error(0,"creating cgroup - %s(%d)", cgroup_strerror(ret), ret);
 	}
 
@@ -638,7 +655,7 @@ int main(int argc, char **argv)
 	be_verbose = be_quiet = 0;
 	show_help = show_version = 0;
 	opterr = 0;
-	while ( (opt = getopt_long(argc,argv,"+r:u:g:t:C:m:f:p:co:e:s:E:T:vq",long_opts,(int *) 0))!=-1 ) {
+	while ( (opt = getopt_long(argc,argv,"+r:u:g:t:C:m:f:p:P:co:e:s:E:T:vq",long_opts,(int *) 0))!=-1 ) {
 		switch ( opt ) {
 		case 0:   /* long-only option */
 			break;
@@ -694,6 +711,9 @@ int main(int argc, char **argv)
 		case 'p': /* nproc option */
 			nproc = (rlim_t) readoptarg("process limit",1,LONG_MAX);
 			break;
+		case 'P': /* cpuset option */
+			cpuset = optarg;
+			break;
 		case 'c': /* no-core option */
 			no_coredump = 1;
 			break;
diff --git a/judge/testcase_run.sh b/judge/testcase_run.sh
index efaa29a..82259ef 100755
--- a/judge/testcase_run.sh
+++ b/judge/testcase_run.sh
@@ -73,8 +73,33 @@ runcheck ()
 # Error and logging functions
 . "$DJ_LIBDIR/lib.error.sh"
 
+
+CPUSET=""
+CPUSET_OPT=""
+# Do argument parsing
+OPTIND=1 # reset if necessary
+while getopts "n:" opt; do
+	case $opt in
+		n)
+			CPUSET="$OPTARG"
+			;;
+		:)
+			echo "Option -$OPTARG requires an argument." >&2
+			;;
+	esac
+done
+# Shift any of the arguments out of the way
+shift $((OPTIND-1))
+[ "$1" = "--" ] && shift
+
+if [ -n "$CPUSET" ]; then
+	CPUSET_OPT="-P $CPUSET"
+	LOGFILE="$DJ_LOGDIR/judge.`hostname | cut -d . -f 1`-$CPUSET.log"
+else
+	LOGFILE="$DJ_LOGDIR/judge.`hostname | cut -d . -f 1`.log"
+fi
+
 # Logging:
-LOGFILE="$DJ_LOGDIR/judge.`hostname | cut -d . -f 1`.log"
 LOGLEVEL=$LOG_DEBUG
 PROGNAME="`basename $0`"
 
@@ -170,7 +195,7 @@ $GAINROOT cp -pR /dev/null ../dev/null
 logmsg $LOG_INFO "running program (USE_CHROOT = ${USE_CHROOT:-0})"
 
 runcheck ./run testdata.in program.out \
-	$GAINROOT $RUNGUARD ${DEBUG:+-v} ${USE_CHROOT:+-r "$PWD/.."} -u "$RUNUSER" \
+	$GAINROOT $RUNGUARD ${DEBUG:+-v} $CPUSET_OPT ${USE_CHROOT:+-r "$PWD/.."} -u "$RUNUSER" \
 	-C $TIMELIMIT -t $((2*TIMELIMIT)) -m $MEMLIMIT -f $FILELIMIT -p $PROCLIMIT \
 	-c -s $FILELIMIT -e program.err -E program.exit -T program.time -- \
 	$PREFIX/$PROGRAM 2>error.tmp
-- 
1.7.9.5