From ce87fcf092a18e987f26863034632d4b57a145a1 Mon Sep 17 00:00:00 2001 From: Chaskiel Grundman Date: Fri, 13 Dec 2024 13:28:12 -0500 Subject: [PATCH 1/4] build docker args with list.append instead of constructing new list objects every time the list has items appended to it, use python's list mutator functions (append and extend) --- vmms/localDocker.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vmms/localDocker.py b/vmms/localDocker.py index 45dda03d..81bd965d 100644 --- a/vmms/localDocker.py +++ b/vmms/localDocker.py @@ -156,15 +156,15 @@ def runJob(self, vm, runTimeout, maxOutputFileSize, disableNetwork): os.getenv("DOCKER_TANGO_HOST_VOLUME_PATH"), instanceName ) args = ["docker", "run", "--name", instanceName, "-v"] - args = args + ["%s:%s" % (volumePath, "/home/mount")] + args.append("%s:%s" % (volumePath, "/home/mount")) if vm.cores: - args = args + [f"--cpus={vm.cores}"] + args.append(f"--cpus={vm.cores}") if vm.memory: - args = args + ["-m", f"{vm.memory}m"] + args.extend(("-m", f"{vm.memory}m")) if disableNetwork: - args = args + ["--network", "none"] - args = args + [vm.image] - args = args + ["sh", "-c"] + args.append("--network", "none") + args.append(vm.image) + args.extend(("sh", "-c")) autodriverCmd = ( "autodriver -u %d -f %d -t %d -o %d autolab > output/feedback 2>&1" @@ -176,11 +176,11 @@ def runJob(self, vm, runTimeout, maxOutputFileSize, disableNetwork): ) ) - args = args + [ + args.append( 'cp -r mount/* autolab/; su autolab -c "%s"; \ cp output/feedback mount/feedback' % autodriverCmd - ] + ) self.log.debug("Running job: %s" % str(args)) ret = timeout(args, runTimeout * 2) From bff657d915be7c3d36234d06bf21017d0f6ae955 Mon Sep 17 00:00:00 2001 From: Chaskiel Grundman Date: Thu, 12 Dec 2024 16:43:44 -0500 Subject: [PATCH 2/4] Build docker run args the same in distDocker Make distDocker's runJob similar to localDocker's. Build a list of args instead of assembling nested strings. Make sure to preserve quoting of interior arguments. This has the effect of making the vm core and memory limits effective for distDocker --- vmms/distDocker.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/vmms/distDocker.py b/vmms/distDocker.py index b6d498e5..7ac69422 100644 --- a/vmms/distDocker.py +++ b/vmms/distDocker.py @@ -280,24 +280,20 @@ def runJob(self, vm, runTimeout, maxOutputFileSize, disableNetwork): config.Config.MAX_OUTPUT_FILE_SIZE, ) ) - - # IMPORTANT: The single and double quotes are important, since we - # are switching to the autolab user and then running - # bash commands. - setupCmd = ( - 'cp -r mount/* autolab/; su autolab -c "%s"; \ - cp output/feedback mount/feedback' - % autodriverCmd - ) - - disableNetworkArg = "--network none" if disableNetwork else "" - - args = "(docker run --name %s -v %s:/home/mount %s %s sh -c '%s')" % ( - instanceName, - volumePath, - disableNetworkArg, - vm.image, - setupCmd, + args = ["docker", "run", "--name", instanceName, "-v"] + args.append("%s:%s" % (volumePath, "/home/mount")) + if vm.cores: + args.append(f"--cpus={vm.cores}") + if vm.memory: + args.extend(("-m", f"{vm.memory}m")) + if disableNetwork: + args.append("--network", "none") + + args.append(vm.image) + args.extend(("sh", "-c")) + args.append( + f"\"cp -r mount/* autolab/; su autolab -c '{autodriverCmd}'; \ + cp output/feedback mount/feedback\"" ) self.log.debug("Running job: %s" % args) @@ -306,7 +302,8 @@ def runJob(self, vm, runTimeout, maxOutputFileSize, disableNetwork): ["ssh"] + DistDocker._SSH_FLAGS + vm.ssh_flags - + ["%s@%s" % (self.hostUser, vm.domain_name), args], + + ["%s@%s" % (self.hostUser, vm.domain_name)] + + args, runTimeout * 2, ) From bf8ea3a4ab79a8185b647026db5b64cdd32da3a5 Mon Sep 17 00:00:00 2001 From: Chaskiel Grundman Date: Wed, 18 Dec 2024 11:36:14 -0500 Subject: [PATCH 3/4] Make sure args.append only has one string appended append only takes a single value, make sure the arg is a single string. use f-strings to build string values from multiple components. --- vmms/distDocker.py | 4 ++-- vmms/localDocker.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vmms/distDocker.py b/vmms/distDocker.py index 7ac69422..7a257abf 100644 --- a/vmms/distDocker.py +++ b/vmms/distDocker.py @@ -285,9 +285,9 @@ def runJob(self, vm, runTimeout, maxOutputFileSize, disableNetwork): if vm.cores: args.append(f"--cpus={vm.cores}") if vm.memory: - args.extend(("-m", f"{vm.memory}m")) + args.append(f"--memory={vm.memory}m") if disableNetwork: - args.append("--network", "none") + args.append("--network=none") args.append(vm.image) args.extend(("sh", "-c")) diff --git a/vmms/localDocker.py b/vmms/localDocker.py index 81bd965d..4fb63954 100644 --- a/vmms/localDocker.py +++ b/vmms/localDocker.py @@ -160,9 +160,9 @@ def runJob(self, vm, runTimeout, maxOutputFileSize, disableNetwork): if vm.cores: args.append(f"--cpus={vm.cores}") if vm.memory: - args.extend(("-m", f"{vm.memory}m")) + args.append(f"--memory{vm.memory}m") if disableNetwork: - args.append("--network", "none") + args.append("--network=none") args.append(vm.image) args.extend(("sh", "-c")) From e7d7ee0949c993b5d0eb256f4ff9ec359418c855 Mon Sep 17 00:00:00 2001 From: Chaskiel Grundman Date: Wed, 18 Dec 2024 11:39:22 -0500 Subject: [PATCH 4/4] Build autodriverCmd with fstrings --- vmms/distDocker.py | 17 ++++++++--------- vmms/localDocker.py | 11 ++++------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/vmms/distDocker.py b/vmms/distDocker.py index 7a257abf..12136e2e 100644 --- a/vmms/distDocker.py +++ b/vmms/distDocker.py @@ -271,15 +271,6 @@ def runJob(self, vm, runTimeout, maxOutputFileSize, disableNetwork): self.log.debug("Lost persistent SSH connection") return ret - autodriverCmd = ( - "autodriver -u %d -f %d -t %d -o %d autolab > output/feedback 2>&1" - % ( - config.Config.VM_ULIMIT_USER_PROC, - config.Config.VM_ULIMIT_FILE_SIZE, - runTimeout, - config.Config.MAX_OUTPUT_FILE_SIZE, - ) - ) args = ["docker", "run", "--name", instanceName, "-v"] args.append("%s:%s" % (volumePath, "/home/mount")) if vm.cores: @@ -291,6 +282,14 @@ def runJob(self, vm, runTimeout, maxOutputFileSize, disableNetwork): args.append(vm.image) args.extend(("sh", "-c")) + + autodriverCmd = ( + f"autodriver -u {config.Config.VM_ULIMIT_USER_PROC} " + f"-f {config.Config.VM_ULIMIT_FILE_SIZE} " + f"-t {runTimeout} -o {config.Config.MAX_OUTPUT_FILE_SIZE} " + "autolab > output/feedback 2>&1" + ) + args.append( f"\"cp -r mount/* autolab/; su autolab -c '{autodriverCmd}'; \ cp output/feedback mount/feedback\"" diff --git a/vmms/localDocker.py b/vmms/localDocker.py index 4fb63954..c5bf9313 100644 --- a/vmms/localDocker.py +++ b/vmms/localDocker.py @@ -167,13 +167,10 @@ def runJob(self, vm, runTimeout, maxOutputFileSize, disableNetwork): args.extend(("sh", "-c")) autodriverCmd = ( - "autodriver -u %d -f %d -t %d -o %d autolab > output/feedback 2>&1" - % ( - config.Config.VM_ULIMIT_USER_PROC, - config.Config.VM_ULIMIT_FILE_SIZE, - runTimeout, - config.Config.MAX_OUTPUT_FILE_SIZE, - ) + f"autodriver -u {config.Config.VM_ULIMIT_USER_PROC} " + f"-f {config.Config.VM_ULIMIT_FILE_SIZE} " + f"-t {runTimeout} -o {config.Config.MAX_OUTPUT_FILE_SIZE} " + "autolab > output/feedback 2>&1" ) args.append(