From 23b4fd59a267c40da212811f587c52d78ab88fd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sj=C3=B6lund?= Date: Fri, 14 Aug 2026 19:47:32 +0200 Subject: [PATCH] Limit the memory of a dockerized test job test.py caps the address space of one test at a time, so nothing bounds the sum of the tests it runs in parallel. That is worst for wasm-jit, whose tests are allowed 16 GB of address space each because the JIT reserves ~4 GB of it per wasm memory. Cap the container the job runs in with `docker run --memory=N --memory-swap=N`, N = 85% of the node's RAM, since these jobs are pinned to the high-memory nodes. A cgroup limit charges memory in use rather than address space, covers every process test.py spawns, and on a breach the kernel kills the greediest omc - so a model can now fail because of a model tested in parallel with it, which the log makes visible: the limit is printed before the run and the peak plus the OOM kill count after it. Only the targets that build their own image (wasm-jit today) get this; the jobs running directly on a node are unchanged. Co-authored-by: Claude Opus 5 (1M context) --- .CI/Jenkinsfile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index c8327f8..931bbf0 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -598,6 +598,18 @@ done """ } +/** + * `docker run` flags capping the container's cgroup at 90% of the node's RAM. test.py limits one + * test at a time, not the sum of the parallel ones. --memory-swap has to repeat the limit; docker + * reads 0 as "unset" and then allows swapping. + */ +def memoryLimitArgs() { + def mb = sh(script: '''awk '/^MemTotal:/ { print int($2 / 1024 * 0.85) }' /proc/meminfo''', + returnStdout: true).trim() + echo "Test container memory limit: ${mb} MB, no swap" + return "--memory=${mb}m --memory-swap=${mb}m" +} + /** * Runs `body` with the environment of the shared Rust compile cache of the OpenModelica job * (OpenModelica/.CI/sccache/), which builds the same commits first. Hitting its keys needs the @@ -713,10 +725,17 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla // build, the libraries (HOME during the test) and the ssh key; test.py writes a hash next to // every reference file. rust-cargo-registry is the volume the OpenModelica job uses. def dockerArgs = "--init" + + (image ? " ${memoryLimitArgs()}" : '') + " -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro" + " -v ${env.HOME}:${env.HOME}" + " -v /mnt/ReferenceFiles:/mnt/ReferenceFiles" + " --mount type=volume,source=rust-cargo-registry,target=/opt/rust/cargo/registry" + // Only in a container is the cgroup this job's own; a breach kills the greediest omc, which need + // not be the model that caused it. + def cgroupReport = image ? """ + cat /sys/fs/cgroup/memory.max || true + trap 'cat /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.events || true' EXIT + """ : '' // Jenkins exports the node's environment into the container, hiding the image's. def dockerEnv = ['PATH+VENV=/opt/libtest-venv/bin', 'PATH+CARGO=/opt/rust/cargo/bin', @@ -956,6 +975,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla # too bad if we cannot do it, just continue ln -s -t \${HOME} \${PREVIOUSHOME}/.local .local || true + ${cgroupReport} cd OpenModelicaLibraryTesting # Force /usr/bin/omc as being used for generating the mos-files. Ensures consistent behavior among all tested OMC versions stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1