From 07f981ba25de72082b08d05e0e111d780560043f Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 13 Aug 2026 18:34:06 +0200 Subject: [PATCH 1/2] drop pip cmake from the pytorch base * Install cmake via apt and uninstall the pip package. Its shims in /usr/local/bin shadow /usr/bin/cmake, and they fail with ModuleNotFoundError when a build calls them from inside a pip build isolation environment -- which is why `pip install cellplm` could not build louvain. * Add a test checking that cmake, cpack and ctest resolve to /usr/bin * Bump the version to 1.2.0 --- base_images/CHANGELOG.md | 14 +++++++ base_images/_viash.yaml | 2 +- .../src/pytorch_nvidia/config.vsh.yaml | 8 ++++ base_images/src/pytorch_nvidia/test_cmake.py | 40 +++++++++++++++++++ 4 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 base_images/src/pytorch_nvidia/test_cmake.py diff --git a/base_images/CHANGELOG.md b/base_images/CHANGELOG.md index 9d777bc..fa3bc04 100644 --- a/base_images/CHANGELOG.md +++ b/base_images/CHANGELOG.md @@ -1,3 +1,17 @@ +# OpenProblems Base Images v1.2.0 + +## BUG FIXES + +* `base_pytorch_nvidia`: drop the pip-installed `cmake` in favour of apt's (PR #48). The pip + shims in `/usr/local/bin` shadow `/usr/bin/cmake`, and they fail with `ModuleNotFoundError: + No module named 'cmake'` when a build calls them from inside a pip build isolation + environment -- which broke, among others, building `louvain` for `cellplm`. + +## TESTING + +* Check that `cmake`, `cpack` and `ctest` resolve to the apt-provided binaries in + `base_pytorch_nvidia` (PR #48). + # OpenProblems Base Images v1.1.0 ## MAJOR CHANGES diff --git a/base_images/_viash.yaml b/base_images/_viash.yaml index 8814033..6626333 100644 --- a/base_images/_viash.yaml +++ b/base_images/_viash.yaml @@ -1,5 +1,5 @@ organization: openproblems -version: 1.1.0 +version: 1.2.0 viash_version: 0.9.7 links: repository: https://github.com/openproblems-bio/base_images diff --git a/base_images/src/pytorch_nvidia/config.vsh.yaml b/base_images/src/pytorch_nvidia/config.vsh.yaml index 4290886..6009ccd 100644 --- a/base_images/src/pytorch_nvidia/config.vsh.yaml +++ b/base_images/src/pytorch_nvidia/config.vsh.yaml @@ -3,6 +3,8 @@ description: An nvcr.io pytorch with anndata preinstalled. test_resources: - type: python_script path: test_pytorch.py + - type: python_script + path: test_cmake.py - type: python_script path: ../python/test_anndata.py engines: @@ -13,6 +15,12 @@ engines: packages: - procps - git + - cmake + # The nvcr image pip-installs cmake, whose shims in /usr/local/bin shadow + # apt's cmake because /usr/local/bin comes first on PATH. Drop the package + # so builds pick up the working /usr/bin/cmake instead. + - type: docker + run: pip uninstall -y cmake - type: python packages: - anndata~=0.12.0 diff --git a/base_images/src/pytorch_nvidia/test_cmake.py b/base_images/src/pytorch_nvidia/test_cmake.py new file mode 100644 index 0000000..8ff0d38 --- /dev/null +++ b/base_images/src/pytorch_nvidia/test_cmake.py @@ -0,0 +1,40 @@ +import shutil +import subprocess +import tempfile + +## VIASH START +meta = { + "temp_dir": tempfile.gettempdir(), +} +## VIASH END + +# The nvcr pytorch image ships a pip-installed cmake whose shims land in +# /usr/local/bin and shadow apt's cmake. If a base image rebuild ever +# reintroduces them, components that compile from source break in confusing +# ways, so check that cmake resolves to the apt-provided binary and runs. +print("--- Checking cmake ---", flush=True) + +cmake = shutil.which("cmake") +print(f"cmake resolves to: {cmake}", flush=True) + +assert cmake is not None, "cmake not found on PATH" +assert cmake == "/usr/bin/cmake", ( + f"expected apt's /usr/bin/cmake on PATH, got {cmake} -- a pip-installed " + "cmake is probably shadowing it" +) + +out = subprocess.run([cmake, "--version"], capture_output=True, text=True) +print(out.stdout.strip(), flush=True) + +assert out.returncode == 0, f"`cmake --version` exited {out.returncode}: {out.stderr.strip()}" + +# The pip package ships cpack and ctest alongside cmake; removing only the +# cmake shim would leave those two broken. +for tool in ["cpack", "ctest"]: + path = shutil.which(tool) + print(f"{tool} resolves to: {path}", flush=True) + assert path is None or path.startswith("/usr/bin/"), ( + f"{tool} resolves to {path}, expected /usr/bin/{tool} or nothing" + ) + +print("\ncmake test passed!", flush=True) From 536aaed837b199ac300ee70b8a3a5af3a004ce9a Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Thu, 13 Aug 2026 18:34:58 +0200 Subject: [PATCH 2/2] rebuild the tensorflow base on python:3.12 nvcr.io stopped publishing tensorflow images after 25.02, which left this image stuck on tensorflow 2.17, numpy 1.26 and scanpy 1.10. * Build on the same python:3.12 base as base_python, and take tensorflow from the and-cuda extra so the CUDA runtimes are pinned to whatever the wheel was built against * Put those CUDA libraries on the linker path with ldconfig. Tensorflow does not look under site-packages/nvidia/*/lib by itself, so without this it finds no GPU and quietly trains on the CPU. * Bump tensorflow 2.17 -> 2.21, numpy 1.26 -> 2.x, scanpy 1.10 -> 1.12 * Check the CUDA libraries are on the linker path in the test -- there is no GPU in CI, but the linker can be asked whether it would find them Verified on a Tesla T4 on the de.NBI cluster: tensorflow finds the GPU, matmul and a keras training step run on it. --- base_images/CHANGELOG.md | 20 ++++++++++++++++ .../src/tensorflow_nvidia/config.vsh.yaml | 24 +++++++++++++++---- .../src/tensorflow_nvidia/test_tensorflow.py | 9 +++++++ 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/base_images/CHANGELOG.md b/base_images/CHANGELOG.md index fa3bc04..164adef 100644 --- a/base_images/CHANGELOG.md +++ b/base_images/CHANGELOG.md @@ -1,7 +1,23 @@ # OpenProblems Base Images v1.2.0 +## MAJOR CHANGES + +* Rebuild `base_tensorflow_nvidia` on `python:3.12` with `tensorflow[and-cuda]` (PR #49). + nvcr.io stopped publishing TensorFlow images after 25.02, which left the image stuck on + TensorFlow 2.17, NumPy 1.26 and Scanpy 1.10. Taking CUDA from the `and-cuda` extra pins it + to whatever the TensorFlow wheel was built against, so the two can no longer drift apart + and silently fall back to the CPU. + +* Bump TensorFlow from 2.17 to 2.21, NumPy from 1.26 to 2.x and Scanpy from 1.10 to 1.12 in + `base_tensorflow_nvidia` (PR #49). + ## BUG FIXES +* `base_tensorflow_nvidia`: put the CUDA libraries that the `and-cuda` extra installs under + `site-packages/nvidia/*/lib` on the linker path with `ldconfig` (PR #49). TensorFlow does + not look there by itself, so without this it reports `Cannot dlopen some GPU libraries`, + finds no GPU and quietly trains on the CPU. + * `base_pytorch_nvidia`: drop the pip-installed `cmake` in favour of apt's (PR #48). The pip shims in `/usr/local/bin` shadow `/usr/bin/cmake`, and they fail with `ModuleNotFoundError: No module named 'cmake'` when a build calls them from inside a pip build isolation @@ -12,6 +28,10 @@ * Check that `cmake`, `cpack` and `ctest` resolve to the apt-provided binaries in `base_pytorch_nvidia` (PR #48). +* Check that the CUDA libraries are on the linker path in `base_tensorflow_nvidia` (PR #49). + There is no GPU in CI, but the linker can be asked whether it would find them, which is + enough to catch the failure above. + # OpenProblems Base Images v1.1.0 ## MAJOR CHANGES diff --git a/base_images/src/tensorflow_nvidia/config.vsh.yaml b/base_images/src/tensorflow_nvidia/config.vsh.yaml index 9eed723..23551e7 100644 --- a/base_images/src/tensorflow_nvidia/config.vsh.yaml +++ b/base_images/src/tensorflow_nvidia/config.vsh.yaml @@ -1,5 +1,13 @@ name: base_tensorflow_nvidia -description: An nvcr.io tensorflow with anndata preinstalled. +description: | + A Python 3.12 image with GPU-enabled TensorFlow and anndata preinstalled. + + nvcr.io stopped publishing TensorFlow images after 25.02, so this image is + built on the same `python:3.12` base as `base_python`. The CUDA and cuDNN + runtimes come from the `and-cuda` extra, which pins them to whatever the + TensorFlow wheel was built against -- installing TensorFlow on a base image + that ships its own CUDA lets the two drift apart and silently fall back to + the CPU. test_resources: - type: python_script path: test_tensorflow.py @@ -7,7 +15,7 @@ test_resources: path: ../python/test_anndata.py engines: - type: docker - image: nvcr.io/nvidia/tensorflow:25.02-tf2-py3 + image: python:3.12 setup: - type: apt packages: @@ -15,11 +23,19 @@ engines: - git - type: python packages: - - numpy~=1.26.0 + - tensorflow[and-cuda]~=2.21.0 - anndata~=0.12.0 - - scanpy~=1.10.0 + - scanpy~=1.12 - pyyaml - requests - jsonschema github: - openproblems-bio/core#subdirectory=packages/python/openproblems + # TensorFlow does not find the CUDA libraries that the `and-cuda` extra + # installs under `site-packages/nvidia/*/lib`, so put them on the linker + # path. Without this it silently falls back to the CPU. + - type: docker + run: | + find "$(python -c 'import nvidia, os; print(os.path.dirname(nvidia.__file__))')" -maxdepth 2 -name lib -type d > /etc/ld.so.conf.d/nvidia-pip.conf && \ + test -s /etc/ld.so.conf.d/nvidia-pip.conf && \ + ldconfig diff --git a/base_images/src/tensorflow_nvidia/test_tensorflow.py b/base_images/src/tensorflow_nvidia/test_tensorflow.py index 92e2eb9..8874302 100644 --- a/base_images/src/tensorflow_nvidia/test_tensorflow.py +++ b/base_images/src/tensorflow_nvidia/test_tensorflow.py @@ -1,5 +1,6 @@ import tensorflow as tf import numpy as np +import ctypes import tempfile import os @@ -14,6 +15,14 @@ print(f"Using TensorFlow version: {tf.__version__}", flush=True) +# The CUDA libraries live in site-packages/nvidia/*/lib, which is not on the +# linker path by default. There is no GPU in CI to catch that, but the linker +# can be asked whether it would find them. +print("\n--- Checking that the CUDA libraries are on the linker path ---", flush=True) +for soname in ["libcudnn.so.9", "libcublas.so.12"]: + ctypes.CDLL(soname) + print(f" - {soname} loaded", flush=True) + # Check for and list available physical devices (CPU/GPU) gpus = tf.config.list_physical_devices('GPU') if gpus: