From 599a0d3fe44392eb86cb073cb8daa8d32759f3fd Mon Sep 17 00:00:00 2001 From: Juan Carlos Graciosa Date: Wed, 9 Sep 2026 18:52:43 +1000 Subject: [PATCH 1/8] ci(container): automate the Gadi Singularity image builds Adds two workflows for docs/developer/gadi_singularity/, which has had no CI since #133 and has only ever been built by hand: - gadi-uw3-image.yml builds underworld3.rhel on pushes to main/development, v* tags and releases, publishing underworld3-gadi:. - gadi-petsc-image.yml builds petsc.rhel, triggered only by that file and petsc-custom/patches/**, publishing petsc-gadi:-ompi. Both are amd64-only and publish Docker v2 media types with buildx attestations disabled, which is what Gadi's Singularity pulls reliably. Image names derive from github.repository_owner so the files work unchanged in a fork. Each build is smoke-tested, then old untagged versions are pruned (keeping 3, tagged exempt) only if that test passed. Fixes five pre-existing problems the automation exposed, none of them visible when building by hand on macOS: - underworld3.rhel omitted requests, which utilities/_utils.py imports unguarded at module level, so import underworld3 raised ModuleNotFoundError. - underworld3.rhel:22 was missing a leading '#'. podman continues a comment across a trailing backslash; BuildKit read it as an instruction. - setuptools-scm was absent from the builder, so with --no-build-isolation the dynamic version never resolved and the image reported 0.0.0. - petsc.rhel pinned cython>=3.1 unbounded. petsc4py 3.25.0's PC.pyx:1256 does not cythonize under 3.3.0; bisected, 3.2.9 is the last that builds. - ARGs re-declared after FROM carried no defaults, so PYVER could expand empty and yum would look for python-pip. Also adds ARG UW3_REPO, since the clone URL was hardcoded and a fork could not build its own branches, and ARG PETSC_MAKE_NP. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/gadi-petsc-image.yml | 171 +++++++++++++ .github/workflows/gadi-uw3-image.yml | 230 ++++++++++++++++++ docs/developer/gadi_singularity/README.md | 57 ++++- docs/developer/gadi_singularity/petsc.rhel | 22 +- .../gadi_singularity/underworld3.rhel | 32 ++- 5 files changed, 494 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/gadi-petsc-image.yml create mode 100644 .github/workflows/gadi-uw3-image.yml diff --git a/.github/workflows/gadi-petsc-image.yml b/.github/workflows/gadi-petsc-image.yml new file mode 100644 index 000000000..ccaeffb9b --- /dev/null +++ b/.github/workflows/gadi-petsc-image.yml @@ -0,0 +1,171 @@ +name: "GHCR: Gadi PETSc Base Image" + +# Builds docs/developer/gadi_singularity/petsc.rhel — the from-source PETSc base +# that gadi-uw3-image.yml layers on. +# +# Triggers are deliberately narrow: this compiles PETSc with mumps, hypre, +# superlu_dist, mmg/parmmg, ptscotch and slepc, a build measured in hours, and +# only petsc.rhel and its patches can change the result. + +on: + push: + branches: + - main + - development + paths: + - 'docs/developer/gadi_singularity/petsc.rhel' + - 'petsc-custom/patches/**' + - '.github/workflows/gadi-petsc-image.yml' + workflow_dispatch: + inputs: + petsc_version: + description: 'PETSc version to build' + type: string + default: '3.25.0' + make_np: + description: 'PETSc build parallelism (--with-make-np)' + type: string + default: '4' + force_rebuild: + description: 'Force full rebuild (no cache)' + type: boolean + default: false + +concurrency: + group: gadi-petsc-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-and-push: + runs-on: ubuntu-latest + # Measured at ~25 min with ~25% run-to-run variance, so this is ~5x headroom + # for a heavier future PETSc while still failing a hang in hours, not a day. + timeout-minutes: 120 + permissions: + contents: read + packages: write + outputs: + image: ${{ steps.meta.outputs.image }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Measured: the whole build consumes ~7 GB against 86 GB free on a fresh + # runner, so no space needs reclaiming. Kept as instrumentation in case a + # future PETSc pulls in far more --download- packages. + - name: Report disk headroom + run: df -h / + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve names and tags + id: meta + run: | + set -euo pipefail + + OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + IMAGE="ghcr.io/${OWNER}/petsc-gadi" + + PETSC_VERSION="${{ inputs.petsc_version }}" + PETSC_VERSION="${PETSC_VERSION:-3.25.0}" + MAKE_NP="${{ inputs.make_np }}" + MAKE_NP="${MAKE_NP:-4}" + + { + echo "image=${IMAGE}:${PETSC_VERSION}-ompi" + echo "tags=${IMAGE}:${PETSC_VERSION}-ompi,${IMAGE}:latest" + echo "petsc_version=${PETSC_VERSION}" + echo "make_np=${MAKE_NP}" + } >> "$GITHUB_OUTPUT" + + echo "Building ${IMAGE}:${PETSC_VERSION}-ompi with --with-make-np=${MAKE_NP}" + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: docs/developer/gadi_singularity/petsc.rhel + # Docker v2 media types — see the note in gadi-uw3-image.yml. + outputs: type=image,push=true,oci-mediatypes=false + platforms: linux/amd64 + # See the note in gadi-uw3-image.yml — attestations break some + # `singularity pull` versions. + provenance: false + sbom: false + no-cache: ${{ inputs.force_rebuild || false }} + build-args: | + PYTHON_VERSION=3.12 + PETSC_VERSION=${{ steps.meta.outputs.petsc_version }} + PETSC_MAKE_NP=${{ steps.meta.outputs.make_np }} + tags: ${{ steps.meta.outputs.tags }} + + - name: Report build size + run: | + docker image ls ${{ steps.meta.outputs.image }} || true + df -h / + + smoke-test: + needs: build-and-push + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + packages: read + steps: + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull image + run: docker pull ${{ needs.build-and-push.outputs.image }} + + # petsc4py importing is the thing that actually breaks when the PETSc build + # goes subtly wrong (shared libraries, PYTHONPATH, the meson petsc4py step). + - name: Import petsc4py and slepc4py + run: | + docker run --rm ${{ needs.build-and-push.outputs.image }} python -c " + import petsc4py, slepc4py + petsc4py.init() + from petsc4py import PETSc + print('petsc4py', petsc4py.__version__) + print('slepc4py', slepc4py.__version__) + print('PETSc', PETSc.Sys.getVersion()) + " + + - name: MPI sanity (2 ranks) + run: | + docker run --rm ${{ needs.build-and-push.outputs.image }} \ + mpiexec -n 2 python -c " + from mpi4py import MPI + c = MPI.COMM_WORLD + print(f'rank {c.rank} of {c.size}') + assert c.size == 2 + " + + cleanup: + needs: [build-and-push, smoke-test] + runs-on: ubuntu-latest + permissions: + packages: write + steps: + # Housekeeping only: see the note in gadi-uw3-image.yml. + - name: Delete old untagged image versions + continue-on-error: true + uses: actions/delete-package-versions@v5 + with: + owner: ${{ github.repository_owner }} + package-name: petsc-gadi + package-type: container + min-versions-to-keep: 2 + delete-only-untagged-versions: true diff --git a/.github/workflows/gadi-uw3-image.yml b/.github/workflows/gadi-uw3-image.yml new file mode 100644 index 000000000..db9c97ea3 --- /dev/null +++ b/.github/workflows/gadi-uw3-image.yml @@ -0,0 +1,230 @@ +name: "GHCR: Gadi Singularity Image" + +# Builds docs/developer/gadi_singularity/underworld3.rhel and pushes to GHCR for +# `singularity pull docker://...` on Gadi. +# +# This is the layer that moves; the expensive PETSc base under it is built rarely +# by gadi-petsc-image.yml. Unlike docker-image.yaml and binder-image.yml, this +# image is Rocky Linux 8.10 + from-source PETSc, to match Gadi's ABI. + +on: + push: + branches: + - main + - development + tags: ['v*'] + paths: + - 'docs/developer/gadi_singularity/underworld3.rhel' + - '.github/workflows/gadi-uw3-image.yml' + - 'pixi.toml' + - 'pyproject.toml' + - 'setup.py' + - 'src/**' + # Releases are the guaranteed path for a tagged image: the `tags:` filter above + # is combined with `paths:`, so a tag whose diff touches nothing in that list + # would not build on its own. + release: + types: [published] + workflow_dispatch: + inputs: + uw3_branch: + description: 'UW3 branch/tag to clone and build (e.g. development, v3.1.0). Empty = triggering ref.' + type: string + default: '' + petsc_image: + description: 'PETSc base image. Empty = ghcr.io//petsc-gadi:3.25.0-ompi' + type: string + default: '' + image_tag: + description: 'Image tag override (e.g. ci-test). Empty = derive from ref.' + type: string + default: '' + force_rebuild: + description: 'Force full rebuild (no cache)' + type: boolean + default: false + +concurrency: + group: gadi-uw3-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-and-push: + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + packages: write + outputs: + image: ${{ steps.meta.outputs.image }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve names and tags + id: meta + run: | + set -euo pipefail + + # Image names must be lowercase; repository_owner may not be. + OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + IMAGE="ghcr.io/${OWNER}/underworld3-gadi" + + # Same ref-name / IS_RELEASE convention as binder-image.yml. + if [[ "$GITHUB_REF" == refs/tags/* ]]; then + REF_NAME=${GITHUB_REF#refs/tags/} + IS_RELEASE=true + else + REF_NAME=${GITHUB_REF#refs/heads/} + IS_RELEASE=false + fi + + if [[ -n "${{ inputs.uw3_branch }}" ]]; then + REF_NAME="${{ inputs.uw3_branch }}" + [[ "$REF_NAME" == v* ]] && IS_RELEASE=true + fi + if [[ -n "${{ inputs.image_tag }}" ]]; then + DOCKER_TAG="${{ inputs.image_tag }}" + else + # Docker tags cannot contain '/' + DOCKER_TAG=$(echo "$REF_NAME" | tr '/' '-') + fi + + # Dispatch input wins, then the GADI_PETSC_IMAGE repo variable (lets a + # fork reuse a base image it already published under another name), + # then the canonical name. + PETSC_IMAGE="${{ inputs.petsc_image }}" + PETSC_IMAGE="${PETSC_IMAGE:-${{ vars.GADI_PETSC_IMAGE }}}" + PETSC_IMAGE="${PETSC_IMAGE:-ghcr.io/${OWNER}/petsc-gadi:3.25.0-ompi}" + + TAGS="${IMAGE}:${DOCKER_TAG}" + if [[ "$IS_RELEASE" == "true" ]]; then + TAGS="${TAGS},${IMAGE}:latest" + fi + + # A release tag is the version; anything else gets a PEP440 dev + # version carrying the commit, since branch names are not versions. + if [[ "$IS_RELEASE" == "true" ]]; then + UW3_VERSION="${REF_NAME#v}" + else + UW3_VERSION="0.0.0.dev0+g${GITHUB_SHA:0:7}" + fi + + { + echo "image=${IMAGE}:${DOCKER_TAG}" + echo "tags=${TAGS}" + echo "uw3_branch=${REF_NAME}" + echo "petsc_image=${PETSC_IMAGE}" + echo "uw3_version=${UW3_VERSION}" + } >> "$GITHUB_OUTPUT" + + echo "Building ${IMAGE}:${DOCKER_TAG}" + echo " UW3 ref: ${REF_NAME}" + echo " PETSc base: ${PETSC_IMAGE}" + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: docs/developer/gadi_singularity/underworld3.rhel + # Docker v2 media types rather than buildx's OCI default, matching the + # podman `--format docker` images already validated on Gadi. Older + # Singularity builds read OCI patchily; this removes the variable. + outputs: type=image,push=true,oci-mediatypes=false + # Gadi is x86_64. No arm64: it would need QEMU for no benefit, and the + # gmsh / vtk-osmesa wheels this image needs are x86_64-only anyway. + platforms: linux/amd64 + # Attestations turn the result into a multi-manifest index containing an + # "unknown/unknown" entry, which some Singularity/Apptainer versions + # refuse to pull. Keep these off — see README "Automated builds". + provenance: false + sbom: false + no-cache: ${{ inputs.force_rebuild || false }} + build-args: | + PYTHON_VERSION=3.12 + UW3_BRANCH=${{ steps.meta.outputs.uw3_branch }} + UW3_REPO=${{ github.server_url }}/${{ github.repository }}.git + PETSC_IMAGE=${{ steps.meta.outputs.petsc_image }} + SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.meta.outputs.uw3_version }} + tags: ${{ steps.meta.outputs.tags }} + # Cache lives in its own package: sharing one with the image put stale + # cache manifests into the same untagged pool the cleanup job prunes, + # so they competed with old images for min-versions-to-keep. + cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/underworld3-gadi-buildcache:latest + cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/underworld3-gadi-buildcache:latest,mode=max + + smoke-test: + needs: build-and-push + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + packages: read + steps: + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull image + run: docker pull ${{ needs.build-and-push.outputs.image }} + + # A bare import is the cheapest check and catches a missing runtime dep + # (this is exactly how the absent `requests` would have shown up). + - name: Import underworld3 + run: | + docker run --rm ${{ needs.build-and-push.outputs.image }} \ + python -c "import underworld3; print('underworld3', underworld3.__version__)" + + # The Containerfile installs gmsh and vtk-osmesa behind `|| echo skip`, so a + # failed install is silent. Import them explicitly or it ships broken. + - name: Import optional stack (gmsh, vtk, pyvista, h5py, petsc4py) + run: | + docker run --rm ${{ needs.build-and-push.outputs.image }} \ + python -c "import gmsh, vtk, pyvista, h5py, petsc4py; print('optional stack OK')" + + - name: MPI sanity (2 ranks) + run: | + docker run --rm ${{ needs.build-and-push.outputs.image }} \ + mpiexec -n 2 python -c " + import underworld3 as uw + from mpi4py import MPI + c = MPI.COMM_WORLD + assert c.size == 2 + print(f'rank {c.rank} of {c.size}, underworld3 {uw.__version__}') + " + + cleanup: + # Only prune once the image is known good — otherwise a broken build would + # delete the last working one. + needs: [build-and-push, smoke-test] + runs-on: ubuntu-latest + permissions: + packages: write + steps: + # Housekeeping only: a failed prune must not fail a run whose image is + # good. The action 404s partway through on user-owned (vs org-owned) + # packages, having already deleted what it could. + - name: Delete old untagged image versions + continue-on-error: true + uses: actions/delete-package-versions@v5 + with: + owner: ${{ github.repository_owner }} + package-name: underworld3-gadi + package-type: container + min-versions-to-keep: 3 + # Load-bearing: every TAGGED image (:latest, :v3.1.0, :main, and the + # current :development) is exempt regardless of age. Only superseded + # rolling builds are eligible. + delete-only-untagged-versions: true diff --git a/docs/developer/gadi_singularity/README.md b/docs/developer/gadi_singularity/README.md index 6b579a451..46a00453f 100644 --- a/docs/developer/gadi_singularity/README.md +++ b/docs/developer/gadi_singularity/README.md @@ -4,6 +4,57 @@ This directory contains two Containerfiles to build the Underworld3 (UW3) Singul Both use Rocky Linux 8.10 to match Gadi's OS for ABI compatibility. +## Automated builds + +Both images are built by GitHub Actions; the manual steps below are only needed for +local iteration or if you are building outside the `underworldcode` org. + +| Workflow | Builds | Publishes | Runs when | +|---|---|---|---| +| `.github/workflows/gadi-uw3-image.yml` | `underworld3.rhel` | `ghcr.io//underworld3-gadi:` (`:latest` on release) | push to `main`/`development` touching `src/**`, `pixi.toml`, `pyproject.toml`, `setup.py` or this directory; published releases; manual dispatch | +| `.github/workflows/gadi-petsc-image.yml` | `petsc.rhel` | `ghcr.io//petsc-gadi:-ompi` (+ `:latest`) | push touching `petsc.rhel` or `petsc-custom/patches/**`; manual dispatch | + +PETSc is a ~25 min from-source build, so it is kept as a separate, rarely-triggered +workflow and the published image is reused as the base for every Underworld3 build. +To rebuild it explicitly: + +```bash +gh workflow run gadi-petsc-image.yml -f petsc_version=3.25.0 -f make_np=4 +``` + +To build the Underworld3 image from a specific ref: + +```bash +gh workflow run gadi-uw3-image.yml -f uw3_branch=v3.1.0 -f image_tag=v3.1.0 +``` + +Each build is followed by a smoke test: `import underworld3`, an import check for the +optional gmsh/vtk-osmesa stack that the Containerfile installs behind `|| echo skip`, +and a 2-rank MPI check. + +### Retention + +Only the **three most recent untagged** `underworld3-gadi` versions are kept (two for +`petsc-gadi`); older ones are pruned automatically after the smoke test passes. Tagged +images — `:latest`, `:main`, `:development`, and every `v*` release — are never pruned. + +### Package visibility + +A package published by these workflows inherits the repository's visibility, so on a +public repo both images are publicly pullable and Gadi compute nodes — which have no +GitHub credentials — can `singularity pull` them directly. A package created by hand +(`podman push` from a laptop) is **not** linked to the repository and does default to +private; set it public under its package settings, or authenticate the pull: + +```bash +export SINGULARITY_DOCKER_USERNAME= +export SINGULARITY_DOCKER_PASSWORD= +``` + +Note the workflows build single-platform `linux/amd64` images with buildx attestations +disabled (`provenance: false`, `sbom: false`). Attestations add an `unknown/unknown` +entry to the manifest index that some Singularity/Apptainer versions refuse to pull. + ## Build Order Build commands must be run from the top-level `underworld3/` directory (the build context). @@ -15,14 +66,14 @@ Builds targeting Gadi must use `--platform linux/amd64`. podman build . \ --platform linux/amd64 \ --format docker \ - -t ghcr.io//petsc:3.25.0-ompi \ + -t ghcr.io//petsc-gadi:3.25.0-ompi \ -f ./docs/developer/gadi_singularity/petsc.rhel ``` ### 2. Push PETSc image to registry ```bash -podman push ghcr.io//petsc:3.25.0-ompi +podman push ghcr.io//petsc-gadi:3.25.0-ompi ``` ### 3. Build Underworld3 @@ -31,7 +82,7 @@ podman push ghcr.io//petsc:3.25.0-ompi podman build . \ --platform linux/amd64 \ --format docker \ - --build-arg PETSC_IMAGE=ghcr.io//petsc:3.25.0-ompi \ + --build-arg PETSC_IMAGE=ghcr.io//petsc-gadi:3.25.0-ompi \ --build-arg UW3_BRANCH=development \ -t ghcr.io//underworld3-gadi:latest \ -f ./docs/developer/gadi_singularity/underworld3.rhel diff --git a/docs/developer/gadi_singularity/petsc.rhel b/docs/developer/gadi_singularity/petsc.rhel index e612826dc..a13ead632 100644 --- a/docs/developer/gadi_singularity/petsc.rhel +++ b/docs/developer/gadi_singularity/petsc.rhel @@ -36,8 +36,10 @@ ARG BASE_IMAGE="quay.io/rockylinux/rockylinux:8.10" FROM ${BASE_IMAGE} as runtime LABEL maintainer="https://github.com/underworldcode/" -# need to repeat ARGS after every FROM -ARG PYTHON_VERSION +# Repeat the DEFAULT, not just the name: bare `ARG PYTHON_VERSION` relies on +# inheriting the pre-FROM default, which some buildah/podman versions don't — +# PYVER expands empty and yum then looks for a package called "python-pip". +ARG PYTHON_VERSION="3.12" #### Containerfile ENV vars - for all image stages ENV LANG=C.UTF-8 @@ -85,8 +87,13 @@ RUN python${PYVER} -m pip install wheel \ # 2. Define the builder layer FROM runtime as builder -ARG PETSC_VERSION -ARG PYTHON_VERSION +# defaults repeated deliberately — see the note in the runtime stage +ARG PETSC_VERSION="3.25.0" +ARG PYTHON_VERSION="3.12" +# Parallelism for PETSc's own build. 2 is safe inside a memory-constrained podman +# machine (compiling PETSc.c is memory-heavy and OOMs at higher values); CI on a +# native runner passes 4. +ARG PETSC_MAKE_NP="2" RUN yum install -y \ ca-certificates \ @@ -108,7 +115,10 @@ RUN yum install -y \ && rm -rf /var/cache/yum # NOTE flex and bison are needed to build -RUN python${PYVER} -m pip install "cython>=3.1" \ +# Upper bound is load-bearing: petsc4py 3.25.0 declares only "cython >= 3" but +# PC.pyx:1256 fails to cythonize under 3.3.0 ("Invalid index type 'int'"). +# Bisected against the 3.25.0 source: 3.2.9 is the last version that builds. +RUN python${PYVER} -m pip install "cython>=3.1,<3.3" \ "setuptools>=75" \ "meson" \ "meson-python" \ @@ -165,7 +175,7 @@ RUN python${PYVER} ./configure \ --download-zlib=1 \ --download-ctetgen=1 \ --download-triangle=1 \ - --with-make-np=2 + --with-make-np=${PETSC_MAKE_NP} RUN make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt all RUN make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt install \ || (echo "=== petsc4py build log ===" && \ diff --git a/docs/developer/gadi_singularity/underworld3.rhel b/docs/developer/gadi_singularity/underworld3.rhel index 21a712f30..e865576e4 100644 --- a/docs/developer/gadi_singularity/underworld3.rhel +++ b/docs/developer/gadi_singularity/underworld3.rhel @@ -19,7 +19,7 @@ # # To build use podman from the top level underworld didrectory. i.e. # $ podman build . \ - --platform linux/amd64 \ +# --platform linux/amd64 \ # --format docker \ # --build-arg UW3_BRANCH=xxx \ # -t new_image_name \ @@ -31,8 +31,11 @@ # https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact ARG PYTHON_VERSION="3.12" ARG BASE_IMAGE="quay.io/rockylinux/rockylinux:8.10" -ARG PETSC_IMAGE="ghcr.io/jcgraciosa/petsc:3.25.0-ompi" +ARG PETSC_IMAGE="ghcr.io/underworldcode/petsc-gadi:3.25.0-ompi" ARG UW3_BRANCH="development" +# Overridden by CI to the repository being built, so a fork builds its own +# branches rather than looking for them upstream. +ARG UW3_REPO="https://github.com/underworldcode/underworld3.git" # 'petsc-image' will be used later on in builder stage COPY command FROM ${PETSC_IMAGE} as petsc-image @@ -43,8 +46,10 @@ FROM ${PETSC_IMAGE} as petsc-image FROM ${BASE_IMAGE} as runtime LABEL maintainer="https://github.com/underworldcode/" -# need to repeat ARGS after every FROM -ARG PYTHON_VERSION +# Repeat the DEFAULT, not just the name: bare `ARG PYTHON_VERSION` relies on +# inheriting the pre-FROM default, which some buildah/podman versions don't — +# PYVER expands empty and yum then looks for a package called "python-pip". +ARG PYTHON_VERSION="3.12" #### Containerfile ENV vars - for all image stages ENV LANG=C.UTF-8 @@ -102,8 +107,11 @@ ENV PYTHONPATH=$PYTHONPATH:$PETSC_DIR/lib # Stage 2: 'builder' #################### FROM petsc-image as builder -ARG PYTHON_VERSION -ARG UW3_BRANCH +# defaults repeated deliberately — see the note in the runtime stage +ARG PYTHON_VERSION="3.12" +ARG UW3_BRANCH="development" +ARG UW3_REPO="https://github.com/underworldcode/underworld3.git" +ARG SETUPTOOLS_SCM_PRETEND_VERSION="" # root to install with yum USER root @@ -123,6 +131,7 @@ RUN yum install -y\ # install python build time and runtime requirements here RUN python${PYVER} -m pip install --no-cache-dir \ "setuptools>=75" \ + "setuptools-scm>=8" \ "cython>=3.1" \ "scipy>=1.15" \ "numpy<2" \ @@ -136,9 +145,10 @@ RUN python${PYVER} -m pip install --no-cache-dir \ psutil \ typing_extensions \ xxhash \ + requests \ trimesh \ ipython \ - jupyterlab \ + "jupyterlab>=4.3,<5" \ "ipywidgets<9.0.0" \ jupyter-server-proxy \ "trame>=2.5.2" \ @@ -168,9 +178,13 @@ RUN pip install --no-cache-dir --extra-index-url https://wheels.vtk.org vtk-osme # Clone and install uw3 RUN git clone --depth 1 --branch ${UW3_BRANCH} \ - https://github.com/underworldcode/underworld3.git /tmp/underworld3 + ${UW3_REPO} /tmp/underworld3 WORKDIR /tmp/underworld3 -RUN pip install --no-build-isolation --no-cache-dir . +# The clone above is shallow and forks carry no tags, so setuptools-scm has +# nothing to derive from and would stamp the build 0.0.0. CI passes the real +# version; unset when empty so a local build keeps setuptools-scm's own logic. +RUN [ -n "$SETUPTOOLS_SCM_PRETEND_VERSION" ] || unset SETUPTOOLS_SCM_PRETEND_VERSION; \ + pip install --no-build-isolation --no-cache-dir . # record 'builder' stage packages used RUN python${PYVER} -m pip freeze >/opt/requirements.txt \ From 5229bf6ac2251e79492a15d1fc89367b3f2f8ecc Mon Sep 17 00:00:00 2001 From: Juan Carlos Graciosa Date: Thu, 17 Sep 2026 13:11:17 +1000 Subject: [PATCH 2/8] Add gcc-c++ to the runtime image so UW3 can be rebuilt in place --- docs/developer/gadi_singularity/underworld3.rhel | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/developer/gadi_singularity/underworld3.rhel b/docs/developer/gadi_singularity/underworld3.rhel index e865576e4..69ba58ba1 100644 --- a/docs/developer/gadi_singularity/underworld3.rhel +++ b/docs/developer/gadi_singularity/underworld3.rhel @@ -60,7 +60,10 @@ ENV NB_USER jovyan ENV NB_HOME /home/$NB_USER RUN useradd -m -s /bin/bash -N $NB_USER -# runtime packages - [vim, git] are optional +# runtime packages - [vim, git] are optional. +# gcc-c++ looks out of place in a runtime image but is deliberate: it lets a +# developer rebuild UW3 in place on an HPC system. underworld3.ckdtree is a C++ +# extension, and gcc alone (pulled in by python-devel) cannot build it. RUN yum update -y \ && yum install -y \ ca-certificates \ @@ -70,6 +73,7 @@ RUN yum update -y \ openmpi \ python${PYVER}-pip \ python${PYVER}-devel \ + gcc-c++ \ zlib \ mesa-libGL \ mesa-libGLU \ From 374ce0ebe54fe769bdeb135818d8fe20f7ba6457 Mon Sep 17 00:00:00 2001 From: Juan Carlos Graciosa Date: Thu, 17 Sep 2026 20:18:30 +1000 Subject: [PATCH 3/8] ci(container): add per-platform base images (Rocky+OpenMPI, Ubuntu+MPICH) --- .github/workflows/hpc-base-images.yml | 113 ++++++++++++++++++ docs/developer/hpc_containers/base.rocky-ompi | 91 ++++++++++++++ .../hpc_containers/base.ubuntu-mpich | 87 ++++++++++++++ 3 files changed, 291 insertions(+) create mode 100644 .github/workflows/hpc-base-images.yml create mode 100644 docs/developer/hpc_containers/base.rocky-ompi create mode 100644 docs/developer/hpc_containers/base.ubuntu-mpich diff --git a/.github/workflows/hpc-base-images.yml b/.github/workflows/hpc-base-images.yml new file mode 100644 index 000000000..9fd877d75 --- /dev/null +++ b/.github/workflows/hpc-base-images.yml @@ -0,0 +1,113 @@ +name: "GHCR: HPC platform base images" + +# Builds docs/developer/hpc_containers/base.* — one image per platform family, +# published as :runtime and :builder tags. Everything OS/MPI-specific lives in +# these; hpc-petsc-image.yml and hpc-uw3-image.yml stack on them. +# +# Changes rarely. Both downstream workflows pull the tags by name, so after a +# base change rebuild PETSc (hpc-petsc-image.yml), then UW3. + +on: + push: + branches: + - main + - development + - ci/gadi-container # TESTING ONLY — remove before the upstream PR + paths: + - 'docs/developer/hpc_containers/base.*' + - '.github/workflows/hpc-base-images.yml' + workflow_dispatch: + inputs: + force_rebuild: + description: 'Force full rebuild (no cache)' + type: boolean + default: false + +concurrency: + group: hpc-base-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + base: [rocky-ompi, ubuntu-mpich] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve image name + id: meta + run: | + OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + echo "image=ghcr.io/${OWNER}/uw3-base-${{ matrix.base }}" >> "$GITHUB_OUTPUT" + + # Two targets of one file, pushed as two tags. Docker v2 media types and + # no attestations, for the same Singularity reasons as hpc-uw3-image.yml. + - name: Build and push :runtime + uses: docker/build-push-action@v6 + with: + context: . + file: docs/developer/hpc_containers/base.${{ matrix.base }} + target: runtime + outputs: type=image,push=true,oci-mediatypes=false + platforms: linux/amd64 + provenance: false + sbom: false + no-cache: ${{ inputs.force_rebuild || false }} + build-args: PYTHON_VERSION=3.12 + tags: ${{ steps.meta.outputs.image }}:runtime + + - name: Build and push :builder + uses: docker/build-push-action@v6 + with: + context: . + file: docs/developer/hpc_containers/base.${{ matrix.base }} + target: builder + outputs: type=image,push=true,oci-mediatypes=false + platforms: linux/amd64 + provenance: false + sbom: false + no-cache: ${{ inputs.force_rebuild || false }} + build-args: PYTHON_VERSION=3.12 + tags: ${{ steps.meta.outputs.image }}:builder + + # The contract the recipes rely on: mpicc and a venv python on PATH. + # command -v, not which: Rocky's minimal image has no `which`. + - name: Smoke test + run: | + docker run --rm ${{ steps.meta.outputs.image }}:builder bash -c ' + set -e + for t in mpicc g++ gfortran cmake flex bison patch git wget; do command -v $t >/dev/null || { echo "missing $t"; exit 1; }; done + mpicc --version | head -1 + python -c "import sys; assert sys.prefix == \"/opt/venv\", sys.prefix; print(sys.version)"' + docker run --rm ${{ steps.meta.outputs.image }}:runtime bash -c ' + set -e + for t in mpiexec python g++; do command -v $t >/dev/null || { echo "missing $t"; exit 1; }; done + id jovyan' + + - name: Delete old untagged image versions + continue-on-error: true + uses: actions/delete-package-versions@v5 + with: + owner: ${{ github.repository_owner }} + package-name: uw3-base-${{ matrix.base }} + package-type: container + min-versions-to-keep: 2 + delete-only-untagged-versions: true diff --git a/docs/developer/hpc_containers/base.rocky-ompi b/docs/developer/hpc_containers/base.rocky-ompi new file mode 100644 index 000000000..199ca1b14 --- /dev/null +++ b/docs/developer/hpc_containers/base.rocky-ompi @@ -0,0 +1,91 @@ +##################################################################### +# Platform base for the Rocky Linux + OpenMPI family (NCI Gadi, ANU Kaiju). +# +# Everything OS- and MPI-specific lives in a base file like this one; the +# recipes stacked on it (petsc.hpc, underworld3.hpc) build against whatever +# mpicc is on PATH and never call a package manager. Two stages, published as +# two tags of one image: +# :runtime what the final images stand on — no toolchain except g++, which +# lets a developer rebuild UW3 in place on the HPC system +# (underworld3.ckdtree is a C++ extension) +# :builder runtime + compilers and build tools +# +# podman build . --platform linux/amd64 --format docker \ +# --target builder -t ghcr.io//uw3-base-rocky-ompi:builder \ +# -f docs/developer/hpc_containers/base.rocky-ompi +##################################################################### + +ARG PYTHON_VERSION="3.12" +ARG OS_IMAGE="quay.io/rockylinux/rockylinux:8.10" + +FROM ${OS_IMAGE} AS runtime +LABEL maintainer="https://github.com/underworldcode/" + +# Repeat the DEFAULT, not just the name: bare `ARG PYTHON_VERSION` relies on +# inheriting the pre-FROM default, which some buildah/podman versions don't. +ARG PYTHON_VERSION="3.12" + +ENV LANG=C.UTF-8 +ENV PYVER=${PYTHON_VERSION} +ENV OPENBLAS_NUM_THREADS=1 +ENV OMPI_MCA_io=ompio + +ENV NB_USER=jovyan +ENV NB_HOME=/home/$NB_USER +RUN useradd -m -s /bin/bash -N $NB_USER + +# mesa/X11: vtk-osmesa and pyvista. gcc-c++: in-place UW3 rebuilds (see header). +RUN yum update -y \ +&& yum install -y \ + ca-certificates \ + bash-completion \ + openssh \ + openblas \ + openmpi \ + findutils \ + python${PYVER}-pip \ + python${PYVER}-devel \ + gcc-c++ \ + zlib \ + mesa-libGL \ + mesa-libGLU \ + mesa-libOSMesa \ + libX11 \ + libXrender \ + libXext \ + libXfixes \ + libXcursor \ + libXinerama \ + libXrandr \ + libXft \ + fontconfig \ +&& yum clean all \ +&& rm -rf /var/cache/yum + +ENV PATH=/usr/lib64/openmpi/bin:$PATH +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +# One venv shared by every stage above; prepending its bin/ means a bare +# `pip install` lands there. +ENV PYOPT=/opt/venv +RUN python${PYVER} -m venv $PYOPT \ +&& chmod ugo+rwx $PYOPT +ENV PATH=$PYOPT/bin:$PATH +ENV PYTHONPATH=$PYTHONPATH:$PYOPT/lib/python${PYVER}/site-packages + +FROM runtime AS builder +# flex/bison: PETSc's --download-ptscotch needs them. +RUN yum install -y \ + wget \ + make \ + cmake \ + patch \ + git \ + gcc \ + gcc-gfortran \ + flex \ + bison \ + zlib-devel \ + openmpi-devel \ +&& yum clean all \ +&& rm -rf /var/cache/yum diff --git a/docs/developer/hpc_containers/base.ubuntu-mpich b/docs/developer/hpc_containers/base.ubuntu-mpich new file mode 100644 index 000000000..0e046a5d3 --- /dev/null +++ b/docs/developer/hpc_containers/base.ubuntu-mpich @@ -0,0 +1,87 @@ +##################################################################### +# Platform base for Pawsey Setonix: Ubuntu 24.04 + MPICH 3.4.3. +# +# Same two-stage contract as base.rocky-ompi (:runtime, :builder); see there. +# +# Why this base and not a Rocky one: Setonix's `singularity/4.1.0-mpi` module +# bind-mounts the host Cray MPICH (libmpi.so.12) over the container's, and +# Cray MPICH is built against SLES 15 glibc 2.31 — a Rocky 8 image (glibc +# 2.28) fails on missing symbols. Ubuntu 24.04 has 2.39. +# +# Why 3.4.3 and not Pawsey's 4.2.2 tag: Cray MPICH 8.1 is MPICH-3.4-based. +# Anything compiled against 4.x headers references MPI-4 symbols +# (MPI_Buffer_iflush) that the bind-mounted host library lacks — measured as +# an ImportError on Setonix, 2026-09-17. Same ABI, smaller API surface. +##################################################################### + +ARG PYTHON_VERSION="3.12" +ARG OS_IMAGE="quay.io/pawsey/mpich-base:3.4.3_ubuntu24.04" + +FROM ${OS_IMAGE} AS runtime +LABEL maintainer="https://github.com/underworldcode/" + +ARG PYTHON_VERSION="3.12" + +ENV LANG=C.UTF-8 +ENV PYVER=${PYTHON_VERSION} +ENV OPENBLAS_NUM_THREADS=1 +ENV DEBIAN_FRONTEND=noninteractive + +ENV NB_USER=jovyan +ENV NB_HOME=/home/$NB_USER +RUN useradd -m -s /bin/bash -N $NB_USER + +# The Pawsey base already carries gcc/g++/gfortran/git/make/wget. Same package +# set as the Rocky runtime, in Debian names. +RUN apt-get update \ +&& apt-get install -y --no-install-recommends \ + ca-certificates \ + bash-completion \ + openssh-client \ + libopenblas0 \ + python3 \ + python3-dev \ + python3-venv \ + python3-pip \ + g++ \ + zlib1g \ + libgl1 \ + libglu1-mesa \ + libosmesa6 \ + libx11-6 \ + libxrender1 \ + libxext6 \ + libxfixes3 \ + libxcursor1 \ + libxinerama1 \ + libxrandr2 \ + libxft2 \ + fontconfig \ +&& rm -rf /var/lib/apt/lists/* + +# PYVER must match the distro python, because the venv path below embeds it. +RUN python3 -c "import sys; assert f'{sys.version_info[0]}.{sys.version_info[1]}' == '${PYVER}', sys.version" + +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +ENV PYOPT=/opt/venv +RUN python3 -m venv $PYOPT \ +&& chmod ugo+rwx $PYOPT +ENV PATH=$PYOPT/bin:$PATH +ENV PYTHONPATH=$PYTHONPATH:$PYOPT/lib/python${PYVER}/site-packages + +FROM runtime AS builder +RUN apt-get update \ +&& apt-get install -y --no-install-recommends \ + build-essential \ + gfortran \ + cmake \ + make \ + patch \ + git \ + wget \ + flex \ + bison \ + pkg-config \ + zlib1g-dev \ +&& rm -rf /var/lib/apt/lists/* From 6c13cac82d6a6c0202417f06e1a50295064967fe Mon Sep 17 00:00:00 2001 From: Juan Carlos Graciosa Date: Thu, 17 Sep 2026 20:25:21 +1000 Subject: [PATCH 4/8] ci(container): build PETSc per platform from one recipe on the base images --- ...di-petsc-image.yml => hpc-petsc-image.yml} | 104 ++++----- docs/developer/gadi_singularity/petsc.rhel | 207 ------------------ docs/developer/hpc_containers/petsc.hpc | 116 ++++++++++ 3 files changed, 163 insertions(+), 264 deletions(-) rename .github/workflows/{gadi-petsc-image.yml => hpc-petsc-image.yml} (55%) delete mode 100644 docs/developer/gadi_singularity/petsc.rhel create mode 100644 docs/developer/hpc_containers/petsc.hpc diff --git a/.github/workflows/gadi-petsc-image.yml b/.github/workflows/hpc-petsc-image.yml similarity index 55% rename from .github/workflows/gadi-petsc-image.yml rename to .github/workflows/hpc-petsc-image.yml index ccaeffb9b..3710da920 100644 --- a/.github/workflows/gadi-petsc-image.yml +++ b/.github/workflows/hpc-petsc-image.yml @@ -1,21 +1,22 @@ -name: "GHCR: Gadi PETSc Base Image" +name: "GHCR: HPC PETSc images" -# Builds docs/developer/gadi_singularity/petsc.rhel — the from-source PETSc base -# that gadi-uw3-image.yml layers on. +# Builds docs/developer/hpc_containers/petsc.hpc once per platform base — the +# from-source PETSc that hpc-uw3-image.yml layers on. # # Triggers are deliberately narrow: this compiles PETSc with mumps, hypre, -# superlu_dist, mmg/parmmg, ptscotch and slepc, a build measured in hours, and -# only petsc.rhel and its patches can change the result. +# superlu_dist, mmg/parmmg, ptscotch and slepc (~25 min each), and only +# petsc.hpc, its patches and the bases can change the result. on: push: branches: - main - development + - ci/gadi-container # TESTING ONLY — remove before the upstream PR paths: - - 'docs/developer/gadi_singularity/petsc.rhel' + - 'docs/developer/hpc_containers/petsc.hpc' - 'petsc-custom/patches/**' - - '.github/workflows/gadi-petsc-image.yml' + - '.github/workflows/hpc-petsc-image.yml' workflow_dispatch: inputs: petsc_version: @@ -32,27 +33,33 @@ on: default: false concurrency: - group: gadi-petsc-${{ github.ref }} + group: hpc-petsc-${{ github.ref }} cancel-in-progress: true jobs: - build-and-push: + build: runs-on: ubuntu-latest - # Measured at ~25 min with ~25% run-to-run variance, so this is ~5x headroom - # for a heavier future PETSc while still failing a hang in hours, not a day. + # Measured at ~25 min with ~25% run-to-run variance; ~5x headroom. timeout-minutes: 120 permissions: contents: read packages: write - outputs: - image: ${{ steps.meta.outputs.image }} + strategy: + fail-fast: false + matrix: + include: + - platform: gadi + base: rocky-ompi + mpi: ompi + mpi_name: "Open MPI" + - platform: setonix + base: ubuntu-mpich + mpi: mpich + mpi_name: "MPICH" steps: - name: Checkout repository uses: actions/checkout@v4 - # Measured: the whole build consumes ~7 GB against 86 GB free on a fresh - # runner, so no space needs reclaiming. Kept as instrumentation in case a - # future PETSc pulls in far more --download- packages. - name: Report disk headroom run: df -h / @@ -70,9 +77,9 @@ jobs: id: meta run: | set -euo pipefail - OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') - IMAGE="ghcr.io/${OWNER}/petsc-gadi" + IMAGE="ghcr.io/${OWNER}/petsc-${{ matrix.platform }}" + BASE="ghcr.io/${OWNER}/uw3-base-${{ matrix.base }}" PETSC_VERSION="${{ inputs.petsc_version }}" PETSC_VERSION="${PETSC_VERSION:-3.25.0}" @@ -80,29 +87,28 @@ jobs: MAKE_NP="${MAKE_NP:-4}" { - echo "image=${IMAGE}:${PETSC_VERSION}-ompi" - echo "tags=${IMAGE}:${PETSC_VERSION}-ompi,${IMAGE}:latest" + echo "image=${IMAGE}:${PETSC_VERSION}-${{ matrix.mpi }}" + echo "tags=${IMAGE}:${PETSC_VERSION}-${{ matrix.mpi }},${IMAGE}:latest" + echo "base=${BASE}" echo "petsc_version=${PETSC_VERSION}" echo "make_np=${MAKE_NP}" } >> "$GITHUB_OUTPUT" - - echo "Building ${IMAGE}:${PETSC_VERSION}-ompi with --with-make-np=${MAKE_NP}" + echo "Building ${IMAGE}:${PETSC_VERSION}-${{ matrix.mpi }} on ${BASE} with --with-make-np=${MAKE_NP}" - name: Build and push uses: docker/build-push-action@v6 with: context: . - file: docs/developer/gadi_singularity/petsc.rhel - # Docker v2 media types — see the note in gadi-uw3-image.yml. + file: docs/developer/hpc_containers/petsc.hpc + # Docker v2 media types, no attestations — see hpc-uw3-image.yml. outputs: type=image,push=true,oci-mediatypes=false platforms: linux/amd64 - # See the note in gadi-uw3-image.yml — attestations break some - # `singularity pull` versions. provenance: false sbom: false no-cache: ${{ inputs.force_rebuild || false }} build-args: | - PYTHON_VERSION=3.12 + BASE_BUILDER=${{ steps.meta.outputs.base }}:builder + BASE_RUNTIME=${{ steps.meta.outputs.base }}:runtime PETSC_VERSION=${{ steps.meta.outputs.petsc_version }} PETSC_MAKE_NP=${{ steps.meta.outputs.make_np }} tags: ${{ steps.meta.outputs.tags }} @@ -112,29 +118,14 @@ jobs: docker image ls ${{ steps.meta.outputs.image }} || true df -h / - smoke-test: - needs: build-and-push - runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - contents: read - packages: read - steps: - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull image - run: docker pull ${{ needs.build-and-push.outputs.image }} + run: docker pull ${{ steps.meta.outputs.image }} - # petsc4py importing is the thing that actually breaks when the PETSc build - # goes subtly wrong (shared libraries, PYTHONPATH, the meson petsc4py step). + # petsc4py importing is what actually breaks when the build goes subtly + # wrong (shared libraries, PYTHONPATH, the meson petsc4py step). - name: Import petsc4py and slepc4py run: | - docker run --rm ${{ needs.build-and-push.outputs.image }} python -c " + docker run --rm ${{ steps.meta.outputs.image }} python -c " import petsc4py, slepc4py petsc4py.init() from petsc4py import PETSc @@ -143,29 +134,28 @@ jobs: print('PETSc', PETSc.Sys.getVersion()) " - - name: MPI sanity (2 ranks) + # Assert the MPI flavour: a silently wrong libmpi is exactly the failure + # that would otherwise only surface on a compute node. + - name: MPI sanity (2 ranks, expect ${{ matrix.mpi_name }}) run: | - docker run --rm ${{ needs.build-and-push.outputs.image }} \ + docker run --rm ${{ steps.meta.outputs.image }} \ mpiexec -n 2 python -c " from mpi4py import MPI c = MPI.COMM_WORLD - print(f'rank {c.rank} of {c.size}') + v = MPI.Get_library_version().splitlines()[0] + print(f'rank {c.rank} of {c.size}: {v}') assert c.size == 2 + assert '${{ matrix.mpi_name }}' in v, v " - cleanup: - needs: [build-and-push, smoke-test] - runs-on: ubuntu-latest - permissions: - packages: write - steps: - # Housekeeping only: see the note in gadi-uw3-image.yml. + # Housekeeping only; the action 404s partway through on user-owned + # packages, having already deleted what it could. - name: Delete old untagged image versions continue-on-error: true uses: actions/delete-package-versions@v5 with: owner: ${{ github.repository_owner }} - package-name: petsc-gadi + package-name: petsc-${{ matrix.platform }} package-type: container min-versions-to-keep: 2 delete-only-untagged-versions: true diff --git a/docs/developer/gadi_singularity/petsc.rhel b/docs/developer/gadi_singularity/petsc.rhel deleted file mode 100644 index a13ead632..000000000 --- a/docs/developer/gadi_singularity/petsc.rhel +++ /dev/null @@ -1,207 +0,0 @@ -##################################################################### -# UW3 PETSc container -# Multi stage Containerfile based on UW2 version -# This builds PETSc according to the pixi amr-dev environment -# see https://docs.docker.com/get-started/docker-concepts/building-images/multi-stage-builds/ -# -# Stages: -# 1. 'runtime' -# The runtime environment (packages, permissions, ENV vars.) -# is consistent accross all stages of this Containerfile. -# -# 2. 'builder' -# The builder layer, takes the runtime layer and add compiling / building software -# that is added to /usr/local and /opt/venv -# -# 3. 'final' == runtime + min. builder -# The final image is a composite of the runtime layer and the -# minimal sections of the builder layer's final software stack. -# -# To build use podman from the top level underworld directory. i.e. -# $ podman build . \ -# --platform linux/amd64 \ -# --format docker \ -# -t new_image_name \ -# -f ./docs/developer/gadi_singularity/petsc.rhel -##################################################################### - -# The following are passed in via --build-args -# Must go before the 1st FROM see -# https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact -ARG PYTHON_VERSION="3.12" -ARG PETSC_VERSION="3.25.0" -ARG BASE_IMAGE="quay.io/rockylinux/rockylinux:8.10" - -# 1. Stage 1: 'runtime' -FROM ${BASE_IMAGE} as runtime -LABEL maintainer="https://github.com/underworldcode/" - -# Repeat the DEFAULT, not just the name: bare `ARG PYTHON_VERSION` relies on -# inheriting the pre-FROM default, which some buildah/podman versions don't — -# PYVER expands empty and yum then looks for a package called "python-pip". -ARG PYTHON_VERSION="3.12" - -#### Containerfile ENV vars - for all image stages -ENV LANG=C.UTF-8 -ENV PYVER=${PYTHON_VERSION} - -# gadi-specific settings -ENV OPENBLAS_NUM_THREADS=1 -ENV OMPI_MCA_io=ompio - -# add user jovyan -ENV NB_USER jovyan -ENV NB_HOME /home/$NB_USER -RUN useradd -m -s /bin/bash -N $NB_USER - -RUN yum update -y \ -&& yum install -y \ - bash-completion \ - openssh \ - openblas \ - python${PYVER}-pip \ - python${PYVER}-devel \ - openmpi \ - findutils \ -&& yum clean all \ -&& rm -rf /var/cache/yum - -# add system openmpi to $PATH and $LD_LIBRARY_PATH -ENV PATH=/usr/lib64/openmpi/bin:$PATH -ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - -ENV PYOPT=/opt/venv -# build and set open permissions on virtual environment -RUN python${PYVER} -m venv $PYOPT \ -&& chmod ugo+rwx $PYOPT - -# define python env vars. -# prepappending on PATH means all pip install will goto the PYOPT -ENV PATH=$PYOPT/bin:$PATH -ENV PYTHONPATH=$PYTHONPATH:$PYOPT/lib/python${PYVER}/site-packages - -# runtime python requirements -RUN python${PYVER} -m pip install wheel \ - "numpy<2" - -# 2. Define the builder layer -FROM runtime as builder - -# defaults repeated deliberately — see the note in the runtime stage -ARG PETSC_VERSION="3.25.0" -ARG PYTHON_VERSION="3.12" -# Parallelism for PETSc's own build. 2 is safe inside a memory-constrained podman -# machine (compiling PETSc.c is memory-heavy and OOMs at higher values); CI on a -# native runner passes 4. -ARG PETSC_MAKE_NP="2" - -RUN yum install -y \ - ca-certificates \ - wget \ - make \ - gcc \ - gcc-gfortran \ - gcc-c++ \ - cmake \ - patch \ - openblas \ - zlib-devel \ - openmpi-devel \ - findutils \ - git \ - flex \ - bison \ -&& yum clean all \ -&& rm -rf /var/cache/yum -# NOTE flex and bison are needed to build - -# Upper bound is load-bearing: petsc4py 3.25.0 declares only "cython >= 3" but -# PC.pyx:1256 fails to cythonize under 3.3.0 ("Invalid index type 'int'"). -# Bisected against the 3.25.0 source: 3.2.9 is the last version that builds. -RUN python${PYVER} -m pip install "cython>=3.1,<3.3" \ - "setuptools>=75" \ - "meson" \ - "meson-python" \ - "ninja" \ -&& python${PYVER} -m pip install --no-cache-dir --no-binary=mpi4py "mpi4py>=4,<5" -# no-binary for mpi4py to force build against openmpi, rather than mpich (default) - -# copy patches into builder -COPY petsc-custom/patches/scotch-7.0.10-c23-fix.tar.gz /tmp/scotch.tar.gz -COPY petsc-custom/patches/plexfem-internal-boundary-ownership-fix.patch /tmp/ - -# get petsc -RUN mkdir -p /tmp/src -WORKDIR /tmp/src -RUN wget https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-lite-${PETSC_VERSION}.tar.gz --no-check-certificate \ -&& tar -zxf petsc-lite-${PETSC_VERSION}.tar.gz -WORKDIR /tmp/src/petsc-${PETSC_VERSION} - -# apply patch then configure -# patch may already be included in newer PETSc versions - skip gracefully if it doesn't apply -RUN if patch -p1 --dry-run < /tmp/plexfem-internal-boundary-ownership-fix.patch 2>/dev/null; then \ - patch -p1 < /tmp/plexfem-internal-boundary-ownership-fix.patch; \ - echo "plexfem patch applied successfully"; \ - else \ - echo "plexfem patch not applicable (may already be in this PETSc version), skipping"; \ - fi -RUN python${PYVER} ./configure \ - --with-debugging=0 \ - --prefix=/usr/local \ - --with-shared-libraries=1 \ - --with-cxx-dialect=C++11 \ - "--COPTFLAGS=-g -O3" "--CXXOPTFLAGS=-g -O3" "--FOPTFLAGS=-g -O3" \ - --useThreads=0 \ - --with-x=0 \ - --with-pragmatic=1 \ - --with-petsc4py=1 \ - --with-slepc4py=1 \ - --download-eigen=1 \ - --download-metis=1 \ - --download-parmetis=1 \ - --download-mumps=1 \ - --download-scalapack=1 \ - --download-hypre=1 \ - --download-superlu=1 \ - --download-superlu_dist=1 \ - --download-mmg=1 \ - "--download-mmg-cmake-arguments=-DMMG_INSTALL_PRIVATE_HEADERS=ON -DUSE_SCOTCH=OFF" \ - --download-parmmg=1 \ - --download-pragmatic=1 \ - "--download-ptscotch=/tmp/scotch.tar.gz" \ - --download-slepc=1 \ - --download-hdf5=1 \ - --download-fblaslapack=1 \ - --download-zlib=1 \ - --download-ctetgen=1 \ - --download-triangle=1 \ - --with-make-np=${PETSC_MAKE_NP} -RUN make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt all -RUN make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt install \ - || (echo "=== petsc4py build log ===" && \ - cat arch-linux-c-opt/lib/petsc/conf/petsc4py.build.log 2>/dev/null && \ - echo "=== slepc4py build log ===" && \ - cat arch-linux-c-opt/lib/petsc/conf/slepc4py.build.log 2>/dev/null && \ - exit 1) -RUN rm -rf /usr/local/share/petsc - -# record builder stage packages used -RUN python${PYVER} -m pip freeze > /opt/requirements.txt \ -&& dnf history userinstalled > /opt/packages.txt - -# Stage 3: 'final' -FROM runtime as final - -COPY --from=builder /opt /opt -COPY --from=builder /usr/local /usr/local - -# MUST set PETSc environment variables -ENV PETSC_DIR=/usr/local -ENV PYTHONPATH=$PYTHONPATH:$PETSC_DIR/lib - -# switch to not-root user and workspace -USER $NB_USER -WORKDIR $NB_HOME - -# default command is to run jupyter lab -CMD ["jupyter-lab", "--no-browser", "--ip='0.0.0.0'"] diff --git a/docs/developer/hpc_containers/petsc.hpc b/docs/developer/hpc_containers/petsc.hpc new file mode 100644 index 000000000..24373fb0a --- /dev/null +++ b/docs/developer/hpc_containers/petsc.hpc @@ -0,0 +1,116 @@ +##################################################################### +# UW3 PETSc image — platform-neutral. +# +# Builds PETSc from source, with the AMR stack the pixi amr-dev environment +# uses, on top of a platform base (base.rocky-ompi, base.ubuntu-mpich). No +# package manager is called here: compilers come from the :builder base, and +# the result is laid over the :runtime base, so the same file serves every +# platform. +# +# Stages: +# builder BASE_BUILDER + PETSc built into /usr/local, petsc4py/mpi4py in /opt/venv +# final BASE_RUNTIME + /usr/local + /opt from builder +# +# podman build . --platform linux/amd64 --format docker \ +# --build-arg BASE_BUILDER=ghcr.io//uw3-base-rocky-ompi:builder \ +# --build-arg BASE_RUNTIME=ghcr.io//uw3-base-rocky-ompi:runtime \ +# -t ghcr.io//petsc-gadi:3.25.0-ompi \ +# -f docs/developer/hpc_containers/petsc.hpc +##################################################################### + +# Must go before the 1st FROM, see +# https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact +ARG BASE_BUILDER="ghcr.io/underworldcode/uw3-base-rocky-ompi:builder" +ARG BASE_RUNTIME="ghcr.io/underworldcode/uw3-base-rocky-ompi:runtime" +ARG PETSC_VERSION="3.25.0" + +FROM ${BASE_BUILDER} AS builder +# defaults repeated deliberately — see the note in base.rocky-ompi +ARG PETSC_VERSION="3.25.0" +# Parallelism for PETSc's own build. 2 is safe inside a memory-constrained +# podman machine (compiling PETSc.c is memory-heavy and OOMs at higher +# values); CI on a native runner passes 4. +ARG PETSC_MAKE_NP="2" + +# Upper bound is load-bearing: petsc4py 3.25.0 declares only "cython >= 3" but +# PC.pyx:1256 fails to cythonize under 3.3.0 ("Invalid index type 'int'"). +# Bisected against the 3.25.0 source: 3.2.9 is the last version that builds. +# mpi4py from source so it links this base's MPI, not a wheel's bundled one. +RUN pip install --no-cache-dir \ + wheel \ + "numpy<2" \ + "cython>=3.1,<3.3" \ + "setuptools>=75" \ + meson \ + meson-python \ + ninja \ +&& pip install --no-cache-dir --no-binary=mpi4py "mpi4py>=4,<5" + +COPY petsc-custom/patches/scotch-7.0.10-c23-fix.tar.gz /tmp/scotch.tar.gz +COPY petsc-custom/patches/plexfem-internal-boundary-ownership-fix.patch /tmp/ + +RUN mkdir -p /tmp/src +WORKDIR /tmp/src +RUN wget https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-lite-${PETSC_VERSION}.tar.gz --no-check-certificate \ +&& tar -zxf petsc-lite-${PETSC_VERSION}.tar.gz +WORKDIR /tmp/src/petsc-${PETSC_VERSION} + +# The patch may already be upstream in a newer PETSc; skip gracefully then. +RUN if patch -p1 --dry-run < /tmp/plexfem-internal-boundary-ownership-fix.patch 2>/dev/null; then \ + patch -p1 < /tmp/plexfem-internal-boundary-ownership-fix.patch; \ + echo "plexfem patch applied"; \ + else \ + echo "plexfem patch not applicable, skipping"; \ + fi +RUN python3 ./configure \ + --with-debugging=0 \ + --prefix=/usr/local \ + --with-shared-libraries=1 \ + --with-cxx-dialect=C++11 \ + "--COPTFLAGS=-g -O3" "--CXXOPTFLAGS=-g -O3" "--FOPTFLAGS=-g -O3" \ + --useThreads=0 \ + --with-x=0 \ + --with-pragmatic=1 \ + --with-petsc4py=1 \ + --with-slepc4py=1 \ + --download-eigen=1 \ + --download-metis=1 \ + --download-parmetis=1 \ + --download-mumps=1 \ + --download-scalapack=1 \ + --download-hypre=1 \ + --download-superlu=1 \ + --download-superlu_dist=1 \ + --download-mmg=1 \ + "--download-mmg-cmake-arguments=-DMMG_INSTALL_PRIVATE_HEADERS=ON -DUSE_SCOTCH=OFF" \ + --download-parmmg=1 \ + --download-pragmatic=1 \ + "--download-ptscotch=/tmp/scotch.tar.gz" \ + --download-slepc=1 \ + --download-hdf5=1 \ + --download-fblaslapack=1 \ + --download-zlib=1 \ + --download-ctetgen=1 \ + --download-triangle=1 \ + --with-make-np=${PETSC_MAKE_NP} +RUN make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt all +RUN make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt install \ + || (echo "=== petsc4py build log ===" && \ + cat arch-linux-c-opt/lib/petsc/conf/petsc4py.build.log 2>/dev/null && \ + echo "=== slepc4py build log ===" && \ + cat arch-linux-c-opt/lib/petsc/conf/slepc4py.build.log 2>/dev/null && \ + exit 1) +RUN rm -rf /usr/local/share/petsc + +RUN pip freeze > /opt/requirements.txt + +FROM ${BASE_RUNTIME} AS final + +COPY --from=builder /opt /opt +COPY --from=builder /usr/local /usr/local + +ENV PETSC_DIR=/usr/local +ENV PYTHONPATH=$PYTHONPATH:$PETSC_DIR/lib + +USER $NB_USER +WORKDIR $NB_HOME From bc83c789fcd223b2922a435d4f061e7535ffba8a Mon Sep 17 00:00:00 2001 From: Juan Carlos Graciosa Date: Thu, 17 Sep 2026 21:03:35 +1000 Subject: [PATCH 5/8] ci(container): build Underworld3 per platform from one recipe; rename gadi_singularity to hpc_containers --- .../{gadi-uw3-image.yml => hpc-uw3-image.yml} | 125 +++++----- docs/developer/gadi_singularity/README.md | 124 ---------- .../gadi_singularity/underworld3.rhel | 227 ------------------ docs/developer/hpc_containers/README.md | 97 ++++++++ docs/developer/hpc_containers/underworld3.hpc | 118 +++++++++ 5 files changed, 280 insertions(+), 411 deletions(-) rename .github/workflows/{gadi-uw3-image.yml => hpc-uw3-image.yml} (63%) delete mode 100644 docs/developer/gadi_singularity/README.md delete mode 100644 docs/developer/gadi_singularity/underworld3.rhel create mode 100644 docs/developer/hpc_containers/README.md create mode 100644 docs/developer/hpc_containers/underworld3.hpc diff --git a/.github/workflows/gadi-uw3-image.yml b/.github/workflows/hpc-uw3-image.yml similarity index 63% rename from .github/workflows/gadi-uw3-image.yml rename to .github/workflows/hpc-uw3-image.yml index db9c97ea3..c9e900e34 100644 --- a/.github/workflows/gadi-uw3-image.yml +++ b/.github/workflows/hpc-uw3-image.yml @@ -1,21 +1,23 @@ -name: "GHCR: Gadi Singularity Image" +name: "GHCR: HPC Underworld3 images" -# Builds docs/developer/gadi_singularity/underworld3.rhel and pushes to GHCR for -# `singularity pull docker://...` on Gadi. +# Builds docs/developer/hpc_containers/underworld3.hpc once per platform and +# pushes to GHCR for `singularity pull docker://...` on Gadi, Kaiju and Setonix. # -# This is the layer that moves; the expensive PETSc base under it is built rarely -# by gadi-petsc-image.yml. Unlike docker-image.yaml and binder-image.yml, this -# image is Rocky Linux 8.10 + from-source PETSc, to match Gadi's ABI. +# This is the layer that moves; the expensive PETSc images under it are built +# rarely by hpc-petsc-image.yml, and the platform bases rarer still by +# hpc-base-images.yml. Unlike docker-image.yaml and binder-image.yml these +# images match each HPC system's OS and MPI ABI. on: push: branches: - main - development + - ci/gadi-container # TESTING ONLY — remove before the upstream PR tags: ['v*'] paths: - - 'docs/developer/gadi_singularity/underworld3.rhel' - - '.github/workflows/gadi-uw3-image.yml' + - 'docs/developer/hpc_containers/underworld3.hpc' + - '.github/workflows/hpc-uw3-image.yml' - 'pixi.toml' - 'pyproject.toml' - 'setup.py' @@ -32,7 +34,7 @@ on: type: string default: '' petsc_image: - description: 'PETSc base image. Empty = ghcr.io//petsc-gadi:3.25.0-ompi' + description: 'PETSc base image override. Applies to EVERY platform in the matrix, so only useful with a single-platform matrix edit. Empty = per-platform default.' type: string default: '' image_tag: @@ -45,18 +47,35 @@ on: default: false concurrency: - group: gadi-uw3-${{ github.ref }} + group: hpc-uw3-${{ github.ref }} cancel-in-progress: true jobs: - build-and-push: + build: runs-on: ubuntu-latest timeout-minutes: 90 permissions: contents: read packages: write - outputs: - image: ${{ steps.meta.outputs.image }} + strategy: + fail-fast: false + matrix: + include: + - platform: gadi + base: rocky-ompi + mpi: ompi + mpi_name: "Open MPI" + petsc_var: GADI_PETSC_IMAGE + - platform: setonix + base: ubuntu-mpich + mpi: mpich + mpi_name: "MPICH" + petsc_var: SETONIX_PETSC_IMAGE + env: + # Repository variables that let a fork reuse a PETSc image it already + # published under another name. Indexed by matrix.petsc_var below. + GADI_PETSC_IMAGE: ${{ vars.GADI_PETSC_IMAGE }} + SETONIX_PETSC_IMAGE: ${{ vars.SETONIX_PETSC_IMAGE }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -78,7 +97,8 @@ jobs: # Image names must be lowercase; repository_owner may not be. OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') - IMAGE="ghcr.io/${OWNER}/underworld3-gadi" + IMAGE="ghcr.io/${OWNER}/underworld3-${{ matrix.platform }}" + BASE="ghcr.io/${OWNER}/uw3-base-${{ matrix.base }}" # Same ref-name / IS_RELEASE convention as binder-image.yml. if [[ "$GITHUB_REF" == refs/tags/* ]]; then @@ -100,12 +120,12 @@ jobs: DOCKER_TAG=$(echo "$REF_NAME" | tr '/' '-') fi - # Dispatch input wins, then the GADI_PETSC_IMAGE repo variable (lets a - # fork reuse a base image it already published under another name), - # then the canonical name. + # Dispatch input wins, then the per-platform repo variable, then the + # canonical name. + VAR_NAME="${{ matrix.petsc_var }}" PETSC_IMAGE="${{ inputs.petsc_image }}" - PETSC_IMAGE="${PETSC_IMAGE:-${{ vars.GADI_PETSC_IMAGE }}}" - PETSC_IMAGE="${PETSC_IMAGE:-ghcr.io/${OWNER}/petsc-gadi:3.25.0-ompi}" + PETSC_IMAGE="${PETSC_IMAGE:-${!VAR_NAME}}" + PETSC_IMAGE="${PETSC_IMAGE:-ghcr.io/${OWNER}/petsc-${{ matrix.platform }}:3.25.0-${{ matrix.mpi }}}" TAGS="${IMAGE}:${DOCKER_TAG}" if [[ "$IS_RELEASE" == "true" ]]; then @@ -123,6 +143,7 @@ jobs: { echo "image=${IMAGE}:${DOCKER_TAG}" echo "tags=${TAGS}" + echo "base=${BASE}" echo "uw3_branch=${REF_NAME}" echo "petsc_image=${PETSC_IMAGE}" echo "uw3_version=${UW3_VERSION}" @@ -131,18 +152,19 @@ jobs: echo "Building ${IMAGE}:${DOCKER_TAG}" echo " UW3 ref: ${REF_NAME}" echo " PETSc base: ${PETSC_IMAGE}" + echo " builder: ${BASE}:builder" - name: Build and push uses: docker/build-push-action@v6 with: context: . - file: docs/developer/gadi_singularity/underworld3.rhel + file: docs/developer/hpc_containers/underworld3.hpc # Docker v2 media types rather than buildx's OCI default, matching the # podman `--format docker` images already validated on Gadi. Older # Singularity builds read OCI patchily; this removes the variable. outputs: type=image,push=true,oci-mediatypes=false - # Gadi is x86_64. No arm64: it would need QEMU for no benefit, and the - # gmsh / vtk-osmesa wheels this image needs are x86_64-only anyway. + # All three systems are x86_64. No arm64: it would need QEMU for no + # benefit, and the gmsh / vtk-osmesa wheels are x86_64-only anyway. platforms: linux/amd64 # Attestations turn the result into a multi-manifest index containing an # "unknown/unknown" entry, which some Singularity/Apptainer versions @@ -151,77 +173,60 @@ jobs: sbom: false no-cache: ${{ inputs.force_rebuild || false }} build-args: | - PYTHON_VERSION=3.12 UW3_BRANCH=${{ steps.meta.outputs.uw3_branch }} UW3_REPO=${{ github.server_url }}/${{ github.repository }}.git PETSC_IMAGE=${{ steps.meta.outputs.petsc_image }} + BASE_BUILDER=${{ steps.meta.outputs.base }}:builder SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.meta.outputs.uw3_version }} tags: ${{ steps.meta.outputs.tags }} # Cache lives in its own package: sharing one with the image put stale - # cache manifests into the same untagged pool the cleanup job prunes, + # cache manifests into the same untagged pool the cleanup step prunes, # so they competed with old images for min-versions-to-keep. - cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/underworld3-gadi-buildcache:latest - cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/underworld3-gadi-buildcache:latest,mode=max - - smoke-test: - needs: build-and-push - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - packages: read - steps: - - name: Login to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/underworld3-${{ matrix.platform }}-buildcache:latest + cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/underworld3-${{ matrix.platform }}-buildcache:latest,mode=max - name: Pull image - run: docker pull ${{ needs.build-and-push.outputs.image }} + run: docker pull ${{ steps.meta.outputs.image }} # A bare import is the cheapest check and catches a missing runtime dep - # (this is exactly how the absent `requests` would have shown up). + # (this is exactly how the absent `requests` once showed up). - name: Import underworld3 run: | - docker run --rm ${{ needs.build-and-push.outputs.image }} \ + docker run --rm ${{ steps.meta.outputs.image }} \ python -c "import underworld3; print('underworld3', underworld3.__version__)" - # The Containerfile installs gmsh and vtk-osmesa behind `|| echo skip`, so a + # The recipe installs gmsh and vtk-osmesa behind `|| echo skip`, so a # failed install is silent. Import them explicitly or it ships broken. - name: Import optional stack (gmsh, vtk, pyvista, h5py, petsc4py) run: | - docker run --rm ${{ needs.build-and-push.outputs.image }} \ + docker run --rm ${{ steps.meta.outputs.image }} \ python -c "import gmsh, vtk, pyvista, h5py, petsc4py; print('optional stack OK')" - - name: MPI sanity (2 ranks) + # Assert the MPI flavour too: a silently wrong libmpi is exactly the + # failure that would otherwise only surface on a compute node. + - name: MPI sanity (2 ranks, expect ${{ matrix.mpi_name }}) run: | - docker run --rm ${{ needs.build-and-push.outputs.image }} \ + docker run --rm ${{ steps.meta.outputs.image }} \ mpiexec -n 2 python -c " import underworld3 as uw from mpi4py import MPI c = MPI.COMM_WORLD + v = MPI.Get_library_version().splitlines()[0] assert c.size == 2 - print(f'rank {c.rank} of {c.size}, underworld3 {uw.__version__}') + assert '${{ matrix.mpi_name }}' in v, v + print(f'rank {c.rank} of {c.size}, underworld3 {uw.__version__}, {v}') " - cleanup: - # Only prune once the image is known good — otherwise a broken build would - # delete the last working one. - needs: [build-and-push, smoke-test] - runs-on: ubuntu-latest - permissions: - packages: write - steps: - # Housekeeping only: a failed prune must not fail a run whose image is - # good. The action 404s partway through on user-owned (vs org-owned) - # packages, having already deleted what it could. + # Only prune once the image is known good — a failed step above stops + # the job before this runs, so a broken build never deletes the last + # working one. Housekeeping only: a 404 partway through must not fail + # a run whose image is good. - name: Delete old untagged image versions continue-on-error: true uses: actions/delete-package-versions@v5 with: owner: ${{ github.repository_owner }} - package-name: underworld3-gadi + package-name: underworld3-${{ matrix.platform }} package-type: container min-versions-to-keep: 3 # Load-bearing: every TAGGED image (:latest, :v3.1.0, :main, and the diff --git a/docs/developer/gadi_singularity/README.md b/docs/developer/gadi_singularity/README.md deleted file mode 100644 index 46a00453f..000000000 --- a/docs/developer/gadi_singularity/README.md +++ /dev/null @@ -1,124 +0,0 @@ -# Building Underworld3 for Gadi (NCI) - -This directory contains two Containerfiles to build the Underworld3 (UW3) Singularity image for Gadi (nci.org.au). - -Both use Rocky Linux 8.10 to match Gadi's OS for ABI compatibility. - -## Automated builds - -Both images are built by GitHub Actions; the manual steps below are only needed for -local iteration or if you are building outside the `underworldcode` org. - -| Workflow | Builds | Publishes | Runs when | -|---|---|---|---| -| `.github/workflows/gadi-uw3-image.yml` | `underworld3.rhel` | `ghcr.io//underworld3-gadi:` (`:latest` on release) | push to `main`/`development` touching `src/**`, `pixi.toml`, `pyproject.toml`, `setup.py` or this directory; published releases; manual dispatch | -| `.github/workflows/gadi-petsc-image.yml` | `petsc.rhel` | `ghcr.io//petsc-gadi:-ompi` (+ `:latest`) | push touching `petsc.rhel` or `petsc-custom/patches/**`; manual dispatch | - -PETSc is a ~25 min from-source build, so it is kept as a separate, rarely-triggered -workflow and the published image is reused as the base for every Underworld3 build. -To rebuild it explicitly: - -```bash -gh workflow run gadi-petsc-image.yml -f petsc_version=3.25.0 -f make_np=4 -``` - -To build the Underworld3 image from a specific ref: - -```bash -gh workflow run gadi-uw3-image.yml -f uw3_branch=v3.1.0 -f image_tag=v3.1.0 -``` - -Each build is followed by a smoke test: `import underworld3`, an import check for the -optional gmsh/vtk-osmesa stack that the Containerfile installs behind `|| echo skip`, -and a 2-rank MPI check. - -### Retention - -Only the **three most recent untagged** `underworld3-gadi` versions are kept (two for -`petsc-gadi`); older ones are pruned automatically after the smoke test passes. Tagged -images — `:latest`, `:main`, `:development`, and every `v*` release — are never pruned. - -### Package visibility - -A package published by these workflows inherits the repository's visibility, so on a -public repo both images are publicly pullable and Gadi compute nodes — which have no -GitHub credentials — can `singularity pull` them directly. A package created by hand -(`podman push` from a laptop) is **not** linked to the repository and does default to -private; set it public under its package settings, or authenticate the pull: - -```bash -export SINGULARITY_DOCKER_USERNAME= -export SINGULARITY_DOCKER_PASSWORD= -``` - -Note the workflows build single-platform `linux/amd64` images with buildx attestations -disabled (`provenance: false`, `sbom: false`). Attestations add an `unknown/unknown` -entry to the manifest index that some Singularity/Apptainer versions refuse to pull. - -## Build Order - -Build commands must be run from the top-level `underworld3/` directory (the build context). -Builds targeting Gadi must use `--platform linux/amd64`. - -### 1. Build PETSc layer - -```bash -podman build . \ - --platform linux/amd64 \ - --format docker \ - -t ghcr.io//petsc-gadi:3.25.0-ompi \ - -f ./docs/developer/gadi_singularity/petsc.rhel -``` - -### 2. Push PETSc image to registry - -```bash -podman push ghcr.io//petsc-gadi:3.25.0-ompi -``` - -### 3. Build Underworld3 - -```bash -podman build . \ - --platform linux/amd64 \ - --format docker \ - --build-arg PETSC_IMAGE=ghcr.io//petsc-gadi:3.25.0-ompi \ - --build-arg UW3_BRANCH=development \ - -t ghcr.io//underworld3-gadi:latest \ - -f ./docs/developer/gadi_singularity/underworld3.rhel -``` - -### 4. Push Underworld3 image - -```bash -podman push ghcr.io//underworld3-gadi:latest -``` - -## What Each File Does - -- **petsc.rhel** — Builds PETSc 3.25.0 with full AMR support (petsc4py, slepc4py, mmg, parmmg, etc.) -- **underworld3.rhel** — Builds Underworld3 on top of the PETSc image - -## Running on Gadi - -Pull the image on Gadi (redirect cache to scratch to avoid home quota issues): - -```bash -export SINGULARITY_CACHEDIR=/scratch///.singularity -module load singularity -singularity pull docker://ghcr.io//underworld3-gadi:latest -``` - -Run a script with MPI: - -```bash -module load singularity -module load openmpi/4.1.7 -mpiexec -n singularity exec underworld3-gadi_latest.sif python3 -``` - -## Notes - -- OpenFabrics (mlx5_0) warnings in the job error log are harmless -- PostHog telemetry failures on compute nodes are harmless (no outbound internet) -- The ghcr.io images must be set to **public** for Singularity to pull without authentication diff --git a/docs/developer/gadi_singularity/underworld3.rhel b/docs/developer/gadi_singularity/underworld3.rhel deleted file mode 100644 index 69ba58ba1..000000000 --- a/docs/developer/gadi_singularity/underworld3.rhel +++ /dev/null @@ -1,227 +0,0 @@ -##################################################################### -# Multi stage Containerfile for Underworld 3 -# UW3 container based on UW2 version -# This UW3 container is based on pixi amr-dev environment -# see https://docs.docker.com/get-started/docker-concepts/building-images/multi-stage-builds/ -# -# Stages: -# 1. 'runtime' -# The runtime environment (packages, permissions, ENV vars.) -# is consistent accross all stages of this Containerfile. -# -# 2. 'builder' -# The builder layer, takes the runtime layer and add compiling / building software -# that is added to /usr/local and /opt/venv -# -# 3. 'final' == runtime + min. builder -# The final image is a composite of the runtime layer and the -# minimal sections of the builder layer's final software stack. -# -# To build use podman from the top level underworld didrectory. i.e. -# $ podman build . \ -# --platform linux/amd64 \ -# --format docker \ -# --build-arg UW3_BRANCH=xxx \ -# -t new_image_name \ -# -f ./docs/developer/gadi_singularity/underworld3.rhel -##################################################################### - -# The following are passed in via --build-arg -# Must go before the 1st FROM see -# https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact -ARG PYTHON_VERSION="3.12" -ARG BASE_IMAGE="quay.io/rockylinux/rockylinux:8.10" -ARG PETSC_IMAGE="ghcr.io/underworldcode/petsc-gadi:3.25.0-ompi" -ARG UW3_BRANCH="development" -# Overridden by CI to the repository being built, so a fork builds its own -# branches rather than looking for them upstream. -ARG UW3_REPO="https://github.com/underworldcode/underworld3.git" - -# 'petsc-image' will be used later on in builder stage COPY command -FROM ${PETSC_IMAGE} as petsc-image - -#################### -# Stage 1: 'runtime' -#################### -FROM ${BASE_IMAGE} as runtime -LABEL maintainer="https://github.com/underworldcode/" - -# Repeat the DEFAULT, not just the name: bare `ARG PYTHON_VERSION` relies on -# inheriting the pre-FROM default, which some buildah/podman versions don't — -# PYVER expands empty and yum then looks for a package called "python-pip". -ARG PYTHON_VERSION="3.12" - -#### Containerfile ENV vars - for all image stages -ENV LANG=C.UTF-8 -ENV PYVER=${PYTHON_VERSION} - -# add user jovyan -ENV NB_USER jovyan -ENV NB_HOME /home/$NB_USER -RUN useradd -m -s /bin/bash -N $NB_USER - -# runtime packages - [vim, git] are optional. -# gcc-c++ looks out of place in a runtime image but is deliberate: it lets a -# developer rebuild UW3 in place on an HPC system. underworld3.ckdtree is a C++ -# extension, and gcc alone (pulled in by python-devel) cannot build it. -RUN yum update -y \ -&& yum install -y \ - ca-certificates \ - bash-completion \ - openssh \ - openblas \ - openmpi \ - python${PYVER}-pip \ - python${PYVER}-devel \ - gcc-c++ \ - zlib \ - mesa-libGL \ - mesa-libGLU \ - mesa-libOSMesa \ - libX11 \ - libXrender \ - libXext \ - libXfixes \ - libXcursor \ - libXinerama \ - libXrandr \ - libXft \ - fontconfig \ -&& yum clean all \ -&& rm -rf /var/cache/yum - -# add system openmpi to $PATH and $LD_LIBRARY_PATH -ENV PATH=/usr/lib64/openmpi/bin:$PATH -ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - -ENV PYOPT=/opt/venv -# build and set open permissions on virtual environment -RUN python${PYVER} -m venv $PYOPT \ -&& chmod ugo+rwx $PYOPT - -# define python env vars. -# prepappending on PATH means all pip install will goto the PYOPT -ENV PATH=$PYOPT/bin:$PATH -ENV PYTHONPATH=$PYTHONPATH:$PYOPT/lib/python${PYVER}/site-packages -# taken from PETSC images - will be needed in the final image -ENV PETSC_DIR=/usr/local -ENV PYTHONPATH=$PYTHONPATH:$PETSC_DIR/lib - -#################### -# Stage 2: 'builder' -#################### -FROM petsc-image as builder -# defaults repeated deliberately — see the note in the runtime stage -ARG PYTHON_VERSION="3.12" -ARG UW3_BRANCH="development" -ARG UW3_REPO="https://github.com/underworldcode/underworld3.git" -ARG SETUPTOOLS_SCM_PRETEND_VERSION="" - -# root to install with yum -USER root -RUN yum install -y\ - cmake \ - make \ - git \ - gcc \ - gcc-c++ \ - gcc-gfortran \ - findutils \ - openmpi-devel \ -&& yum clean all \ -&& rm -rf /var/cache/yum -# NOTE: underworld2 does not yum install gcc - -# install python build time and runtime requirements here -RUN python${PYVER} -m pip install --no-cache-dir \ - "setuptools>=75" \ - "setuptools-scm>=8" \ - "cython>=3.1" \ - "scipy>=1.15" \ - "numpy<2" \ - matplotlib \ - pint \ - pytest \ - jupytext \ - sympy \ - "pydantic>=2" \ - pyyaml \ - psutil \ - typing_extensions \ - xxhash \ - requests \ - trimesh \ - ipython \ - "jupyterlab>=4.3,<5" \ - "ipywidgets<9.0.0" \ - jupyter-server-proxy \ - "trame>=2.5.2" \ - "trame-vtk>=2.5.8" \ - "trame-vuetify>=2.3.1" \ - cmocean \ - colorcet \ - imageio \ - imageio-ffmpeg \ - "rich<14" \ - meshio \ - pyvista -# gmsh wheels are only available for x86_64; skip gracefully on aarch64 (local Apple Silicon builds) -RUN pip install --no-cache-dir gmsh pygmsh || echo "gmsh/pygmsh not available on this platform, skipping" - -# H5PY with mpi - match pixi version constraint! -RUN CC=mpicc HDF5_MPI="ON" HDF5_DIR=${PETSC_DIR} \ - python${PYVER} -m pip install \ - --no-binary=h5py \ - --no-cache-dir \ - "h5py>=3.12" - -# separate so if this fails, don't need to reinstall everything again with layer caching -# vtk-osmesa wheels only available for x86_64; skip gracefully on aarch64 (local Apple Silicon builds) -RUN pip install --no-cache-dir --extra-index-url https://wheels.vtk.org vtk-osmesa \ - || echo "vtk-osmesa not available on this platform, skipping" - -# Clone and install uw3 -RUN git clone --depth 1 --branch ${UW3_BRANCH} \ - ${UW3_REPO} /tmp/underworld3 -WORKDIR /tmp/underworld3 -# The clone above is shallow and forks carry no tags, so setuptools-scm has -# nothing to derive from and would stamp the build 0.0.0. CI passes the real -# version; unset when empty so a local build keeps setuptools-scm's own logic. -RUN [ -n "$SETUPTOOLS_SCM_PRETEND_VERSION" ] || unset SETUPTOOLS_SCM_PRETEND_VERSION; \ - pip install --no-build-isolation --no-cache-dir . - -# record 'builder' stage packages used -RUN python${PYVER} -m pip freeze >/opt/requirements.txt \ -&& dnf history userinstalled >/opt/installed.txt - -#################### -# Stage 3: 'final', a combination of 'runtime' and 'builder' stages -#################### - -FROM runtime as final - -COPY --from=builder --chown=$NB_USER:users /opt /opt -COPY --from=builder --chown=$NB_USER:users /usr/local /usr/local - -# must make directory before COPY into it for permissions to work () -# set default viewer to a notebook see https://jupytext.readthedocs.io/en/latest/text-notebooks.html#with-a-double-click -RUN mkdir -p $NB_HOME/workspace $NB_HOME/Underworld3/ \ -&& chown $NB_USER:users -R $NB_HOME \ -&& jupyter-server extension enable --sys-prefix jupyter_server_proxy \ -&& jupytext-config set-default-viewer - -# confirm if these are the ones to put into the container -# copy examples, tests, etc. -COPY --chown=$NB_USER:users ./docs/examples $NB_HOME/Underworld3/examples -COPY --chown=$NB_USER:users ./docs/beginner $NB_HOME/Underworld3/beginner -COPY --chown=$NB_USER:users ./docs/advanced $NB_HOME/Underworld3/advanced - -EXPOSE 8888 -WORKDIR $NB_HOME -USER $NB_USER - -# Declare a volume space -VOLUME $NB_HOME/workspace - -CMD ["jupyter-lab", "--no-browser", "--ip='0.0.0.0'"] - diff --git a/docs/developer/hpc_containers/README.md b/docs/developer/hpc_containers/README.md new file mode 100644 index 000000000..3d0dbeb0a --- /dev/null +++ b/docs/developer/hpc_containers/README.md @@ -0,0 +1,97 @@ +# HPC containers for Underworld3 + +Singularity/Apptainer images for the HPC systems we run on, built by GitHub Actions and +published to GHCR. Site-specific run instructions (job templates, MPI settings, shared +image locations) live in +[underworld-community/uw3-hpc-install-run](https://github.com/underworld-community/uw3-hpc-install-run); +this directory is the recipes. + +## Layout + +One platform base per OS/MPI family; everything above it is shared. + +``` +base.rocky-ompi Rocky 8.10 + system OpenMPI -> uw3-base-rocky-ompi:{runtime,builder} Gadi, Kaiju +base.ubuntu-mpich Ubuntu 24.04 + MPICH 3.4.3 -> uw3-base-ubuntu-mpich:{runtime,builder} Setonix +petsc.hpc PETSc from source, on a base -> petsc-{gadi,setonix}:-{ompi,mpich} +underworld3.hpc UW3 + Python stack, on PETSc -> underworld3-{gadi,setonix}: +``` + +`petsc.hpc` and `underworld3.hpc` never call a package manager; they compile against +whatever `mpicc` the `:builder` base provides and lay the result over the `:runtime` base. +Adding a platform means adding a `base.*` file and a matrix row in each workflow. + +| Platform | Base | MPI in image | On the machine | +|---|---|---|---| +| Gadi | Rocky 8.10 | OpenMPI 4.1.1 (`libmpi.so.40`) | host `openmpi/4.1.7` injected via `LD_LIBRARY_PATH` — measured equal to bare metal | +| Kaiju | same image as Gadi | OpenMPI 4.1.1 | runs as-is under `srun --mpi=pmix` | +| Setonix | Ubuntu 24.04 | MPICH 3.4.3 (`libmpi.so.12`) | host Cray MPICH bind-mounted by `singularity/4.1.0-mpi` | + +Setonix needs 3.4.3 specifically: Cray MPICH 8.1 is MPICH-3.4-based, and code compiled +against MPICH 4.x headers references MPI-4 symbols the host library lacks. + +## Automated builds + +| Workflow | Builds | Runs when | +|---|---|---| +| `hpc-base-images.yml` | `base.*` | push touching `base.*`; dispatch | +| `hpc-petsc-image.yml` | `petsc.hpc` per platform | push touching `petsc.hpc` or `petsc-custom/patches/**`; dispatch | +| `hpc-uw3-image.yml` | `underworld3.hpc` per platform | push to `main`/`development` touching `src/**`, `pixi.toml`, `pyproject.toml`, `setup.py` or `underworld3.hpc`; `v*` tags; releases; dispatch | + +PETSc is ~25 min per platform, so it is a separate, rarely-triggered workflow and its +image is reused by every UW3 build. After a base change, rebuild PETSc, then UW3: + +```bash +gh workflow run hpc-petsc-image.yml -f petsc_version=3.25.0 -f make_np=4 +gh workflow run hpc-uw3-image.yml -f uw3_branch=v3.1.0 -f image_tag=v3.1.0 +``` + +Every build is smoke-tested before anything is pruned: `import underworld3`, the optional +gmsh/vtk-osmesa stack, and a 2-rank MPI run that asserts the MPI flavour (Open MPI vs +MPICH) — a silently wrong `libmpi` is the failure that would otherwise only show on a +compute node. + +Retention: the three most recent untagged `underworld3-*` versions are kept (two for +`petsc-*` and the bases). Tagged images are never pruned. + +Images inherit the repository's visibility. Packages created by hand (`podman push`) are +not repo-linked and default to private; make them public or the compute nodes cannot pull. + +The workflows publish single-platform `linux/amd64` with attestations off +(`provenance: false`, `sbom: false`); attestations add an `unknown/unknown` manifest entry +that some Singularity versions refuse. + +## Building by hand + +From the repository root, x86_64 only: + +```bash +podman build . --platform linux/amd64 --format docker --target builder \ + -t ghcr.io//uw3-base-rocky-ompi:builder -f docs/developer/hpc_containers/base.rocky-ompi +podman build . --platform linux/amd64 --format docker --target runtime \ + -t ghcr.io//uw3-base-rocky-ompi:runtime -f docs/developer/hpc_containers/base.rocky-ompi + +podman build . --platform linux/amd64 --format docker \ + --build-arg BASE_BUILDER=ghcr.io//uw3-base-rocky-ompi:builder \ + --build-arg BASE_RUNTIME=ghcr.io//uw3-base-rocky-ompi:runtime \ + -t ghcr.io//petsc-gadi:3.25.0-ompi -f docs/developer/hpc_containers/petsc.hpc + +podman build . --platform linux/amd64 --format docker \ + --build-arg PETSC_IMAGE=ghcr.io//petsc-gadi:3.25.0-ompi \ + --build-arg BASE_BUILDER=ghcr.io//uw3-base-rocky-ompi:builder \ + --build-arg UW3_BRANCH=development \ + -t ghcr.io//underworld3-gadi:development -f docs/developer/hpc_containers/underworld3.hpc +``` + +Substitute `ubuntu-mpich` / `setonix` / `mpich` for the Setonix chain. Pass +`--build-arg PETSC_MAKE_NP=2` inside a memory-constrained podman machine. + +## Pulling + +```bash +export SINGULARITY_CACHEDIR=/scratch///.singularity # not $HOME +module load singularity # singularity/4.1.0-mpi on Setonix +singularity pull docker://ghcr.io/underworldcode/underworld3-gadi:latest +``` + +How to run on each system is in the `uw3-hpc-install-run` repo. diff --git a/docs/developer/hpc_containers/underworld3.hpc b/docs/developer/hpc_containers/underworld3.hpc new file mode 100644 index 000000000..3b13c16cc --- /dev/null +++ b/docs/developer/hpc_containers/underworld3.hpc @@ -0,0 +1,118 @@ +##################################################################### +# Underworld3 image — platform-neutral. +# +# Builds UW3 and its Python stack on top of a petsc.hpc image, using the +# matching platform :builder base for the compile. No package manager is +# called here; see base.rocky-ompi for the contract. +# +# Stages: +# builder BASE_BUILDER + PETSc (copied from PETSC_IMAGE) + UW3 stack in /opt/venv +# final PETSC_IMAGE (which is already the :runtime base + PETSc) + /opt, /usr/local from builder +# +# podman build . --platform linux/amd64 --format docker \ +# --build-arg PETSC_IMAGE=ghcr.io//petsc-gadi:3.25.0-ompi \ +# --build-arg BASE_BUILDER=ghcr.io//uw3-base-rocky-ompi:builder \ +# --build-arg UW3_BRANCH=development \ +# -t ghcr.io//underworld3-gadi:development \ +# -f docs/developer/hpc_containers/underworld3.hpc +##################################################################### + +ARG PETSC_IMAGE="ghcr.io/underworldcode/petsc-gadi:3.25.0-ompi" +ARG BASE_BUILDER="ghcr.io/underworldcode/uw3-base-rocky-ompi:builder" +ARG UW3_BRANCH="development" +# Overridden by CI to the repository being built, so a fork builds its own +# branches rather than looking for them upstream. +ARG UW3_REPO="https://github.com/underworldcode/underworld3.git" + +FROM ${PETSC_IMAGE} AS petsc-image + +FROM ${BASE_BUILDER} AS builder +# defaults repeated deliberately — see the note in base.rocky-ompi +ARG UW3_BRANCH="development" +ARG UW3_REPO="https://github.com/underworldcode/underworld3.git" +ARG SETUPTOOLS_SCM_PRETEND_VERSION="" + +# PETSc and its venv (petsc4py, mpi4py, numpy) come from the PETSc image; the +# compilers from the base. Both venvs were created identically by the base, +# so the copy is a superset, not a conflict. +COPY --from=petsc-image /opt /opt +COPY --from=petsc-image /usr/local /usr/local +ENV PETSC_DIR=/usr/local +ENV PYTHONPATH=$PYTHONPATH:$PETSC_DIR/lib + +RUN pip install --no-cache-dir \ + "setuptools>=75" \ + "setuptools-scm>=8" \ + "cython>=3.1" \ + "scipy>=1.15" \ + "numpy<2" \ + matplotlib \ + pint \ + pytest \ + jupytext \ + sympy \ + "pydantic>=2" \ + pyyaml \ + psutil \ + typing_extensions \ + xxhash \ + requests \ + trimesh \ + ipython \ + "jupyterlab>=4.3,<5" \ + "ipywidgets<9.0.0" \ + jupyter-server-proxy \ + "trame>=2.5.2" \ + "trame-vtk>=2.5.8" \ + "trame-vuetify>=2.3.1" \ + cmocean \ + colorcet \ + imageio \ + imageio-ffmpeg \ + "rich<14" \ + meshio \ + pyvista +# gmsh wheels are x86_64-only; skip gracefully on aarch64 (local Apple Silicon builds) +RUN pip install --no-cache-dir gmsh pygmsh || echo "gmsh/pygmsh not available on this platform, skipping" + +# Parallel h5py against PETSc's HDF5 — match the pixi version constraint. +RUN CC=mpicc HDF5_MPI="ON" HDF5_DIR=${PETSC_DIR} \ + pip install --no-binary=h5py --no-cache-dir "h5py>=3.12" + +# Separate layer so a failure here does not redo everything above. +RUN pip install --no-cache-dir --extra-index-url https://wheels.vtk.org vtk-osmesa \ + || echo "vtk-osmesa not available on this platform, skipping" + +RUN git clone --depth 1 --branch ${UW3_BRANCH} ${UW3_REPO} /tmp/underworld3 +WORKDIR /tmp/underworld3 +# The clone above is shallow and forks carry no tags, so setuptools-scm has +# nothing to derive from and would stamp the build 0.0.0. CI passes the real +# version; unset when empty so a local build keeps setuptools-scm's own logic. +RUN [ -n "$SETUPTOOLS_SCM_PRETEND_VERSION" ] || unset SETUPTOOLS_SCM_PRETEND_VERSION; \ + pip install --no-build-isolation --no-cache-dir . + +RUN pip freeze > /opt/requirements.txt + +##################################################################### +FROM ${PETSC_IMAGE} AS final +USER root + +COPY --from=builder --chown=$NB_USER:users /opt /opt +COPY --from=builder --chown=$NB_USER:users /usr/local /usr/local + +# jupytext default viewer: text notebooks open as notebooks on double-click. +RUN mkdir -p $NB_HOME/workspace $NB_HOME/Underworld3/ \ +&& chown $NB_USER:users -R $NB_HOME \ +&& jupyter-server extension enable --sys-prefix jupyter_server_proxy \ +&& jupytext-config set-default-viewer + +COPY --chown=$NB_USER:users ./docs/examples $NB_HOME/Underworld3/examples +COPY --chown=$NB_USER:users ./docs/beginner $NB_HOME/Underworld3/beginner +COPY --chown=$NB_USER:users ./docs/advanced $NB_HOME/Underworld3/advanced + +EXPOSE 8888 +WORKDIR $NB_HOME +USER $NB_USER +VOLUME $NB_HOME/workspace + +CMD ["jupyter-lab", "--no-browser", "--ip='0.0.0.0'"] From 6781784ecf8afe15271a831d79eab9d5aebf954d Mon Sep 17 00:00:00 2001 From: Juan Carlos Graciosa Date: Mon, 21 Sep 2026 12:52:51 +1000 Subject: [PATCH 6/8] ci(container): drop the fork test-branch triggers --- .github/workflows/hpc-base-images.yml | 1 - .github/workflows/hpc-petsc-image.yml | 1 - .github/workflows/hpc-uw3-image.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/hpc-base-images.yml b/.github/workflows/hpc-base-images.yml index 9fd877d75..bc83c477d 100644 --- a/.github/workflows/hpc-base-images.yml +++ b/.github/workflows/hpc-base-images.yml @@ -12,7 +12,6 @@ on: branches: - main - development - - ci/gadi-container # TESTING ONLY — remove before the upstream PR paths: - 'docs/developer/hpc_containers/base.*' - '.github/workflows/hpc-base-images.yml' diff --git a/.github/workflows/hpc-petsc-image.yml b/.github/workflows/hpc-petsc-image.yml index 3710da920..a651bc926 100644 --- a/.github/workflows/hpc-petsc-image.yml +++ b/.github/workflows/hpc-petsc-image.yml @@ -12,7 +12,6 @@ on: branches: - main - development - - ci/gadi-container # TESTING ONLY — remove before the upstream PR paths: - 'docs/developer/hpc_containers/petsc.hpc' - 'petsc-custom/patches/**' diff --git a/.github/workflows/hpc-uw3-image.yml b/.github/workflows/hpc-uw3-image.yml index c9e900e34..494723b90 100644 --- a/.github/workflows/hpc-uw3-image.yml +++ b/.github/workflows/hpc-uw3-image.yml @@ -13,7 +13,6 @@ on: branches: - main - development - - ci/gadi-container # TESTING ONLY — remove before the upstream PR tags: ['v*'] paths: - 'docs/developer/hpc_containers/underworld3.hpc' From 3bc6b5740f70d433c1bbfb1fd977c8ed1c598c60 Mon Sep 17 00:00:00 2001 From: Juan Carlos Graciosa Date: Mon, 21 Sep 2026 13:01:05 +1000 Subject: [PATCH 7/8] ci(container): stand the UW3 final stage on the runtime base, not the PETSc image --- .github/workflows/hpc-uw3-image.yml | 2 ++ docs/developer/hpc_containers/underworld3.hpc | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/hpc-uw3-image.yml b/.github/workflows/hpc-uw3-image.yml index 494723b90..a195a5869 100644 --- a/.github/workflows/hpc-uw3-image.yml +++ b/.github/workflows/hpc-uw3-image.yml @@ -13,6 +13,7 @@ on: branches: - main - development + - ci/gadi-container # TESTING ONLY — remove before the upstream PR tags: ['v*'] paths: - 'docs/developer/hpc_containers/underworld3.hpc' @@ -176,6 +177,7 @@ jobs: UW3_REPO=${{ github.server_url }}/${{ github.repository }}.git PETSC_IMAGE=${{ steps.meta.outputs.petsc_image }} BASE_BUILDER=${{ steps.meta.outputs.base }}:builder + BASE_RUNTIME=${{ steps.meta.outputs.base }}:runtime SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.meta.outputs.uw3_version }} tags: ${{ steps.meta.outputs.tags }} # Cache lives in its own package: sharing one with the image put stale diff --git a/docs/developer/hpc_containers/underworld3.hpc b/docs/developer/hpc_containers/underworld3.hpc index 3b13c16cc..7328b75b6 100644 --- a/docs/developer/hpc_containers/underworld3.hpc +++ b/docs/developer/hpc_containers/underworld3.hpc @@ -7,11 +7,16 @@ # # Stages: # builder BASE_BUILDER + PETSc (copied from PETSC_IMAGE) + UW3 stack in /opt/venv -# final PETSC_IMAGE (which is already the :runtime base + PETSc) + /opt, /usr/local from builder +# final BASE_RUNTIME + /opt, /usr/local from builder +# +# final stands on the bare runtime base, not on PETSC_IMAGE: the builder's /opt +# and /usr/local already contain PETSc, so layering them over the PETSc image +# would ship PETSc twice (~200 MB more to pull; the SIF is unaffected). # # podman build . --platform linux/amd64 --format docker \ # --build-arg PETSC_IMAGE=ghcr.io//petsc-gadi:3.25.0-ompi \ # --build-arg BASE_BUILDER=ghcr.io//uw3-base-rocky-ompi:builder \ +# --build-arg BASE_RUNTIME=ghcr.io//uw3-base-rocky-ompi:runtime \ # --build-arg UW3_BRANCH=development \ # -t ghcr.io//underworld3-gadi:development \ # -f docs/developer/hpc_containers/underworld3.hpc @@ -19,6 +24,7 @@ ARG PETSC_IMAGE="ghcr.io/underworldcode/petsc-gadi:3.25.0-ompi" ARG BASE_BUILDER="ghcr.io/underworldcode/uw3-base-rocky-ompi:builder" +ARG BASE_RUNTIME="ghcr.io/underworldcode/uw3-base-rocky-ompi:runtime" ARG UW3_BRANCH="development" # Overridden by CI to the repository being built, so a fork builds its own # branches rather than looking for them upstream. @@ -94,12 +100,16 @@ RUN [ -n "$SETUPTOOLS_SCM_PRETEND_VERSION" ] || unset SETUPTOOLS_SCM_PRETEND_VER RUN pip freeze > /opt/requirements.txt ##################################################################### -FROM ${PETSC_IMAGE} AS final -USER root +FROM ${BASE_RUNTIME} AS final COPY --from=builder --chown=$NB_USER:users /opt /opt COPY --from=builder --chown=$NB_USER:users /usr/local /usr/local +# Same as petsc.hpc's final stage — not inherited, since this stage does not +# stand on that image. +ENV PETSC_DIR=/usr/local +ENV PYTHONPATH=$PYTHONPATH:$PETSC_DIR/lib + # jupytext default viewer: text notebooks open as notebooks on double-click. RUN mkdir -p $NB_HOME/workspace $NB_HOME/Underworld3/ \ && chown $NB_USER:users -R $NB_HOME \ From 369635c729d36bdeebda9c59b3baea068d542120 Mon Sep 17 00:00:00 2001 From: Juan Carlos Graciosa Date: Mon, 21 Sep 2026 13:15:47 +1000 Subject: [PATCH 8/8] ci(container): drop the last fork test-branch trigger --- .github/workflows/hpc-uw3-image.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/hpc-uw3-image.yml b/.github/workflows/hpc-uw3-image.yml index a195a5869..03a0694d3 100644 --- a/.github/workflows/hpc-uw3-image.yml +++ b/.github/workflows/hpc-uw3-image.yml @@ -13,7 +13,6 @@ on: branches: - main - development - - ci/gadi-container # TESTING ONLY — remove before the upstream PR tags: ['v*'] paths: - 'docs/developer/hpc_containers/underworld3.hpc'