diff --git a/.github/workflows/hpc-base-images.yml b/.github/workflows/hpc-base-images.yml new file mode 100644 index 000000000..bc83c477d --- /dev/null +++ b/.github/workflows/hpc-base-images.yml @@ -0,0 +1,112 @@ +name: "GHCR: HPC platform base images" + +# Builds docs/developer/hpc_containers/base.* — one image per platform family, +# published as :runtime and :builder tags. Everything OS/MPI-specific lives in +# these; hpc-petsc-image.yml and hpc-uw3-image.yml stack on them. +# +# Changes rarely. Both downstream workflows pull the tags by name, so after a +# base change rebuild PETSc (hpc-petsc-image.yml), then UW3. + +on: + push: + branches: + - main + - development + paths: + - 'docs/developer/hpc_containers/base.*' + - '.github/workflows/hpc-base-images.yml' + workflow_dispatch: + inputs: + force_rebuild: + description: 'Force full rebuild (no cache)' + type: boolean + default: false + +concurrency: + group: hpc-base-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + base: [rocky-ompi, ubuntu-mpich] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve image name + id: meta + run: | + OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + echo "image=ghcr.io/${OWNER}/uw3-base-${{ matrix.base }}" >> "$GITHUB_OUTPUT" + + # Two targets of one file, pushed as two tags. Docker v2 media types and + # no attestations, for the same Singularity reasons as hpc-uw3-image.yml. + - name: Build and push :runtime + uses: docker/build-push-action@v6 + with: + context: . + file: docs/developer/hpc_containers/base.${{ matrix.base }} + target: runtime + outputs: type=image,push=true,oci-mediatypes=false + platforms: linux/amd64 + provenance: false + sbom: false + no-cache: ${{ inputs.force_rebuild || false }} + build-args: PYTHON_VERSION=3.12 + tags: ${{ steps.meta.outputs.image }}:runtime + + - name: Build and push :builder + uses: docker/build-push-action@v6 + with: + context: . + file: docs/developer/hpc_containers/base.${{ matrix.base }} + target: builder + outputs: type=image,push=true,oci-mediatypes=false + platforms: linux/amd64 + provenance: false + sbom: false + no-cache: ${{ inputs.force_rebuild || false }} + build-args: PYTHON_VERSION=3.12 + tags: ${{ steps.meta.outputs.image }}:builder + + # The contract the recipes rely on: mpicc and a venv python on PATH. + # command -v, not which: Rocky's minimal image has no `which`. + - name: Smoke test + run: | + docker run --rm ${{ steps.meta.outputs.image }}:builder bash -c ' + set -e + for t in mpicc g++ gfortran cmake flex bison patch git wget; do command -v $t >/dev/null || { echo "missing $t"; exit 1; }; done + mpicc --version | head -1 + python -c "import sys; assert sys.prefix == \"/opt/venv\", sys.prefix; print(sys.version)"' + docker run --rm ${{ steps.meta.outputs.image }}:runtime bash -c ' + set -e + for t in mpiexec python g++; do command -v $t >/dev/null || { echo "missing $t"; exit 1; }; done + id jovyan' + + - name: Delete old untagged image versions + continue-on-error: true + uses: actions/delete-package-versions@v5 + with: + owner: ${{ github.repository_owner }} + package-name: uw3-base-${{ matrix.base }} + package-type: container + min-versions-to-keep: 2 + delete-only-untagged-versions: true diff --git a/.github/workflows/hpc-petsc-image.yml b/.github/workflows/hpc-petsc-image.yml new file mode 100644 index 000000000..a651bc926 --- /dev/null +++ b/.github/workflows/hpc-petsc-image.yml @@ -0,0 +1,160 @@ +name: "GHCR: HPC PETSc images" + +# Builds docs/developer/hpc_containers/petsc.hpc once per platform base — the +# from-source PETSc that hpc-uw3-image.yml layers on. +# +# Triggers are deliberately narrow: this compiles PETSc with mumps, hypre, +# superlu_dist, mmg/parmmg, ptscotch and slepc (~25 min each), and only +# petsc.hpc, its patches and the bases can change the result. + +on: + push: + branches: + - main + - development + paths: + - 'docs/developer/hpc_containers/petsc.hpc' + - 'petsc-custom/patches/**' + - '.github/workflows/hpc-petsc-image.yml' + workflow_dispatch: + inputs: + petsc_version: + description: 'PETSc version to build' + type: string + default: '3.25.0' + make_np: + description: 'PETSc build parallelism (--with-make-np)' + type: string + default: '4' + force_rebuild: + description: 'Force full rebuild (no cache)' + type: boolean + default: false + +concurrency: + group: hpc-petsc-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + # Measured at ~25 min with ~25% run-to-run variance; ~5x headroom. + timeout-minutes: 120 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - platform: gadi + base: rocky-ompi + mpi: ompi + mpi_name: "Open MPI" + - platform: setonix + base: ubuntu-mpich + mpi: mpich + mpi_name: "MPICH" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Report disk headroom + run: df -h / + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve names and tags + id: meta + run: | + set -euo pipefail + OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + IMAGE="ghcr.io/${OWNER}/petsc-${{ matrix.platform }}" + BASE="ghcr.io/${OWNER}/uw3-base-${{ matrix.base }}" + + PETSC_VERSION="${{ inputs.petsc_version }}" + PETSC_VERSION="${PETSC_VERSION:-3.25.0}" + MAKE_NP="${{ inputs.make_np }}" + MAKE_NP="${MAKE_NP:-4}" + + { + echo "image=${IMAGE}:${PETSC_VERSION}-${{ matrix.mpi }}" + echo "tags=${IMAGE}:${PETSC_VERSION}-${{ matrix.mpi }},${IMAGE}:latest" + echo "base=${BASE}" + echo "petsc_version=${PETSC_VERSION}" + echo "make_np=${MAKE_NP}" + } >> "$GITHUB_OUTPUT" + echo "Building ${IMAGE}:${PETSC_VERSION}-${{ matrix.mpi }} on ${BASE} with --with-make-np=${MAKE_NP}" + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: docs/developer/hpc_containers/petsc.hpc + # Docker v2 media types, no attestations — see hpc-uw3-image.yml. + outputs: type=image,push=true,oci-mediatypes=false + platforms: linux/amd64 + provenance: false + sbom: false + no-cache: ${{ inputs.force_rebuild || false }} + build-args: | + BASE_BUILDER=${{ steps.meta.outputs.base }}:builder + BASE_RUNTIME=${{ steps.meta.outputs.base }}:runtime + PETSC_VERSION=${{ steps.meta.outputs.petsc_version }} + PETSC_MAKE_NP=${{ steps.meta.outputs.make_np }} + tags: ${{ steps.meta.outputs.tags }} + + - name: Report build size + run: | + docker image ls ${{ steps.meta.outputs.image }} || true + df -h / + + - name: Pull image + run: docker pull ${{ steps.meta.outputs.image }} + + # petsc4py importing is what actually breaks when the build goes subtly + # wrong (shared libraries, PYTHONPATH, the meson petsc4py step). + - name: Import petsc4py and slepc4py + run: | + docker run --rm ${{ steps.meta.outputs.image }} python -c " + import petsc4py, slepc4py + petsc4py.init() + from petsc4py import PETSc + print('petsc4py', petsc4py.__version__) + print('slepc4py', slepc4py.__version__) + print('PETSc', PETSc.Sys.getVersion()) + " + + # Assert the MPI flavour: a silently wrong libmpi is exactly the failure + # that would otherwise only surface on a compute node. + - name: MPI sanity (2 ranks, expect ${{ matrix.mpi_name }}) + run: | + docker run --rm ${{ steps.meta.outputs.image }} \ + mpiexec -n 2 python -c " + from mpi4py import MPI + c = MPI.COMM_WORLD + v = MPI.Get_library_version().splitlines()[0] + print(f'rank {c.rank} of {c.size}: {v}') + assert c.size == 2 + assert '${{ matrix.mpi_name }}' in v, v + " + + # Housekeeping only; the action 404s partway through on user-owned + # packages, having already deleted what it could. + - name: Delete old untagged image versions + continue-on-error: true + uses: actions/delete-package-versions@v5 + with: + owner: ${{ github.repository_owner }} + package-name: petsc-${{ matrix.platform }} + package-type: container + min-versions-to-keep: 2 + delete-only-untagged-versions: true diff --git a/.github/workflows/hpc-uw3-image.yml b/.github/workflows/hpc-uw3-image.yml new file mode 100644 index 000000000..03a0694d3 --- /dev/null +++ b/.github/workflows/hpc-uw3-image.yml @@ -0,0 +1,235 @@ +name: "GHCR: HPC Underworld3 images" + +# Builds docs/developer/hpc_containers/underworld3.hpc once per platform and +# pushes to GHCR for `singularity pull docker://...` on Gadi, Kaiju and Setonix. +# +# This is the layer that moves; the expensive PETSc images under it are built +# rarely by hpc-petsc-image.yml, and the platform bases rarer still by +# hpc-base-images.yml. Unlike docker-image.yaml and binder-image.yml these +# images match each HPC system's OS and MPI ABI. + +on: + push: + branches: + - main + - development + tags: ['v*'] + paths: + - 'docs/developer/hpc_containers/underworld3.hpc' + - '.github/workflows/hpc-uw3-image.yml' + - 'pixi.toml' + - 'pyproject.toml' + - 'setup.py' + - 'src/**' + # Releases are the guaranteed path for a tagged image: the `tags:` filter above + # is combined with `paths:`, so a tag whose diff touches nothing in that list + # would not build on its own. + release: + types: [published] + workflow_dispatch: + inputs: + uw3_branch: + description: 'UW3 branch/tag to clone and build (e.g. development, v3.1.0). Empty = triggering ref.' + type: string + default: '' + petsc_image: + description: 'PETSc base image override. Applies to EVERY platform in the matrix, so only useful with a single-platform matrix edit. Empty = per-platform default.' + type: string + default: '' + image_tag: + description: 'Image tag override (e.g. ci-test). Empty = derive from ref.' + type: string + default: '' + force_rebuild: + description: 'Force full rebuild (no cache)' + type: boolean + default: false + +concurrency: + group: hpc-uw3-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - platform: gadi + base: rocky-ompi + mpi: ompi + mpi_name: "Open MPI" + petsc_var: GADI_PETSC_IMAGE + - platform: setonix + base: ubuntu-mpich + mpi: mpich + mpi_name: "MPICH" + petsc_var: SETONIX_PETSC_IMAGE + env: + # Repository variables that let a fork reuse a PETSc image it already + # published under another name. Indexed by matrix.petsc_var below. + GADI_PETSC_IMAGE: ${{ vars.GADI_PETSC_IMAGE }} + SETONIX_PETSC_IMAGE: ${{ vars.SETONIX_PETSC_IMAGE }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve names and tags + id: meta + run: | + set -euo pipefail + + # Image names must be lowercase; repository_owner may not be. + OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + IMAGE="ghcr.io/${OWNER}/underworld3-${{ matrix.platform }}" + BASE="ghcr.io/${OWNER}/uw3-base-${{ matrix.base }}" + + # Same ref-name / IS_RELEASE convention as binder-image.yml. + if [[ "$GITHUB_REF" == refs/tags/* ]]; then + REF_NAME=${GITHUB_REF#refs/tags/} + IS_RELEASE=true + else + REF_NAME=${GITHUB_REF#refs/heads/} + IS_RELEASE=false + fi + + if [[ -n "${{ inputs.uw3_branch }}" ]]; then + REF_NAME="${{ inputs.uw3_branch }}" + [[ "$REF_NAME" == v* ]] && IS_RELEASE=true + fi + if [[ -n "${{ inputs.image_tag }}" ]]; then + DOCKER_TAG="${{ inputs.image_tag }}" + else + # Docker tags cannot contain '/' + DOCKER_TAG=$(echo "$REF_NAME" | tr '/' '-') + fi + + # Dispatch input wins, then the per-platform repo variable, then the + # canonical name. + VAR_NAME="${{ matrix.petsc_var }}" + PETSC_IMAGE="${{ inputs.petsc_image }}" + PETSC_IMAGE="${PETSC_IMAGE:-${!VAR_NAME}}" + PETSC_IMAGE="${PETSC_IMAGE:-ghcr.io/${OWNER}/petsc-${{ matrix.platform }}:3.25.0-${{ matrix.mpi }}}" + + TAGS="${IMAGE}:${DOCKER_TAG}" + if [[ "$IS_RELEASE" == "true" ]]; then + TAGS="${TAGS},${IMAGE}:latest" + fi + + # A release tag is the version; anything else gets a PEP440 dev + # version carrying the commit, since branch names are not versions. + if [[ "$IS_RELEASE" == "true" ]]; then + UW3_VERSION="${REF_NAME#v}" + else + UW3_VERSION="0.0.0.dev0+g${GITHUB_SHA:0:7}" + fi + + { + echo "image=${IMAGE}:${DOCKER_TAG}" + echo "tags=${TAGS}" + echo "base=${BASE}" + echo "uw3_branch=${REF_NAME}" + echo "petsc_image=${PETSC_IMAGE}" + echo "uw3_version=${UW3_VERSION}" + } >> "$GITHUB_OUTPUT" + + echo "Building ${IMAGE}:${DOCKER_TAG}" + echo " UW3 ref: ${REF_NAME}" + echo " PETSc base: ${PETSC_IMAGE}" + echo " builder: ${BASE}:builder" + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: docs/developer/hpc_containers/underworld3.hpc + # Docker v2 media types rather than buildx's OCI default, matching the + # podman `--format docker` images already validated on Gadi. Older + # Singularity builds read OCI patchily; this removes the variable. + outputs: type=image,push=true,oci-mediatypes=false + # All three systems are x86_64. No arm64: it would need QEMU for no + # benefit, and the gmsh / vtk-osmesa wheels are x86_64-only anyway. + platforms: linux/amd64 + # Attestations turn the result into a multi-manifest index containing an + # "unknown/unknown" entry, which some Singularity/Apptainer versions + # refuse to pull. Keep these off — see README "Automated builds". + provenance: false + sbom: false + no-cache: ${{ inputs.force_rebuild || false }} + build-args: | + UW3_BRANCH=${{ steps.meta.outputs.uw3_branch }} + UW3_REPO=${{ github.server_url }}/${{ github.repository }}.git + PETSC_IMAGE=${{ steps.meta.outputs.petsc_image }} + BASE_BUILDER=${{ steps.meta.outputs.base }}:builder + BASE_RUNTIME=${{ steps.meta.outputs.base }}:runtime + SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.meta.outputs.uw3_version }} + tags: ${{ steps.meta.outputs.tags }} + # Cache lives in its own package: sharing one with the image put stale + # cache manifests into the same untagged pool the cleanup step prunes, + # so they competed with old images for min-versions-to-keep. + cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/underworld3-${{ matrix.platform }}-buildcache:latest + cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/underworld3-${{ matrix.platform }}-buildcache:latest,mode=max + + - name: Pull image + run: docker pull ${{ steps.meta.outputs.image }} + + # A bare import is the cheapest check and catches a missing runtime dep + # (this is exactly how the absent `requests` once showed up). + - name: Import underworld3 + run: | + docker run --rm ${{ steps.meta.outputs.image }} \ + python -c "import underworld3; print('underworld3', underworld3.__version__)" + + # The recipe installs gmsh and vtk-osmesa behind `|| echo skip`, so a + # failed install is silent. Import them explicitly or it ships broken. + - name: Import optional stack (gmsh, vtk, pyvista, h5py, petsc4py) + run: | + docker run --rm ${{ steps.meta.outputs.image }} \ + python -c "import gmsh, vtk, pyvista, h5py, petsc4py; print('optional stack OK')" + + # Assert the MPI flavour too: a silently wrong libmpi is exactly the + # failure that would otherwise only surface on a compute node. + - name: MPI sanity (2 ranks, expect ${{ matrix.mpi_name }}) + run: | + docker run --rm ${{ steps.meta.outputs.image }} \ + mpiexec -n 2 python -c " + import underworld3 as uw + from mpi4py import MPI + c = MPI.COMM_WORLD + v = MPI.Get_library_version().splitlines()[0] + assert c.size == 2 + assert '${{ matrix.mpi_name }}' in v, v + print(f'rank {c.rank} of {c.size}, underworld3 {uw.__version__}, {v}') + " + + # Only prune once the image is known good — a failed step above stops + # the job before this runs, so a broken build never deletes the last + # working one. Housekeeping only: a 404 partway through must not fail + # a run whose image is good. + - name: Delete old untagged image versions + continue-on-error: true + uses: actions/delete-package-versions@v5 + with: + owner: ${{ github.repository_owner }} + package-name: underworld3-${{ matrix.platform }} + package-type: container + min-versions-to-keep: 3 + # Load-bearing: every TAGGED image (:latest, :v3.1.0, :main, and the + # current :development) is exempt regardless of age. Only superseded + # rolling builds are eligible. + delete-only-untagged-versions: true diff --git a/docs/developer/gadi_singularity/README.md b/docs/developer/gadi_singularity/README.md deleted file mode 100644 index 6b579a451..000000000 --- a/docs/developer/gadi_singularity/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# Building Underworld3 for Gadi (NCI) - -This directory contains two Containerfiles to build the Underworld3 (UW3) Singularity image for Gadi (nci.org.au). - -Both use Rocky Linux 8.10 to match Gadi's OS for ABI compatibility. - -## Build Order - -Build commands must be run from the top-level `underworld3/` directory (the build context). -Builds targeting Gadi must use `--platform linux/amd64`. - -### 1. Build PETSc layer - -```bash -podman build . \ - --platform linux/amd64 \ - --format docker \ - -t ghcr.io//petsc:3.25.0-ompi \ - -f ./docs/developer/gadi_singularity/petsc.rhel -``` - -### 2. Push PETSc image to registry - -```bash -podman push ghcr.io//petsc:3.25.0-ompi -``` - -### 3. Build Underworld3 - -```bash -podman build . \ - --platform linux/amd64 \ - --format docker \ - --build-arg PETSC_IMAGE=ghcr.io//petsc:3.25.0-ompi \ - --build-arg UW3_BRANCH=development \ - -t ghcr.io//underworld3-gadi:latest \ - -f ./docs/developer/gadi_singularity/underworld3.rhel -``` - -### 4. Push Underworld3 image - -```bash -podman push ghcr.io//underworld3-gadi:latest -``` - -## What Each File Does - -- **petsc.rhel** — Builds PETSc 3.25.0 with full AMR support (petsc4py, slepc4py, mmg, parmmg, etc.) -- **underworld3.rhel** — Builds Underworld3 on top of the PETSc image - -## Running on Gadi - -Pull the image on Gadi (redirect cache to scratch to avoid home quota issues): - -```bash -export SINGULARITY_CACHEDIR=/scratch///.singularity -module load singularity -singularity pull docker://ghcr.io//underworld3-gadi:latest -``` - -Run a script with MPI: - -```bash -module load singularity -module load openmpi/4.1.7 -mpiexec -n singularity exec underworld3-gadi_latest.sif python3 -``` - -## Notes - -- OpenFabrics (mlx5_0) warnings in the job error log are harmless -- PostHog telemetry failures on compute nodes are harmless (no outbound internet) -- The ghcr.io images must be set to **public** for Singularity to pull without authentication diff --git a/docs/developer/gadi_singularity/petsc.rhel b/docs/developer/gadi_singularity/petsc.rhel deleted file mode 100644 index e612826dc..000000000 --- a/docs/developer/gadi_singularity/petsc.rhel +++ /dev/null @@ -1,197 +0,0 @@ -##################################################################### -# UW3 PETSc container -# Multi stage Containerfile based on UW2 version -# This builds PETSc according to the pixi amr-dev environment -# see https://docs.docker.com/get-started/docker-concepts/building-images/multi-stage-builds/ -# -# Stages: -# 1. 'runtime' -# The runtime environment (packages, permissions, ENV vars.) -# is consistent accross all stages of this Containerfile. -# -# 2. 'builder' -# The builder layer, takes the runtime layer and add compiling / building software -# that is added to /usr/local and /opt/venv -# -# 3. 'final' == runtime + min. builder -# The final image is a composite of the runtime layer and the -# minimal sections of the builder layer's final software stack. -# -# To build use podman from the top level underworld directory. i.e. -# $ podman build . \ -# --platform linux/amd64 \ -# --format docker \ -# -t new_image_name \ -# -f ./docs/developer/gadi_singularity/petsc.rhel -##################################################################### - -# The following are passed in via --build-args -# Must go before the 1st FROM see -# https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact -ARG PYTHON_VERSION="3.12" -ARG PETSC_VERSION="3.25.0" -ARG BASE_IMAGE="quay.io/rockylinux/rockylinux:8.10" - -# 1. Stage 1: 'runtime' -FROM ${BASE_IMAGE} as runtime -LABEL maintainer="https://github.com/underworldcode/" - -# need to repeat ARGS after every FROM -ARG PYTHON_VERSION - -#### Containerfile ENV vars - for all image stages -ENV LANG=C.UTF-8 -ENV PYVER=${PYTHON_VERSION} - -# gadi-specific settings -ENV OPENBLAS_NUM_THREADS=1 -ENV OMPI_MCA_io=ompio - -# add user jovyan -ENV NB_USER jovyan -ENV NB_HOME /home/$NB_USER -RUN useradd -m -s /bin/bash -N $NB_USER - -RUN yum update -y \ -&& yum install -y \ - bash-completion \ - openssh \ - openblas \ - python${PYVER}-pip \ - python${PYVER}-devel \ - openmpi \ - findutils \ -&& yum clean all \ -&& rm -rf /var/cache/yum - -# add system openmpi to $PATH and $LD_LIBRARY_PATH -ENV PATH=/usr/lib64/openmpi/bin:$PATH -ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - -ENV PYOPT=/opt/venv -# build and set open permissions on virtual environment -RUN python${PYVER} -m venv $PYOPT \ -&& chmod ugo+rwx $PYOPT - -# define python env vars. -# prepappending on PATH means all pip install will goto the PYOPT -ENV PATH=$PYOPT/bin:$PATH -ENV PYTHONPATH=$PYTHONPATH:$PYOPT/lib/python${PYVER}/site-packages - -# runtime python requirements -RUN python${PYVER} -m pip install wheel \ - "numpy<2" - -# 2. Define the builder layer -FROM runtime as builder - -ARG PETSC_VERSION -ARG PYTHON_VERSION - -RUN yum install -y \ - ca-certificates \ - wget \ - make \ - gcc \ - gcc-gfortran \ - gcc-c++ \ - cmake \ - patch \ - openblas \ - zlib-devel \ - openmpi-devel \ - findutils \ - git \ - flex \ - bison \ -&& yum clean all \ -&& rm -rf /var/cache/yum -# NOTE flex and bison are needed to build - -RUN python${PYVER} -m pip install "cython>=3.1" \ - "setuptools>=75" \ - "meson" \ - "meson-python" \ - "ninja" \ -&& python${PYVER} -m pip install --no-cache-dir --no-binary=mpi4py "mpi4py>=4,<5" -# no-binary for mpi4py to force build against openmpi, rather than mpich (default) - -# copy patches into builder -COPY petsc-custom/patches/scotch-7.0.10-c23-fix.tar.gz /tmp/scotch.tar.gz -COPY petsc-custom/patches/plexfem-internal-boundary-ownership-fix.patch /tmp/ - -# get petsc -RUN mkdir -p /tmp/src -WORKDIR /tmp/src -RUN wget https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-lite-${PETSC_VERSION}.tar.gz --no-check-certificate \ -&& tar -zxf petsc-lite-${PETSC_VERSION}.tar.gz -WORKDIR /tmp/src/petsc-${PETSC_VERSION} - -# apply patch then configure -# patch may already be included in newer PETSc versions - skip gracefully if it doesn't apply -RUN if patch -p1 --dry-run < /tmp/plexfem-internal-boundary-ownership-fix.patch 2>/dev/null; then \ - patch -p1 < /tmp/plexfem-internal-boundary-ownership-fix.patch; \ - echo "plexfem patch applied successfully"; \ - else \ - echo "plexfem patch not applicable (may already be in this PETSc version), skipping"; \ - fi -RUN python${PYVER} ./configure \ - --with-debugging=0 \ - --prefix=/usr/local \ - --with-shared-libraries=1 \ - --with-cxx-dialect=C++11 \ - "--COPTFLAGS=-g -O3" "--CXXOPTFLAGS=-g -O3" "--FOPTFLAGS=-g -O3" \ - --useThreads=0 \ - --with-x=0 \ - --with-pragmatic=1 \ - --with-petsc4py=1 \ - --with-slepc4py=1 \ - --download-eigen=1 \ - --download-metis=1 \ - --download-parmetis=1 \ - --download-mumps=1 \ - --download-scalapack=1 \ - --download-hypre=1 \ - --download-superlu=1 \ - --download-superlu_dist=1 \ - --download-mmg=1 \ - "--download-mmg-cmake-arguments=-DMMG_INSTALL_PRIVATE_HEADERS=ON -DUSE_SCOTCH=OFF" \ - --download-parmmg=1 \ - --download-pragmatic=1 \ - "--download-ptscotch=/tmp/scotch.tar.gz" \ - --download-slepc=1 \ - --download-hdf5=1 \ - --download-fblaslapack=1 \ - --download-zlib=1 \ - --download-ctetgen=1 \ - --download-triangle=1 \ - --with-make-np=2 -RUN make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt all -RUN make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt install \ - || (echo "=== petsc4py build log ===" && \ - cat arch-linux-c-opt/lib/petsc/conf/petsc4py.build.log 2>/dev/null && \ - echo "=== slepc4py build log ===" && \ - cat arch-linux-c-opt/lib/petsc/conf/slepc4py.build.log 2>/dev/null && \ - exit 1) -RUN rm -rf /usr/local/share/petsc - -# record builder stage packages used -RUN python${PYVER} -m pip freeze > /opt/requirements.txt \ -&& dnf history userinstalled > /opt/packages.txt - -# Stage 3: 'final' -FROM runtime as final - -COPY --from=builder /opt /opt -COPY --from=builder /usr/local /usr/local - -# MUST set PETSc environment variables -ENV PETSC_DIR=/usr/local -ENV PYTHONPATH=$PYTHONPATH:$PETSC_DIR/lib - -# switch to not-root user and workspace -USER $NB_USER -WORKDIR $NB_HOME - -# default command is to run jupyter lab -CMD ["jupyter-lab", "--no-browser", "--ip='0.0.0.0'"] diff --git a/docs/developer/gadi_singularity/underworld3.rhel b/docs/developer/gadi_singularity/underworld3.rhel deleted file mode 100644 index 21a712f30..000000000 --- a/docs/developer/gadi_singularity/underworld3.rhel +++ /dev/null @@ -1,209 +0,0 @@ -##################################################################### -# Multi stage Containerfile for Underworld 3 -# UW3 container based on UW2 version -# This UW3 container is based on pixi amr-dev environment -# see https://docs.docker.com/get-started/docker-concepts/building-images/multi-stage-builds/ -# -# Stages: -# 1. 'runtime' -# The runtime environment (packages, permissions, ENV vars.) -# is consistent accross all stages of this Containerfile. -# -# 2. 'builder' -# The builder layer, takes the runtime layer and add compiling / building software -# that is added to /usr/local and /opt/venv -# -# 3. 'final' == runtime + min. builder -# The final image is a composite of the runtime layer and the -# minimal sections of the builder layer's final software stack. -# -# To build use podman from the top level underworld didrectory. i.e. -# $ podman build . \ - --platform linux/amd64 \ -# --format docker \ -# --build-arg UW3_BRANCH=xxx \ -# -t new_image_name \ -# -f ./docs/developer/gadi_singularity/underworld3.rhel -##################################################################### - -# The following are passed in via --build-arg -# Must go before the 1st FROM see -# https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact -ARG PYTHON_VERSION="3.12" -ARG BASE_IMAGE="quay.io/rockylinux/rockylinux:8.10" -ARG PETSC_IMAGE="ghcr.io/jcgraciosa/petsc:3.25.0-ompi" -ARG UW3_BRANCH="development" - -# 'petsc-image' will be used later on in builder stage COPY command -FROM ${PETSC_IMAGE} as petsc-image - -#################### -# Stage 1: 'runtime' -#################### -FROM ${BASE_IMAGE} as runtime -LABEL maintainer="https://github.com/underworldcode/" - -# need to repeat ARGS after every FROM -ARG PYTHON_VERSION - -#### Containerfile ENV vars - for all image stages -ENV LANG=C.UTF-8 -ENV PYVER=${PYTHON_VERSION} - -# add user jovyan -ENV NB_USER jovyan -ENV NB_HOME /home/$NB_USER -RUN useradd -m -s /bin/bash -N $NB_USER - -# runtime packages - [vim, git] are optional -RUN yum update -y \ -&& yum install -y \ - ca-certificates \ - bash-completion \ - openssh \ - openblas \ - openmpi \ - python${PYVER}-pip \ - python${PYVER}-devel \ - zlib \ - mesa-libGL \ - mesa-libGLU \ - mesa-libOSMesa \ - libX11 \ - libXrender \ - libXext \ - libXfixes \ - libXcursor \ - libXinerama \ - libXrandr \ - libXft \ - fontconfig \ -&& yum clean all \ -&& rm -rf /var/cache/yum - -# add system openmpi to $PATH and $LD_LIBRARY_PATH -ENV PATH=/usr/lib64/openmpi/bin:$PATH -ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - -ENV PYOPT=/opt/venv -# build and set open permissions on virtual environment -RUN python${PYVER} -m venv $PYOPT \ -&& chmod ugo+rwx $PYOPT - -# define python env vars. -# prepappending on PATH means all pip install will goto the PYOPT -ENV PATH=$PYOPT/bin:$PATH -ENV PYTHONPATH=$PYTHONPATH:$PYOPT/lib/python${PYVER}/site-packages -# taken from PETSC images - will be needed in the final image -ENV PETSC_DIR=/usr/local -ENV PYTHONPATH=$PYTHONPATH:$PETSC_DIR/lib - -#################### -# Stage 2: 'builder' -#################### -FROM petsc-image as builder -ARG PYTHON_VERSION -ARG UW3_BRANCH - -# root to install with yum -USER root -RUN yum install -y\ - cmake \ - make \ - git \ - gcc \ - gcc-c++ \ - gcc-gfortran \ - findutils \ - openmpi-devel \ -&& yum clean all \ -&& rm -rf /var/cache/yum -# NOTE: underworld2 does not yum install gcc - -# install python build time and runtime requirements here -RUN python${PYVER} -m pip install --no-cache-dir \ - "setuptools>=75" \ - "cython>=3.1" \ - "scipy>=1.15" \ - "numpy<2" \ - matplotlib \ - pint \ - pytest \ - jupytext \ - sympy \ - "pydantic>=2" \ - pyyaml \ - psutil \ - typing_extensions \ - xxhash \ - trimesh \ - ipython \ - jupyterlab \ - "ipywidgets<9.0.0" \ - jupyter-server-proxy \ - "trame>=2.5.2" \ - "trame-vtk>=2.5.8" \ - "trame-vuetify>=2.3.1" \ - cmocean \ - colorcet \ - imageio \ - imageio-ffmpeg \ - "rich<14" \ - meshio \ - pyvista -# gmsh wheels are only available for x86_64; skip gracefully on aarch64 (local Apple Silicon builds) -RUN pip install --no-cache-dir gmsh pygmsh || echo "gmsh/pygmsh not available on this platform, skipping" - -# H5PY with mpi - match pixi version constraint! -RUN CC=mpicc HDF5_MPI="ON" HDF5_DIR=${PETSC_DIR} \ - python${PYVER} -m pip install \ - --no-binary=h5py \ - --no-cache-dir \ - "h5py>=3.12" - -# separate so if this fails, don't need to reinstall everything again with layer caching -# vtk-osmesa wheels only available for x86_64; skip gracefully on aarch64 (local Apple Silicon builds) -RUN pip install --no-cache-dir --extra-index-url https://wheels.vtk.org vtk-osmesa \ - || echo "vtk-osmesa not available on this platform, skipping" - -# Clone and install uw3 -RUN git clone --depth 1 --branch ${UW3_BRANCH} \ - https://github.com/underworldcode/underworld3.git /tmp/underworld3 -WORKDIR /tmp/underworld3 -RUN pip install --no-build-isolation --no-cache-dir . - -# record 'builder' stage packages used -RUN python${PYVER} -m pip freeze >/opt/requirements.txt \ -&& dnf history userinstalled >/opt/installed.txt - -#################### -# Stage 3: 'final', a combination of 'runtime' and 'builder' stages -#################### - -FROM runtime as final - -COPY --from=builder --chown=$NB_USER:users /opt /opt -COPY --from=builder --chown=$NB_USER:users /usr/local /usr/local - -# must make directory before COPY into it for permissions to work () -# set default viewer to a notebook see https://jupytext.readthedocs.io/en/latest/text-notebooks.html#with-a-double-click -RUN mkdir -p $NB_HOME/workspace $NB_HOME/Underworld3/ \ -&& chown $NB_USER:users -R $NB_HOME \ -&& jupyter-server extension enable --sys-prefix jupyter_server_proxy \ -&& jupytext-config set-default-viewer - -# confirm if these are the ones to put into the container -# copy examples, tests, etc. -COPY --chown=$NB_USER:users ./docs/examples $NB_HOME/Underworld3/examples -COPY --chown=$NB_USER:users ./docs/beginner $NB_HOME/Underworld3/beginner -COPY --chown=$NB_USER:users ./docs/advanced $NB_HOME/Underworld3/advanced - -EXPOSE 8888 -WORKDIR $NB_HOME -USER $NB_USER - -# Declare a volume space -VOLUME $NB_HOME/workspace - -CMD ["jupyter-lab", "--no-browser", "--ip='0.0.0.0'"] - diff --git a/docs/developer/hpc_containers/README.md b/docs/developer/hpc_containers/README.md new file mode 100644 index 000000000..3d0dbeb0a --- /dev/null +++ b/docs/developer/hpc_containers/README.md @@ -0,0 +1,97 @@ +# HPC containers for Underworld3 + +Singularity/Apptainer images for the HPC systems we run on, built by GitHub Actions and +published to GHCR. Site-specific run instructions (job templates, MPI settings, shared +image locations) live in +[underworld-community/uw3-hpc-install-run](https://github.com/underworld-community/uw3-hpc-install-run); +this directory is the recipes. + +## Layout + +One platform base per OS/MPI family; everything above it is shared. + +``` +base.rocky-ompi Rocky 8.10 + system OpenMPI -> uw3-base-rocky-ompi:{runtime,builder} Gadi, Kaiju +base.ubuntu-mpich Ubuntu 24.04 + MPICH 3.4.3 -> uw3-base-ubuntu-mpich:{runtime,builder} Setonix +petsc.hpc PETSc from source, on a base -> petsc-{gadi,setonix}:-{ompi,mpich} +underworld3.hpc UW3 + Python stack, on PETSc -> underworld3-{gadi,setonix}: +``` + +`petsc.hpc` and `underworld3.hpc` never call a package manager; they compile against +whatever `mpicc` the `:builder` base provides and lay the result over the `:runtime` base. +Adding a platform means adding a `base.*` file and a matrix row in each workflow. + +| Platform | Base | MPI in image | On the machine | +|---|---|---|---| +| Gadi | Rocky 8.10 | OpenMPI 4.1.1 (`libmpi.so.40`) | host `openmpi/4.1.7` injected via `LD_LIBRARY_PATH` — measured equal to bare metal | +| Kaiju | same image as Gadi | OpenMPI 4.1.1 | runs as-is under `srun --mpi=pmix` | +| Setonix | Ubuntu 24.04 | MPICH 3.4.3 (`libmpi.so.12`) | host Cray MPICH bind-mounted by `singularity/4.1.0-mpi` | + +Setonix needs 3.4.3 specifically: Cray MPICH 8.1 is MPICH-3.4-based, and code compiled +against MPICH 4.x headers references MPI-4 symbols the host library lacks. + +## Automated builds + +| Workflow | Builds | Runs when | +|---|---|---| +| `hpc-base-images.yml` | `base.*` | push touching `base.*`; dispatch | +| `hpc-petsc-image.yml` | `petsc.hpc` per platform | push touching `petsc.hpc` or `petsc-custom/patches/**`; dispatch | +| `hpc-uw3-image.yml` | `underworld3.hpc` per platform | push to `main`/`development` touching `src/**`, `pixi.toml`, `pyproject.toml`, `setup.py` or `underworld3.hpc`; `v*` tags; releases; dispatch | + +PETSc is ~25 min per platform, so it is a separate, rarely-triggered workflow and its +image is reused by every UW3 build. After a base change, rebuild PETSc, then UW3: + +```bash +gh workflow run hpc-petsc-image.yml -f petsc_version=3.25.0 -f make_np=4 +gh workflow run hpc-uw3-image.yml -f uw3_branch=v3.1.0 -f image_tag=v3.1.0 +``` + +Every build is smoke-tested before anything is pruned: `import underworld3`, the optional +gmsh/vtk-osmesa stack, and a 2-rank MPI run that asserts the MPI flavour (Open MPI vs +MPICH) — a silently wrong `libmpi` is the failure that would otherwise only show on a +compute node. + +Retention: the three most recent untagged `underworld3-*` versions are kept (two for +`petsc-*` and the bases). Tagged images are never pruned. + +Images inherit the repository's visibility. Packages created by hand (`podman push`) are +not repo-linked and default to private; make them public or the compute nodes cannot pull. + +The workflows publish single-platform `linux/amd64` with attestations off +(`provenance: false`, `sbom: false`); attestations add an `unknown/unknown` manifest entry +that some Singularity versions refuse. + +## Building by hand + +From the repository root, x86_64 only: + +```bash +podman build . --platform linux/amd64 --format docker --target builder \ + -t ghcr.io//uw3-base-rocky-ompi:builder -f docs/developer/hpc_containers/base.rocky-ompi +podman build . --platform linux/amd64 --format docker --target runtime \ + -t ghcr.io//uw3-base-rocky-ompi:runtime -f docs/developer/hpc_containers/base.rocky-ompi + +podman build . --platform linux/amd64 --format docker \ + --build-arg BASE_BUILDER=ghcr.io//uw3-base-rocky-ompi:builder \ + --build-arg BASE_RUNTIME=ghcr.io//uw3-base-rocky-ompi:runtime \ + -t ghcr.io//petsc-gadi:3.25.0-ompi -f docs/developer/hpc_containers/petsc.hpc + +podman build . --platform linux/amd64 --format docker \ + --build-arg PETSC_IMAGE=ghcr.io//petsc-gadi:3.25.0-ompi \ + --build-arg BASE_BUILDER=ghcr.io//uw3-base-rocky-ompi:builder \ + --build-arg UW3_BRANCH=development \ + -t ghcr.io//underworld3-gadi:development -f docs/developer/hpc_containers/underworld3.hpc +``` + +Substitute `ubuntu-mpich` / `setonix` / `mpich` for the Setonix chain. Pass +`--build-arg PETSC_MAKE_NP=2` inside a memory-constrained podman machine. + +## Pulling + +```bash +export SINGULARITY_CACHEDIR=/scratch///.singularity # not $HOME +module load singularity # singularity/4.1.0-mpi on Setonix +singularity pull docker://ghcr.io/underworldcode/underworld3-gadi:latest +``` + +How to run on each system is in the `uw3-hpc-install-run` repo. diff --git a/docs/developer/hpc_containers/base.rocky-ompi b/docs/developer/hpc_containers/base.rocky-ompi new file mode 100644 index 000000000..199ca1b14 --- /dev/null +++ b/docs/developer/hpc_containers/base.rocky-ompi @@ -0,0 +1,91 @@ +##################################################################### +# Platform base for the Rocky Linux + OpenMPI family (NCI Gadi, ANU Kaiju). +# +# Everything OS- and MPI-specific lives in a base file like this one; the +# recipes stacked on it (petsc.hpc, underworld3.hpc) build against whatever +# mpicc is on PATH and never call a package manager. Two stages, published as +# two tags of one image: +# :runtime what the final images stand on — no toolchain except g++, which +# lets a developer rebuild UW3 in place on the HPC system +# (underworld3.ckdtree is a C++ extension) +# :builder runtime + compilers and build tools +# +# podman build . --platform linux/amd64 --format docker \ +# --target builder -t ghcr.io//uw3-base-rocky-ompi:builder \ +# -f docs/developer/hpc_containers/base.rocky-ompi +##################################################################### + +ARG PYTHON_VERSION="3.12" +ARG OS_IMAGE="quay.io/rockylinux/rockylinux:8.10" + +FROM ${OS_IMAGE} AS runtime +LABEL maintainer="https://github.com/underworldcode/" + +# Repeat the DEFAULT, not just the name: bare `ARG PYTHON_VERSION` relies on +# inheriting the pre-FROM default, which some buildah/podman versions don't. +ARG PYTHON_VERSION="3.12" + +ENV LANG=C.UTF-8 +ENV PYVER=${PYTHON_VERSION} +ENV OPENBLAS_NUM_THREADS=1 +ENV OMPI_MCA_io=ompio + +ENV NB_USER=jovyan +ENV NB_HOME=/home/$NB_USER +RUN useradd -m -s /bin/bash -N $NB_USER + +# mesa/X11: vtk-osmesa and pyvista. gcc-c++: in-place UW3 rebuilds (see header). +RUN yum update -y \ +&& yum install -y \ + ca-certificates \ + bash-completion \ + openssh \ + openblas \ + openmpi \ + findutils \ + python${PYVER}-pip \ + python${PYVER}-devel \ + gcc-c++ \ + zlib \ + mesa-libGL \ + mesa-libGLU \ + mesa-libOSMesa \ + libX11 \ + libXrender \ + libXext \ + libXfixes \ + libXcursor \ + libXinerama \ + libXrandr \ + libXft \ + fontconfig \ +&& yum clean all \ +&& rm -rf /var/cache/yum + +ENV PATH=/usr/lib64/openmpi/bin:$PATH +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +# One venv shared by every stage above; prepending its bin/ means a bare +# `pip install` lands there. +ENV PYOPT=/opt/venv +RUN python${PYVER} -m venv $PYOPT \ +&& chmod ugo+rwx $PYOPT +ENV PATH=$PYOPT/bin:$PATH +ENV PYTHONPATH=$PYTHONPATH:$PYOPT/lib/python${PYVER}/site-packages + +FROM runtime AS builder +# flex/bison: PETSc's --download-ptscotch needs them. +RUN yum install -y \ + wget \ + make \ + cmake \ + patch \ + git \ + gcc \ + gcc-gfortran \ + flex \ + bison \ + zlib-devel \ + openmpi-devel \ +&& yum clean all \ +&& rm -rf /var/cache/yum diff --git a/docs/developer/hpc_containers/base.ubuntu-mpich b/docs/developer/hpc_containers/base.ubuntu-mpich new file mode 100644 index 000000000..0e046a5d3 --- /dev/null +++ b/docs/developer/hpc_containers/base.ubuntu-mpich @@ -0,0 +1,87 @@ +##################################################################### +# Platform base for Pawsey Setonix: Ubuntu 24.04 + MPICH 3.4.3. +# +# Same two-stage contract as base.rocky-ompi (:runtime, :builder); see there. +# +# Why this base and not a Rocky one: Setonix's `singularity/4.1.0-mpi` module +# bind-mounts the host Cray MPICH (libmpi.so.12) over the container's, and +# Cray MPICH is built against SLES 15 glibc 2.31 — a Rocky 8 image (glibc +# 2.28) fails on missing symbols. Ubuntu 24.04 has 2.39. +# +# Why 3.4.3 and not Pawsey's 4.2.2 tag: Cray MPICH 8.1 is MPICH-3.4-based. +# Anything compiled against 4.x headers references MPI-4 symbols +# (MPI_Buffer_iflush) that the bind-mounted host library lacks — measured as +# an ImportError on Setonix, 2026-09-17. Same ABI, smaller API surface. +##################################################################### + +ARG PYTHON_VERSION="3.12" +ARG OS_IMAGE="quay.io/pawsey/mpich-base:3.4.3_ubuntu24.04" + +FROM ${OS_IMAGE} AS runtime +LABEL maintainer="https://github.com/underworldcode/" + +ARG PYTHON_VERSION="3.12" + +ENV LANG=C.UTF-8 +ENV PYVER=${PYTHON_VERSION} +ENV OPENBLAS_NUM_THREADS=1 +ENV DEBIAN_FRONTEND=noninteractive + +ENV NB_USER=jovyan +ENV NB_HOME=/home/$NB_USER +RUN useradd -m -s /bin/bash -N $NB_USER + +# The Pawsey base already carries gcc/g++/gfortran/git/make/wget. Same package +# set as the Rocky runtime, in Debian names. +RUN apt-get update \ +&& apt-get install -y --no-install-recommends \ + ca-certificates \ + bash-completion \ + openssh-client \ + libopenblas0 \ + python3 \ + python3-dev \ + python3-venv \ + python3-pip \ + g++ \ + zlib1g \ + libgl1 \ + libglu1-mesa \ + libosmesa6 \ + libx11-6 \ + libxrender1 \ + libxext6 \ + libxfixes3 \ + libxcursor1 \ + libxinerama1 \ + libxrandr2 \ + libxft2 \ + fontconfig \ +&& rm -rf /var/lib/apt/lists/* + +# PYVER must match the distro python, because the venv path below embeds it. +RUN python3 -c "import sys; assert f'{sys.version_info[0]}.{sys.version_info[1]}' == '${PYVER}', sys.version" + +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +ENV PYOPT=/opt/venv +RUN python3 -m venv $PYOPT \ +&& chmod ugo+rwx $PYOPT +ENV PATH=$PYOPT/bin:$PATH +ENV PYTHONPATH=$PYTHONPATH:$PYOPT/lib/python${PYVER}/site-packages + +FROM runtime AS builder +RUN apt-get update \ +&& apt-get install -y --no-install-recommends \ + build-essential \ + gfortran \ + cmake \ + make \ + patch \ + git \ + wget \ + flex \ + bison \ + pkg-config \ + zlib1g-dev \ +&& rm -rf /var/lib/apt/lists/* diff --git a/docs/developer/hpc_containers/petsc.hpc b/docs/developer/hpc_containers/petsc.hpc new file mode 100644 index 000000000..24373fb0a --- /dev/null +++ b/docs/developer/hpc_containers/petsc.hpc @@ -0,0 +1,116 @@ +##################################################################### +# UW3 PETSc image — platform-neutral. +# +# Builds PETSc from source, with the AMR stack the pixi amr-dev environment +# uses, on top of a platform base (base.rocky-ompi, base.ubuntu-mpich). No +# package manager is called here: compilers come from the :builder base, and +# the result is laid over the :runtime base, so the same file serves every +# platform. +# +# Stages: +# builder BASE_BUILDER + PETSc built into /usr/local, petsc4py/mpi4py in /opt/venv +# final BASE_RUNTIME + /usr/local + /opt from builder +# +# podman build . --platform linux/amd64 --format docker \ +# --build-arg BASE_BUILDER=ghcr.io//uw3-base-rocky-ompi:builder \ +# --build-arg BASE_RUNTIME=ghcr.io//uw3-base-rocky-ompi:runtime \ +# -t ghcr.io//petsc-gadi:3.25.0-ompi \ +# -f docs/developer/hpc_containers/petsc.hpc +##################################################################### + +# Must go before the 1st FROM, see +# https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact +ARG BASE_BUILDER="ghcr.io/underworldcode/uw3-base-rocky-ompi:builder" +ARG BASE_RUNTIME="ghcr.io/underworldcode/uw3-base-rocky-ompi:runtime" +ARG PETSC_VERSION="3.25.0" + +FROM ${BASE_BUILDER} AS builder +# defaults repeated deliberately — see the note in base.rocky-ompi +ARG PETSC_VERSION="3.25.0" +# Parallelism for PETSc's own build. 2 is safe inside a memory-constrained +# podman machine (compiling PETSc.c is memory-heavy and OOMs at higher +# values); CI on a native runner passes 4. +ARG PETSC_MAKE_NP="2" + +# Upper bound is load-bearing: petsc4py 3.25.0 declares only "cython >= 3" but +# PC.pyx:1256 fails to cythonize under 3.3.0 ("Invalid index type 'int'"). +# Bisected against the 3.25.0 source: 3.2.9 is the last version that builds. +# mpi4py from source so it links this base's MPI, not a wheel's bundled one. +RUN pip install --no-cache-dir \ + wheel \ + "numpy<2" \ + "cython>=3.1,<3.3" \ + "setuptools>=75" \ + meson \ + meson-python \ + ninja \ +&& pip install --no-cache-dir --no-binary=mpi4py "mpi4py>=4,<5" + +COPY petsc-custom/patches/scotch-7.0.10-c23-fix.tar.gz /tmp/scotch.tar.gz +COPY petsc-custom/patches/plexfem-internal-boundary-ownership-fix.patch /tmp/ + +RUN mkdir -p /tmp/src +WORKDIR /tmp/src +RUN wget https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-lite-${PETSC_VERSION}.tar.gz --no-check-certificate \ +&& tar -zxf petsc-lite-${PETSC_VERSION}.tar.gz +WORKDIR /tmp/src/petsc-${PETSC_VERSION} + +# The patch may already be upstream in a newer PETSc; skip gracefully then. +RUN if patch -p1 --dry-run < /tmp/plexfem-internal-boundary-ownership-fix.patch 2>/dev/null; then \ + patch -p1 < /tmp/plexfem-internal-boundary-ownership-fix.patch; \ + echo "plexfem patch applied"; \ + else \ + echo "plexfem patch not applicable, skipping"; \ + fi +RUN python3 ./configure \ + --with-debugging=0 \ + --prefix=/usr/local \ + --with-shared-libraries=1 \ + --with-cxx-dialect=C++11 \ + "--COPTFLAGS=-g -O3" "--CXXOPTFLAGS=-g -O3" "--FOPTFLAGS=-g -O3" \ + --useThreads=0 \ + --with-x=0 \ + --with-pragmatic=1 \ + --with-petsc4py=1 \ + --with-slepc4py=1 \ + --download-eigen=1 \ + --download-metis=1 \ + --download-parmetis=1 \ + --download-mumps=1 \ + --download-scalapack=1 \ + --download-hypre=1 \ + --download-superlu=1 \ + --download-superlu_dist=1 \ + --download-mmg=1 \ + "--download-mmg-cmake-arguments=-DMMG_INSTALL_PRIVATE_HEADERS=ON -DUSE_SCOTCH=OFF" \ + --download-parmmg=1 \ + --download-pragmatic=1 \ + "--download-ptscotch=/tmp/scotch.tar.gz" \ + --download-slepc=1 \ + --download-hdf5=1 \ + --download-fblaslapack=1 \ + --download-zlib=1 \ + --download-ctetgen=1 \ + --download-triangle=1 \ + --with-make-np=${PETSC_MAKE_NP} +RUN make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt all +RUN make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt install \ + || (echo "=== petsc4py build log ===" && \ + cat arch-linux-c-opt/lib/petsc/conf/petsc4py.build.log 2>/dev/null && \ + echo "=== slepc4py build log ===" && \ + cat arch-linux-c-opt/lib/petsc/conf/slepc4py.build.log 2>/dev/null && \ + exit 1) +RUN rm -rf /usr/local/share/petsc + +RUN pip freeze > /opt/requirements.txt + +FROM ${BASE_RUNTIME} AS final + +COPY --from=builder /opt /opt +COPY --from=builder /usr/local /usr/local + +ENV PETSC_DIR=/usr/local +ENV PYTHONPATH=$PYTHONPATH:$PETSC_DIR/lib + +USER $NB_USER +WORKDIR $NB_HOME diff --git a/docs/developer/hpc_containers/underworld3.hpc b/docs/developer/hpc_containers/underworld3.hpc new file mode 100644 index 000000000..7328b75b6 --- /dev/null +++ b/docs/developer/hpc_containers/underworld3.hpc @@ -0,0 +1,128 @@ +##################################################################### +# Underworld3 image — platform-neutral. +# +# Builds UW3 and its Python stack on top of a petsc.hpc image, using the +# matching platform :builder base for the compile. No package manager is +# called here; see base.rocky-ompi for the contract. +# +# Stages: +# builder BASE_BUILDER + PETSc (copied from PETSC_IMAGE) + UW3 stack in /opt/venv +# final BASE_RUNTIME + /opt, /usr/local from builder +# +# final stands on the bare runtime base, not on PETSC_IMAGE: the builder's /opt +# and /usr/local already contain PETSc, so layering them over the PETSc image +# would ship PETSc twice (~200 MB more to pull; the SIF is unaffected). +# +# podman build . --platform linux/amd64 --format docker \ +# --build-arg PETSC_IMAGE=ghcr.io//petsc-gadi:3.25.0-ompi \ +# --build-arg BASE_BUILDER=ghcr.io//uw3-base-rocky-ompi:builder \ +# --build-arg BASE_RUNTIME=ghcr.io//uw3-base-rocky-ompi:runtime \ +# --build-arg UW3_BRANCH=development \ +# -t ghcr.io//underworld3-gadi:development \ +# -f docs/developer/hpc_containers/underworld3.hpc +##################################################################### + +ARG PETSC_IMAGE="ghcr.io/underworldcode/petsc-gadi:3.25.0-ompi" +ARG BASE_BUILDER="ghcr.io/underworldcode/uw3-base-rocky-ompi:builder" +ARG BASE_RUNTIME="ghcr.io/underworldcode/uw3-base-rocky-ompi:runtime" +ARG UW3_BRANCH="development" +# Overridden by CI to the repository being built, so a fork builds its own +# branches rather than looking for them upstream. +ARG UW3_REPO="https://github.com/underworldcode/underworld3.git" + +FROM ${PETSC_IMAGE} AS petsc-image + +FROM ${BASE_BUILDER} AS builder +# defaults repeated deliberately — see the note in base.rocky-ompi +ARG UW3_BRANCH="development" +ARG UW3_REPO="https://github.com/underworldcode/underworld3.git" +ARG SETUPTOOLS_SCM_PRETEND_VERSION="" + +# PETSc and its venv (petsc4py, mpi4py, numpy) come from the PETSc image; the +# compilers from the base. Both venvs were created identically by the base, +# so the copy is a superset, not a conflict. +COPY --from=petsc-image /opt /opt +COPY --from=petsc-image /usr/local /usr/local +ENV PETSC_DIR=/usr/local +ENV PYTHONPATH=$PYTHONPATH:$PETSC_DIR/lib + +RUN pip install --no-cache-dir \ + "setuptools>=75" \ + "setuptools-scm>=8" \ + "cython>=3.1" \ + "scipy>=1.15" \ + "numpy<2" \ + matplotlib \ + pint \ + pytest \ + jupytext \ + sympy \ + "pydantic>=2" \ + pyyaml \ + psutil \ + typing_extensions \ + xxhash \ + requests \ + trimesh \ + ipython \ + "jupyterlab>=4.3,<5" \ + "ipywidgets<9.0.0" \ + jupyter-server-proxy \ + "trame>=2.5.2" \ + "trame-vtk>=2.5.8" \ + "trame-vuetify>=2.3.1" \ + cmocean \ + colorcet \ + imageio \ + imageio-ffmpeg \ + "rich<14" \ + meshio \ + pyvista +# gmsh wheels are x86_64-only; skip gracefully on aarch64 (local Apple Silicon builds) +RUN pip install --no-cache-dir gmsh pygmsh || echo "gmsh/pygmsh not available on this platform, skipping" + +# Parallel h5py against PETSc's HDF5 — match the pixi version constraint. +RUN CC=mpicc HDF5_MPI="ON" HDF5_DIR=${PETSC_DIR} \ + pip install --no-binary=h5py --no-cache-dir "h5py>=3.12" + +# Separate layer so a failure here does not redo everything above. +RUN pip install --no-cache-dir --extra-index-url https://wheels.vtk.org vtk-osmesa \ + || echo "vtk-osmesa not available on this platform, skipping" + +RUN git clone --depth 1 --branch ${UW3_BRANCH} ${UW3_REPO} /tmp/underworld3 +WORKDIR /tmp/underworld3 +# The clone above is shallow and forks carry no tags, so setuptools-scm has +# nothing to derive from and would stamp the build 0.0.0. CI passes the real +# version; unset when empty so a local build keeps setuptools-scm's own logic. +RUN [ -n "$SETUPTOOLS_SCM_PRETEND_VERSION" ] || unset SETUPTOOLS_SCM_PRETEND_VERSION; \ + pip install --no-build-isolation --no-cache-dir . + +RUN pip freeze > /opt/requirements.txt + +##################################################################### +FROM ${BASE_RUNTIME} AS final + +COPY --from=builder --chown=$NB_USER:users /opt /opt +COPY --from=builder --chown=$NB_USER:users /usr/local /usr/local + +# Same as petsc.hpc's final stage — not inherited, since this stage does not +# stand on that image. +ENV PETSC_DIR=/usr/local +ENV PYTHONPATH=$PYTHONPATH:$PETSC_DIR/lib + +# jupytext default viewer: text notebooks open as notebooks on double-click. +RUN mkdir -p $NB_HOME/workspace $NB_HOME/Underworld3/ \ +&& chown $NB_USER:users -R $NB_HOME \ +&& jupyter-server extension enable --sys-prefix jupyter_server_proxy \ +&& jupytext-config set-default-viewer + +COPY --chown=$NB_USER:users ./docs/examples $NB_HOME/Underworld3/examples +COPY --chown=$NB_USER:users ./docs/beginner $NB_HOME/Underworld3/beginner +COPY --chown=$NB_USER:users ./docs/advanced $NB_HOME/Underworld3/advanced + +EXPOSE 8888 +WORKDIR $NB_HOME +USER $NB_USER +VOLUME $NB_HOME/workspace + +CMD ["jupyter-lab", "--no-browser", "--ip='0.0.0.0'"]