From b58a5b81acbaf00499ff3f291821cde7fdfe828e Mon Sep 17 00:00:00 2001 From: mjnitz02 Date: Wed, 18 Nov 2026 11:59:48 -0500 Subject: [PATCH] Split apart and optimize CI/CD pipelines --- .github/actions/setup/action.yml | 51 +++++ .github/workflows/backward-compatibility.yml | 20 +- .github/workflows/ci.yml | 212 ++++++++++++------- .pre-commit-config.yaml | 2 +- CONTRIBUTING.md | 7 +- 5 files changed, 192 insertions(+), 100 deletions(-) create mode 100644 .github/actions/setup/action.yml diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 0000000..88d3e34 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,51 @@ +name: Set up Task and uv +description: Install Task and uv, then sync the dependencies a job needs. + +inputs: + dependencies: + description: >- + "project" syncs the package and every group, torch included. + "dev" syncs only the dev group, for tools that never import the package. + "none" installs uv without syncing. + required: true + +runs: + using: composite + steps: + - name: Install Task + uses: arduino/setup-task@c0bc642852239c2689f73f4ea6459c29405f3c52 # v3.0.0 + with: + version: 3.x + repo-token: ${{ github.token }} + + # On Linux the locked torch is the CUDA build, so the project cache is + # gigabytes; only jobs that import the package restore or save it. + - name: Install uv + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.10.12" + python-version: "3.13.3" + enable-cache: ${{ inputs.dependencies == 'project' }} + cache-dependency-glob: | + uv.lock + pyproject.toml + + - name: Sync the project and every group + if: inputs.dependencies == 'project' + shell: bash + run: uv sync --locked --all-groups + + # UV_NO_SYNC stops later `uv run` calls from installing the project. + - name: Sync the dev group only + if: inputs.dependencies == 'dev' + shell: bash + run: | + uv sync --locked --only-group dev + echo "UV_NO_SYNC=1" >> "$GITHUB_ENV" + + - name: Refuse an unknown dependencies value + if: inputs.dependencies != 'project' && inputs.dependencies != 'dev' && inputs.dependencies != 'none' + shell: bash + run: | + echo "::error::dependencies must be project, dev, or none, got '${{ inputs.dependencies }}'" + exit 1 diff --git a/.github/workflows/backward-compatibility.yml b/.github/workflows/backward-compatibility.yml index 26d0d46..6c9672e 100644 --- a/.github/workflows/backward-compatibility.yml +++ b/.github/workflows/backward-compatibility.yml @@ -23,24 +23,10 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Install Task - uses: arduino/setup-task@c0bc642852239c2689f73f4ea6459c29405f3c52 # v3.0.0 + - name: Set up Task and uv + uses: ./.github/actions/setup with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Install uv with cache - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 - with: - version: "0.10.12" - python-version: "3.13.3" - enable-cache: true - cache-dependency-glob: | - uv.lock - pyproject.toml - - - name: Sync project dependencies - run: uv sync --locked --all-groups + dependencies: project - name: Load and predict with every released checkpoint layout run: task test:compatibility diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d76b69c..83cb35c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,3 +1,6 @@ +# One job per check, so a red check names what failed. Only the unit tests run +# on both platforms: every other check reads files or types and gives the same +# answer on either. name: CI on: @@ -13,115 +16,164 @@ permissions: contents: read jobs: - pre-commit: - name: task pre-commit (${{ matrix.os }}) - runs-on: ${{ matrix.os }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] + lint: + name: Lint (ruff) + runs-on: ubuntu-latest + timeout-minutes: 10 steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Install Task - uses: arduino/setup-task@c0bc642852239c2689f73f4ea6459c29405f3c52 # v3.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/setup with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} + dependencies: dev + - run: task lint - - name: Install uv with cache - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + format: + name: Format (ruff) + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/setup with: - version: "0.10.12" - python-version: "3.13.3" - enable-cache: true - cache-dependency-glob: | - uv.lock - pyproject.toml + dependencies: dev + - run: task format:check - - name: Cache pre-commit environments - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + types: + name: Types (ty) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/setup with: - path: ~/.cache/pre-commit - key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} - restore-keys: | - pre-commit-${{ runner.os }}- + dependencies: project + - run: task typecheck - - name: Install typos CLI - uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3 + spelling: + name: Spelling (typos) + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/setup + with: + dependencies: none + - uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3 with: tool: typos-cli@1.45.1 + - run: task typos - - name: Sync project dependencies - run: uv sync --locked --all-groups + license-headers: + name: License headers + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/setup + with: + dependencies: none + - run: task license-check - # macOS runners ship without docker, which the skywalking-eyes - # hook needs; the Ubuntu job checks the license headers. - # backward-compatibility runs in its own workflow so its failure is named. - - name: Run task pre-commit - env: - SKIP: ${{ runner.os == 'macOS' && 'insert-license,backward-compatibility' || 'backward-compatibility' }} - run: task pre-commit + openapi: + name: OpenAPI contract is current + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/setup + with: + dependencies: project + - run: task schema:check - - name: Build distribution - run: task build + tests: + name: Unit tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/setup + with: + dependencies: project + - run: task coverage - leak-guard-history: - name: task leak-guard:history + build: + name: Build package runs-on: ubuntu-latest timeout-minutes: 10 steps: - - name: Checkout with full history - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/setup with: - fetch-depth: 0 + dependencies: none + - run: task build - - name: Install Task - uses: arduino/setup-task@c0bc642852239c2689f73f4ea6459c29405f3c52 # v3.0.0 + leak-guard: + name: Leak guard + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Install uv with cache - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + fetch-depth: 0 + - uses: ./.github/actions/setup with: - version: "0.10.12" - python-version: "3.13.3" - enable-cache: true - cache-dependency-glob: | - uv.lock - pyproject.toml - + dependencies: dev + - name: Scan the working tree + run: task leak-guard - name: Scan every reachable commit run: task leak-guard:history - secret-scan-history: - name: task secret-scan:history + secret-scan: + name: Secret scan (gitleaks) runs-on: ubuntu-latest timeout-minutes: 10 steps: - - name: Checkout with full history - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - - - name: Install Task - uses: arduino/setup-task@c0bc642852239c2689f73f4ea6459c29405f3c52 # v3.0.0 + - uses: ./.github/actions/setup with: - version: 3.x - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Install uv with cache - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + dependencies: dev + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - version: "0.10.12" - python-version: "3.13.3" - enable-cache: true - cache-dependency-glob: | - uv.lock - pyproject.toml - + path: ~/.cache/pre-commit + key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + pre-commit-${{ runner.os }}- - name: Scan every reachable commit for secrets run: task secret-scan:history + + # The one check the ruleset requires from this workflow, so adding or + # renaming a job above never leaves a required check pending forever. + ci-passed: + name: CI passed + if: always() + needs: + - lint + - format + - types + - spelling + - license-headers + - openapi + - tests + - build + - leak-guard + - secret-scan + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Require every check to pass + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + for result in $RESULTS; do + if [ "$result" != "success" ]; then + echo "::error::A check failed or was skipped; see the named checks on this run." + exit 1 + fi + done + echo "OK: every check passed" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4fe59ab..8d16c89 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,7 +72,7 @@ repos: language: system pass_filenames: false always_run: true - # CI skips this hook and runs it as the Backward compatibility workflow. + # CI runs this as its own Backward compatibility workflow. - id: backward-compatibility name: backward compatibility of released JointFM 1 checkpoints entry: task test:compatibility diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd404e1..0114aab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,8 +61,11 @@ Run `task pre-commit` and fix every reported issue before opening a pull request | `task release` | Cut a release: bump the version, write `CHANGELOG.md`, commit, and tag | | `task release:publish` | Push the release commit and its tag to `origin` | -Every task prints an `OK:` or `FAIL:` line. CI runs `task pre-commit` and `task build` on Ubuntu -and macOS; the macOS job skips the license header hook because its runners have no docker. +Every task prints an `OK:` or `FAIL:` line. CI runs the same checks as `task pre-commit`, each as +its own job named for what it checks, plus `task build`. Only the unit tests run on both Ubuntu and +macOS; every other check gives the same answer on either platform. The ruleset requires the +`CI passed` job, which fails when any CI job fails, so adding or renaming a job needs no ruleset +change. ## Package layout