diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d0fdbee..45ed92b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,3 +4,10 @@ ## Rationale + + +## Compatibility + + +- Stops a released checkpoint from loading, or changes what it predicts: yes / no +- Stops a documented request from working, or changes its response: yes / no diff --git a/.github/workflows/backward-compatibility.yml b/.github/workflows/backward-compatibility.yml new file mode 100644 index 0000000..26d0d46 --- /dev/null +++ b/.github/workflows/backward-compatibility.yml @@ -0,0 +1,61 @@ +# Its own workflow so a broken released checkpoint fails a check named for it. +# No paths filter: a required check must start on every pull request. +name: Backward compatibility + +on: + pull_request: + push: + branches: [main] + +concurrency: + group: backward-compatibility-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + jointfm-1-checkpoints: + name: Released JointFM 1 checkpoints still load + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install Task + uses: arduino/setup-task@c0bc642852239c2689f73f4ea6459c29405f3c52 # v3.0.0 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install uv with cache + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.10.12" + python-version: "3.13.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + pyproject.toml + + - name: Sync project dependencies + run: uv sync --locked --all-groups + + - name: Load and predict with every released checkpoint layout + run: task test:compatibility + + - name: Explain the failure + if: failure() + run: | + echo "::error title=Backward compatibility broken::This change stops a released JointFM 1 checkpoint from loading or changes what it predicts. See tests/compatibility/README.md." + { + echo "## Backward compatibility broken" + echo + echo "This change stops a JointFM 1 checkpoint that people hold from loading, or changes what it predicts." + echo + echo "- Do not regenerate the frozen checkpoints or edit the expected outputs to make this pass." + echo "- Keep the old layout loading, or treat the change as a deprecation: it needs the maintainers' approval, a higher \`CHECKPOINT_FORMAT_VERSION\`, a clear refusal, and notice." + echo + echo "Details: \`tests/compatibility/README.md\` and \"Compatibility of released exports\" in \`CONTRIBUTING.md\`." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c76b44f..d76b69c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,9 +59,10 @@ jobs: # macOS runners ship without docker, which the skywalking-eyes # hook needs; the Ubuntu job checks the license headers. + # backward-compatibility runs in its own workflow so its failure is named. - name: Run task pre-commit env: - SKIP: ${{ runner.os == 'macOS' && 'insert-license' || '' }} + SKIP: ${{ runner.os == 'macOS' && 'insert-license,backward-compatibility' || 'backward-compatibility' }} run: task pre-commit - name: Build distribution diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 22d887b..4fe59ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,3 +72,10 @@ repos: language: system pass_filenames: false always_run: true + # CI skips this hook and runs it as the Backward compatibility workflow. + - id: backward-compatibility + name: backward compatibility of released JointFM 1 checkpoints + entry: task test:compatibility + language: system + pass_filenames: false + always_run: true diff --git a/AGENTS.md b/AGENTS.md index 013c405..829a557 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,8 @@ Dependencies point one way: this project depends only on public packages and nev ## Engineering -- **Fail fast.** No silent fallbacks, no compatibility paths, no deprecation flags, no `try/except` that swallows or rewrites errors. Exceptions are for IO and races; otherwise re-raise. +- **Fail fast.** No silent fallbacks and no `try/except` that swallows or rewrites errors. Exceptions are for IO and races; otherwise re-raise. +- **Released exports keep working.** Every released JointFM 1 checkpoint keeps loading and predicting as it did, and every documented request keeps working. Code that keeps an older layout loading is required, not a fallback. A change that cannot keep one working is a deprecation and follows "Compatibility of released exports" in [CONTRIBUTING.md](CONTRIBUTING.md); never retire one on your own. - **Ground in facts.** Before planning, claiming, or editing, read the relevant source, configuration, docs, and test output, and act only on beliefs backed by something just read or run. - **Commands.** Run ad-hoc Python and CLI commands through `task run -- `, never bare `python` or hand-written `uv run`, so no invocation re-resolves the `.venv` or `uv.lock` by accident. Explain in one sentence what each command does and why before running it. - **Taskfile targets** print an explicit `OK:` or `FAIL:` line; success is never inferred from silence. diff --git a/CHANGELOG.md b/CHANGELOG.md index c7b2cd6..3c1437c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,10 @@ ## v0.3.0 (2026-11-06) +### BREAKING CHANGE + +- JointFM 1 checkpoints exported for `v0.2.0` and earlier no longer load. They still carry format version 1, so the load fails on their parameter names and shapes. Re-export the checkpoint, or serve it with `v0.2.0`. + ### Feat - add release management tasks and update pyproject.toml for changelog handling (#23) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afb8d88..cd404e1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,6 +37,7 @@ Run `task pre-commit` and fix every reported issue before opening a pull request | `task run -- ` | Run a command inside the project environment | | `task test` | Run the unit tests | | `task coverage` | Run the unit tests and enforce the coverage floor | +| `task test:compatibility` | Check that every released JointFM 1 checkpoint layout still loads and predicts the same; CI runs it as the Backward compatibility check | | `task check` | Read-only static checks: typos, Ruff lint and format check, ty, OpenAPI document check | | `task license-check` | Fail when a Python file lacks the license header (needs docker) | | `task license-fix` | Add the license header to every Python file that lacks it (needs docker) | @@ -93,11 +94,33 @@ layout, is refused before a backend is resolved: marker to a serving backend through the `jointfm_inference.backends` entry-point group and fails at load when no installed distribution registers it. This distribution registers `jointfm-1`. -The format version covers this mapping, not the network inside it: the parameter names and shapes -of `model_state` belong to the generation's network, which loads them with `strict=True`. A state -dict written for another layout of that network is refused at load with an error that names every -missing, unexpected, and reshaped parameter. The JointFM 1 network holds no parameter indexed by a -column's position, so permuting a request's columns permutes every per-column output. +The format version covers this mapping and the network inside it: the parameter names, shapes, and +meaning of `model_state` are part of the format, and the network loads them with `strict=True`. The +JointFM 1 network holds no parameter indexed by a column's position, so permuting a request's +columns permutes every per-column output. + +### Compatibility of released exports + +JointFM 1 is released, and people hold its serving exports. Every change keeps each released export +loading and predicting as it did, and keeps each documented request working. A new setting or +parameter is optional, and an export without it behaves as before. + +A change that cannot keep a released export loading is a deprecation, not a refactor. It needs: + +- the maintainers' approval before it merges; +- a higher `CHECKPOINT_FORMAT_VERSION`, so the exports it drops are told apart by their version; +- a refusal of the retired version, raised before any parameter is loaded, that names the file's + version, the versions this runner reads, and what the holder does next; +- a notice in `CHANGELOG.md` at least 90 days before the release that drops support. + +One layout was dropped before this policy: exports written for `v0.2.0` and earlier do not load +from `v0.3.0` on, and are refused with the strict-load error that names their parameters. + +`tests/compatibility/` holds a tiny synthetic export of each released layout, written by the release +that introduced it and never regenerated, and the tests that load them. They run as the separate +**Backward compatibility** check (`task test:compatibility`), which fails when a supported export +stops loading or predicts differently, and when a layout is retired without a version bump. Its +README says how to freeze a new layout. ## Keeping the repository publishable diff --git a/Taskfile.yml b/Taskfile.yml index 41b2335..b2bf724 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -14,6 +14,8 @@ vars: # Starting an xdist worker imports torch, so beyond a few workers # startup outweighs the parallel test time. TEST_MAX_WORKERS: '{{.TEST_MAX_WORKERS | default "8"}}' + # Run only by test:compatibility, so their failure is reported as its own check. + COMPATIBILITY_TESTS: tests/compatibility # The committed OpenAPI document, shipped as package data. OPENAPI_PATH: src/jointfm_inference/model/service/wire_schema/openapi.json # Shared opening of every deploy:* target: install the Pulumi program's @@ -211,17 +213,28 @@ tasks: test: desc: Run the unit test suite cmd: | - if uv run pytest -n auto --maxprocesses={{.TEST_MAX_WORKERS}} {{.CLI_ARGS}}; then + if uv run pytest -n auto --maxprocesses={{.TEST_MAX_WORKERS}} --ignore={{.COMPATIBILITY_TESTS}} {{.CLI_ARGS}}; then echo "OK: tests passed" else echo "FAIL: tests failed" exit 1 fi + # A separate check, so a broken released checkpoint is reported by name. + test:compatibility: + desc: Check that every released JointFM 1 checkpoint layout still loads and predicts the same + cmd: | + if uv run pytest -v -rxX -p no:cacheprovider {{.COMPATIBILITY_TESTS}}; then + echo "OK: every released JointFM 1 checkpoint layout still loads and predicts the same" + else + echo "FAIL: backward compatibility broken; see {{.COMPATIBILITY_TESTS}}/README.md" + exit 1 + fi + coverage: desc: Run the unit test suite with coverage enforcement cmd: | - if uv run pytest -q -n auto --maxprocesses={{.TEST_MAX_WORKERS}} -p no:cacheprovider --cov=jointfm_inference --cov-report=term-missing:skip-covered --cov-fail-under={{.COVERAGE_FAIL_UNDER}}; then + if uv run pytest -q -n auto --maxprocesses={{.TEST_MAX_WORKERS}} -p no:cacheprovider --ignore={{.COMPATIBILITY_TESTS}} --cov=jointfm_inference --cov-report=term-missing:skip-covered --cov-fail-under={{.COVERAGE_FAIL_UNDER}}; then echo "OK: tests passed with coverage at or above {{.COVERAGE_FAIL_UNDER}}%" else echo "FAIL: tests failed or coverage is below {{.COVERAGE_FAIL_UNDER}}%" diff --git a/tests/compatibility/README.md b/tests/compatibility/README.md new file mode 100644 index 0000000..e5b68ae --- /dev/null +++ b/tests/compatibility/README.md @@ -0,0 +1,56 @@ +# Backward compatibility tests + +People hold JointFM 1 checkpoints exported for earlier releases of this runner. These tests prove +that every one of those checkpoint layouts still loads and still predicts what it did. They run as +their own check, **Backward compatibility**, so a failure is named for what it is. + +Run them with `task test:compatibility`. `task test` and `task coverage` leave them out, and the +`backward-compatibility` pre-commit hook runs them locally. + +## The frozen checkpoints + +`jointfm_1_checkpoints/` holds tiny JointFM 1 checkpoints with random weights. Each was written by +the release that introduced its layout, and none is ever regenerated: a regenerated file is written +by the current code, so it proves nothing about older files. + +| File | Holds | +|---|---| +| `.pt` | A complete checkpoint in that layout: `cfg`, `model_state`, `runtime_metadata`, and `inference_metadata` | +| `.outputs.pt` | For a supported layout: a fixed input (`past`, `horizons`) and the network outputs it produced when frozen | + +`test_jointfm_1_checkpoints.py` lists each layout as `SUPPORTED` (must load and match its frozen +outputs) or `RETIRED` (must be refused for its format version). + +| Layout | Written by | Status | +|---|---|---| +| `column-equivariant` | `v0.3.0` and later | Supported | +| `column-slot` | `v0.2.0` and earlier | Retired before this policy, so it still carries format version 1; its refusal test is an expected failure | + +## When a test fails + +The change breaks a checkpoint someone may hold. Do not edit the frozen files, the expected outputs, +or the `SUPPORTED` list to make the test pass. Either keep the old layout loading, or treat the +change as a deprecation and follow "Compatibility of released exports" in `CONTRIBUTING.md`, which +needs the maintainers' approval, a higher `CHECKPOINT_FORMAT_VERSION`, a clear refusal, and notice. + +## Freezing a new layout + +A change that adds a layout, while keeping the old ones loading, freezes the new one in the same +pull request. Write it with the code of that change: + +```sh +task run -- python -c " +import torch +from tests.support.models import checkpoint_mapping, tiny_model +model = tiny_model(seed=0) +torch.save(checkpoint_mapping(model), 'tests/compatibility/jointfm_1_checkpoints/.pt') +generator = torch.Generator().manual_seed(0) +past, horizons = torch.randn(2, 8, 3, generator=generator), torch.arange(4) +with torch.no_grad(): + outputs = model(past, horizons) +torch.save({'past': past, 'horizons': horizons, 'outputs': dict(outputs)}, + 'tests/compatibility/jointfm_1_checkpoints/.outputs.pt') +" +``` + +Then add the layout to `SUPPORTED` with the release that first writes it. diff --git a/tests/compatibility/jointfm_1_checkpoints/column-equivariant.outputs.pt b/tests/compatibility/jointfm_1_checkpoints/column-equivariant.outputs.pt new file mode 100644 index 0000000..6fbd42b Binary files /dev/null and b/tests/compatibility/jointfm_1_checkpoints/column-equivariant.outputs.pt differ diff --git a/tests/compatibility/jointfm_1_checkpoints/column-equivariant.pt b/tests/compatibility/jointfm_1_checkpoints/column-equivariant.pt new file mode 100644 index 0000000..8d7d7d5 Binary files /dev/null and b/tests/compatibility/jointfm_1_checkpoints/column-equivariant.pt differ diff --git a/tests/compatibility/jointfm_1_checkpoints/column-slot.pt b/tests/compatibility/jointfm_1_checkpoints/column-slot.pt new file mode 100644 index 0000000..1bd8a82 Binary files /dev/null and b/tests/compatibility/jointfm_1_checkpoints/column-slot.pt differ diff --git a/tests/compatibility/test_jointfm_1_checkpoints.py b/tests/compatibility/test_jointfm_1_checkpoints.py new file mode 100644 index 0000000..dbc218e --- /dev/null +++ b/tests/compatibility/test_jointfm_1_checkpoints.py @@ -0,0 +1,140 @@ +# Copyright 2026 DataRobot, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility: every released JointFM 1 checkpoint layout keeps loading. + +People hold JointFM 1 checkpoints exported for earlier releases of this +runner. A failure here means this change stops one of them from loading or +changes what it predicts. Fixing the test is never the remedy: restore the old +behavior, or follow "Compatibility of released exports" in CONTRIBUTING.md, +which needs the maintainers' approval. See README.md beside this file. +""" + +from pathlib import Path +from typing import Any + +import pytest +import torch + +from jointfm_inference.model.checkpoint import ( + CHECKPOINT_FORMAT_VERSION, + CHECKPOINT_FORMAT_VERSION_KEY, + CHECKPOINT_RUNTIME_METADATA_KEY, + read_checkpoint_file, +) +from jointfm_inference.model.jointfm_1.loader import load_model_from_checkpoint + +CHECKPOINTS = Path(__file__).parent / "jointfm_1_checkpoints" +OUTPUTS_SUFFIX = ".outputs" + +# Layout name to the first release that wrote it. Each loads and predicts its frozen outputs. +SUPPORTED = {"column-equivariant": "v0.3.0"} +# Layout name to the last release that read it. Each is refused for its format version. +RETIRED = {"column-slot": "v0.2.0"} + +BROKEN = ( + "BACKWARD COMPATIBILITY BROKEN: a released JointFM 1 checkpoint ({layout}, " + "first written by {release}) {what}. Restore the old behavior, or follow " + "'Compatibility of released exports' in CONTRIBUTING.md. Never regenerate " + "the frozen files to make this pass." +) + + +def _checkpoint(layout: str) -> dict[str, Any]: + return read_checkpoint_file(str(CHECKPOINTS / f"{layout}.pt")) + + +def _format_version(layout: str) -> int: + return _checkpoint(layout)[CHECKPOINT_RUNTIME_METADATA_KEY][ + CHECKPOINT_FORMAT_VERSION_KEY + ] + + +def test_every_frozen_checkpoint_is_declared_supported_or_retired() -> None: + frozen = { + path.stem + for path in CHECKPOINTS.glob("*.pt") + if not path.stem.endswith(OUTPUTS_SUFFIX) + } + assert frozen == SUPPORTED.keys() | RETIRED.keys(), ( + "Every file in jointfm_1_checkpoints/ must be listed in SUPPORTED or " + "RETIRED, and every listed layout must have its file. Deleting a file " + "does not retire a layout." + ) + assert not SUPPORTED.keys() & RETIRED.keys() + + +def test_the_current_format_version_has_a_frozen_checkpoint() -> None: + versions = {_format_version(layout) for layout in SUPPORTED} + assert CHECKPOINT_FORMAT_VERSION in versions, ( + f"CHECKPOINT_FORMAT_VERSION is {CHECKPOINT_FORMAT_VERSION}, but no " + "supported frozen checkpoint carries it. Freeze one with the release " + "that introduces the version (see README.md beside this file)." + ) + + +@pytest.mark.parametrize("layout", sorted(SUPPORTED)) +def test_a_released_checkpoint_still_loads(layout: str) -> None: + try: + load_model_from_checkpoint(_checkpoint(layout)) + except Exception as error: + pytest.fail( + BROKEN.format( + layout=layout, + release=SUPPORTED[layout], + what=f"no longer loads:\n{error}", + ) + ) + + +@pytest.mark.parametrize("layout", sorted(SUPPORTED)) +def test_a_released_checkpoint_still_predicts_what_it_did(layout: str) -> None: + model, _ = load_model_from_checkpoint(_checkpoint(layout)) + frozen = torch.load(CHECKPOINTS / f"{layout}{OUTPUTS_SUFFIX}.pt", weights_only=True) + + with torch.no_grad(): + outputs = model(frozen["past"], frozen["horizons"]) + + broken = BROKEN.format( + layout=layout, release=SUPPORTED[layout], what="predicts differently" + ) + assert outputs.keys() == frozen["outputs"].keys(), broken + for key, expected in frozen["outputs"].items(): + torch.testing.assert_close( + outputs[key], expected, msg=lambda detail: f"{broken}\n{key}: {detail}" + ) + + +@pytest.mark.parametrize("layout", sorted(RETIRED)) +@pytest.mark.xfail( + strict=True, + reason="the column-slot layout was dropped before format versions covered " + "the network, so it shares version 1 with the current layout", +) +def test_a_retired_checkpoint_is_refused_for_its_format_version(layout: str) -> None: + version = _format_version(layout) + assert version < CHECKPOINT_FORMAT_VERSION, ( + f"The retired {layout} layout carries format version {version}, which " + "this runner still reads. Retiring a layout requires raising " + "CHECKPOINT_FORMAT_VERSION." + ) + + with pytest.raises( + ValueError, + match=( + f"unsupported checkpoint format version {version}; this runner reads " + f"version {CHECKPOINT_FORMAT_VERSION}" + ), + ): + load_model_from_checkpoint(_checkpoint(layout))