From 98ec1ff4da959ec4328812b5f1c80cfafe396398 Mon Sep 17 00:00:00 2001 From: rzonyx <77541804+rzonyx@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:46:50 +0700 Subject: [PATCH] Leave terminal-bench@2.0 out of the dataset index terminal-bench@2.0.1 is 2.0's tasks with each environment image pinned by digest to its mirror, but hbench datasets list showed both with nearly the same description and no hint which to use. superseded.yaml now maps 2.0 to 2.0.1, and dataset_index.py leaves out any version it maps, so the list and the interactive run offer 2.0.1 only. The registry keeps serving 2.0 to jobs that name it. The README's examples use 2.0.1. --- CONTRIBUTING.md | 3 ++- README.md | 32 ++++++++++++++++++-------------- scripts/dataset_index.py | 17 ++++++++++++----- scripts/source_descriptor.py | 6 ++++++ superseded.yaml | 1 + tests/test_dataset_index.py | 11 +++++++++-- 6 files changed, 48 insertions(+), 22 deletions(-) create mode 100644 superseded.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f2636e..1208a27 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -116,7 +116,8 @@ When every task needs its own prebuilt environment, as in SWE-bench, the environ pushed, signed, attested and with its Croissant file attached. - A change of bytes is a new version, in a new directory: a task fixed upstream, a new upstream commit, or images pinned by digest. A change that keeps upstream's version and changes only how it - is packaged bumps the last component, as 2.0 to 2.0.1. The old version's directory stays. + is packaged bumps the last component, as 2.0 to 2.0.1. The old version's directory stays, and + [`superseded.yaml`](superseded.yaml) maps it to its successor so the dataset index leaves it out. - A publish run that fails after pushing leaves a tag that is not signed. A maintainer deletes that package version before running the workflow again. diff --git a/README.md b/README.md index b59f1c9..f399775 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ new version is a new tag. Each dataset is also mirrored to a second registry, no | Dataset | Tasks | Licence | Status | | --- | ---: | --- | --- | -| `terminal-bench@2.0` | 89 | Apache-2.0 | Configured, not published yet | +| `terminal-bench@2.0` | 89 | Apache-2.0 | Superseded by `terminal-bench@2.0.1`; not listed | | `terminal-bench@2.0.1` | 89 | Apache-2.0 | Configured | | `humaneval-plus@1.0` | 164 | Apache-2.0 | Generated; waits for its image to be built | | `bigcodebench@1.0` | 1,135 | Apache-2.0 | Generated, with the Hard subset's 147 tasks tagged `hard`; waits for its image to be built | @@ -50,25 +50,29 @@ new version is a new tag. Each dataset is also mirrored to a second registry, no | `swebench-multilingual@1.0` | 283 | Apache-2.0, with its repositories' licences | Generated; environments built locally | | `multi-swe-bench@1.0` | 1594 | Apache-2.0, with its repositories' licences | Generated; environments built locally | +A version that [`superseded.yaml`](superseded.yaml) maps to a successor is left out of the dataset index, so +`hbench datasets list` and the interactive run offer only the successor; the registry keeps serving it to jobs +that name it. + ## Running a dataset This needs an hbench release with the OCI read path (`--registry oci://…`), which is being built. ```sh -hbench run -d terminal-bench@2.0 --registry oci://ghcr.io/node-q/hbench-datasets +hbench run -d terminal-bench@2.0.1 --registry oci://ghcr.io/node-q/hbench-datasets ``` -hbench resolves `terminal-bench:2.0` to its manifest digest and checks every layer against it -before unpacking a task. In `terminal-bench@2.0` the tasks pull their prebuilt environment images -from Docker Hub by tag (`alexgshaw/:20251031`). `terminal-bench@2.0.1` is the same tasks with -each image pinned by digest to its mirror (see [Mirrored environments](#mirrored-environments)). +hbench resolves `terminal-bench:2.0.1` to its manifest digest and checks every layer against it +before unpacking a task. Each task's environment image is pinned by digest to its mirror (see +[Mirrored environments](#mirrored-environments)); in `terminal-bench@2.0`, which 2.0.1 supersedes, the +tasks pull them from Docker Hub by tag (`alexgshaw/:20251031`). ## Verifying a dataset The signature, made by this repository's publish workflow: ```sh -cosign verify ghcr.io/node-q/hbench-datasets/terminal-bench:2.0 \ +cosign verify ghcr.io/node-q/hbench-datasets/terminal-bench:2.0.1 \ --certificate-identity-regexp '^https://github.com/node-q/hbench-datasets/.github/workflows/publish.yml@' \ --certificate-oidc-issuer https://token.actions.githubusercontent.com ``` @@ -76,19 +80,19 @@ cosign verify ghcr.io/node-q/hbench-datasets/terminal-bench:2.0 \ The build provenance: ```sh -gh attestation verify oci://ghcr.io/node-q/hbench-datasets/terminal-bench:2.0 --repo node-q/hbench-datasets +gh attestation verify oci://ghcr.io/node-q/hbench-datasets/terminal-bench:2.0.1 --repo node-q/hbench-datasets ``` The Croissant metadata is a referrer of the dataset's manifest: ```sh -oras discover --artifact-type application/ld+json ghcr.io/node-q/hbench-datasets/terminal-bench:2.0 +oras discover --artifact-type application/ld+json ghcr.io/node-q/hbench-datasets/terminal-bench:2.0.1 ``` ## Mirroring ```sh -oras copy -r ghcr.io/node-q/hbench-datasets/terminal-bench:2.0 //terminal-bench:2.0 +oras copy -r ghcr.io/node-q/hbench-datasets/terminal-bench:2.0.1 //terminal-bench:2.0.1 ``` `-r` copies the referrers too: the signature, the attestation and the Croissant metadata. Digests do @@ -299,11 +303,11 @@ hbench's `export` command, which is not in a release yet, writes an OCI image la directory, and hbench reads it with `--registry oci-layout:`: ```sh -hbench datasets export --oci-layout ./hbench-layout terminal-bench@2.0 --registry oci://ghcr.io/node-q/hbench-datasets -hbench run -d terminal-bench@2.0 --registry oci-layout:./hbench-layout +hbench datasets export --oci-layout ./hbench-layout terminal-bench@2.0.1 --registry oci://ghcr.io/node-q/hbench-datasets +hbench run -d terminal-bench@2.0.1 --registry oci-layout:./hbench-layout ``` -`oras copy -r --to-oci-layout ghcr.io/node-q/hbench-datasets/terminal-bench:2.0 ./hbench-layout:2.0` +`oras copy -r --to-oci-layout ghcr.io/node-q/hbench-datasets/terminal-bench:2.0.1 ./hbench-layout:2.0.1` writes the same layout today. The tasks' environment images are not part of the layout: pull or load them before going offline. @@ -312,7 +316,7 @@ load them before going offline. The tooling in this repository is Apache-2.0 ([`LICENSE`](LICENSE)). Each dataset keeps its own licence, in its directory and published with it. -- **terminal-bench@2.0** is Apache-2.0 and configured. +- **terminal-bench@2.0** is Apache-2.0; 2.0.1 supersedes it. - **terminal-bench@2.0.1** is the same tasks, Apache-2.0. Its environment images are redistributed unmodified from upstream's published images; the software in them carries its own licences. - **humaneval-plus@1.0** is generated by node-q from HumanEval+ (EvalPlus, Apache-2.0), which diff --git a/scripts/dataset_index.py b/scripts/dataset_index.py index 40eca06..338daff 100644 --- a/scripts/dataset_index.py +++ b/scripts/dataset_index.py @@ -4,9 +4,11 @@ Resolves /: for each datasets///source.yaml with `oras resolve`, as an anonymous client does, and leaves out a version the registry does not serve (not published yet) or serves only to -signed-in clients (a package still private), saying which. The index is the JSON layer of -/index:latest, which hbench reads to list the namespace's datasets (assets/schemas/dataset-index.v1.json in -node-q/hbench, schema/dataset-index.v1.json here). Entries are sorted by name, then version. +signed-in clients (a package still private), saying which. It also leaves out each version superseded.yaml +replaces with another: the registry still serves it by name, but hbench no longer lists it. The index is the JSON +layer of /index:latest, which hbench reads to list the namespace's datasets +(assets/schemas/dataset-index.v1.json in node-q/hbench, schema/dataset-index.v1.json here). Entries are sorted by +name, then version. """ import argparse @@ -20,7 +22,7 @@ import jsonschema -from source_descriptor import DESCRIPTOR_FILENAME, REPOSITORY_ROOT, load_descriptor, version_directories +from source_descriptor import DESCRIPTOR_FILENAME, REPOSITORY_ROOT, load_descriptor, load_superseded, version_directories INDEX_SCHEMA_PATH = REPOSITORY_ROOT / "schema" / "dataset-index.v1.json" UNSERVED = re.compile(r"not found|unauthorized|denied|forbidden|manifest unknown|name unknown", re.IGNORECASE) @@ -45,12 +47,15 @@ def oras_resolve(reference): raise DatasetIndexError(f"oras resolve {reference}: {completed.stderr.strip()}") -def published_versions(namespace, resolve=None, directories=None): +def published_versions(namespace, resolve=None, directories=None, superseded=None): resolve = resolve or oras_resolve + superseded = load_superseded() if superseded is None else superseded entries, left_out = [], [] for directory in directories if directories is not None else version_directories(): descriptor = load_descriptor(directory / DESCRIPTOR_FILENAME) name, version = descriptor["name"], descriptor["version"] + if f"{name}@{version}" in superseded: + continue digest = resolve(f"{namespace}/{name}:{version}") if digest is None: left_out.append(f"{name}@{version}") @@ -90,6 +95,8 @@ def main(arguments=None): options.output.write_text(dump_index(index), encoding="utf-8") for dataset in left_out: print(f"left out {dataset}: {options.namespace}/{dataset.replace('@', ':')} is not served to anonymous clients") + for dataset, successor in sorted(load_superseded().items()): + print(f"left out {dataset}: superseded by {successor}") print(f"indexed {len(entries)} dataset versions in {options.output}") return 0 diff --git a/scripts/source_descriptor.py b/scripts/source_descriptor.py index e372a22..56af49e 100644 --- a/scripts/source_descriptor.py +++ b/scripts/source_descriptor.py @@ -6,6 +6,7 @@ DATASETS_DIR = REPOSITORY_ROOT / "datasets" SCHEMA_PATH = REPOSITORY_ROOT / "schema" / "source.v1.json" DESCRIPTOR_FILENAME = "source.yaml" +SUPERSEDED_PATH = REPOSITORY_ROOT / "superseded.yaml" PUBLISHED_REPOSITORY_URL = "https://github.com/node-q/hbench-datasets" @@ -32,3 +33,8 @@ def load_descriptor(path): def version_directories(): return sorted(path for path in DATASETS_DIR.glob("*/*") if path.is_dir()) + + +def load_superseded(path=SUPERSEDED_PATH): + with open(path, encoding="utf-8") as stream: + return yaml.load(stream, Loader=UniqueKeySafeLoader) or {} diff --git a/superseded.yaml b/superseded.yaml new file mode 100644 index 0000000..9ebfe41 --- /dev/null +++ b/superseded.yaml @@ -0,0 +1 @@ +terminal-bench@2.0: terminal-bench@2.0.1 diff --git a/tests/test_dataset_index.py b/tests/test_dataset_index.py index 41d45fb..e411680 100644 --- a/tests/test_dataset_index.py +++ b/tests/test_dataset_index.py @@ -32,7 +32,7 @@ def test_served_versions_are_listed_sorted_and_the_rest_left_out(self): f"{NAMESPACE}/terminal-bench:2.0": DIGEST_A, f"{NAMESPACE}/humaneval-plus:1.0": DIGEST_C, } - entries, left_out = dataset_index.published_versions(NAMESPACE, resolver(served), DIRECTORIES) + entries, left_out = dataset_index.published_versions(NAMESPACE, resolver(served), DIRECTORIES, superseded={}) self.assertEqual( entries, [ @@ -43,6 +43,12 @@ def test_served_versions_are_listed_sorted_and_the_rest_left_out(self): ) self.assertEqual(left_out, ["bigcodebench@1.0"]) + def test_a_superseded_version_is_left_out_though_served(self): + served = {f"{NAMESPACE}/terminal-bench:2.0.1": DIGEST_B, f"{NAMESPACE}/terminal-bench:2.0": DIGEST_A} + entries, left_out = dataset_index.published_versions(NAMESPACE, resolver(served), DIRECTORIES[2:3] + DIRECTORIES[:1]) + self.assertEqual(entries, [{"name": "terminal-bench", "version": "2.0.1", "digest": DIGEST_B}]) + self.assertEqual(left_out, []) + def test_versions_sort_by_number_not_text(self): self.assertLess(dataset_index.version_key("2.0.9"), dataset_index.version_key("2.0.10")) @@ -92,11 +98,12 @@ def test_writes_the_index_and_names_what_it_left_out(self): with ( mock.patch.object(dataset_index, "oras_resolve", resolver(served)), mock.patch.object(dataset_index, "version_directories", lambda: DIRECTORIES), + mock.patch.object(dataset_index, "load_superseded", lambda: {"terminal-bench@2.0.1": "terminal-bench@2.0.2"}), contextlib.redirect_stdout(printed), ): self.assertEqual(dataset_index.main(["--namespace", NAMESPACE + "/", "--output", str(output)]), 0) self.assertEqual(json.loads(output.read_text())["datasets"], [{"name": "terminal-bench", "version": "2.0", "digest": DIGEST_A}]) - self.assertIn("left out terminal-bench@2.0.1", printed.getvalue()) + self.assertIn("left out terminal-bench@2.0.1: superseded by terminal-bench@2.0.2", printed.getvalue()) self.assertIn("indexed 1 dataset versions", printed.getvalue())