From 40e2c92f2cf9e157d982a2c4a3f995c97635eed5 Mon Sep 17 00:00:00 2001 From: rzonyx <77541804+rzonyx@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:59:21 +0700 Subject: [PATCH 1/2] Release node-q's SWE-bench Verified as swebench-verified@2.0 swebench-verified@1.0 already names a third-party set of 500 task directories in other registries, and nodeq.tech's task catalog pins those. node-q's generated set has different bytes (456 tasks, environments built locally), so publishing it under the same name and version would make one reference mean two datasets. It has not been published yet, so it moves to 2.0 now. Only the version changes: the task files do not carry it, and regenerating 2.0 gives the same digests as generated.lock.json. --- README.md | 8 +++++--- datasets/swebench-verified/{1.0 => 2.0}/LICENSE | 0 datasets/swebench-verified/{1.0 => 2.0}/NOTICE | 2 +- .../swebench-verified/{1.0 => 2.0}/generated.lock.json | 0 datasets/swebench-verified/{1.0 => 2.0}/images.lock.json | 0 datasets/swebench-verified/{1.0 => 2.0}/source.yaml | 4 ++-- docs/licensing.md | 9 +++++---- 7 files changed, 13 insertions(+), 10 deletions(-) rename datasets/swebench-verified/{1.0 => 2.0}/LICENSE (100%) rename datasets/swebench-verified/{1.0 => 2.0}/NOTICE (99%) rename datasets/swebench-verified/{1.0 => 2.0}/generated.lock.json (100%) rename datasets/swebench-verified/{1.0 => 2.0}/images.lock.json (100%) rename datasets/swebench-verified/{1.0 => 2.0}/source.yaml (99%) diff --git a/README.md b/README.md index b59f1c9..cb32a32 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ new version is a new tag. Each dataset is also mirrored to a second registry, no | `mbpp-plus@1.0` | 378 | Apache-2.0 | Generated; waits for its image lock | | `cruxeval-input@1.0` | 800 | Apache-2.0 | Generated; waits for its image lock | | `exercism-polyglot@1.0` | 210 | Apache-2.0 | Generated, 35 tasks in each of six languages; waits for its six images to be built | -| `swebench-verified@1.0` | 456 | Apache-2.0, with its repositories' licences | Generated; environments built locally | +| `swebench-verified@2.0` | 456 | Apache-2.0, with its repositories' licences | Generated; environments built locally | | `swebench-lite@1.0` | 270 | Apache-2.0, with its repositories' licences | Generated; environments built locally | | `swebench-multilingual@1.0` | 283 | Apache-2.0, with its repositories' licences | Generated; environments built locally | | `multi-swe-bench@1.0` | 1594 | Apache-2.0, with its repositories' licences | Generated; environments built locally | @@ -248,9 +248,11 @@ runners, and reports every task whose oracle or `nop` result is wrong. ### SWE-bench -`swebench-verified@1.0`, `swebench-lite@1.0` and `swebench-multilingual@1.0` are generated by +`swebench-verified@2.0`, `swebench-lite@1.0` and `swebench-multilingual@1.0` are generated by `adapters/swebench` from SWE-bench's own dataset files, and keep only the instances from repositories under a permissive licence (see [`docs/licensing.md`](docs/licensing.md#swe-bench-verified-lite-and-multilingual)). +Verified starts at 2.0 because `swebench-verified@1.0` already names a third-party set of 500 task +directories in other registries, with different bytes. Each task is named by its instance id, which `[metadata].instance_id` repeats, beside the repository, its licence, the base commit and the pull request. @@ -328,7 +330,7 @@ licence, in its directory and published with it. - **exercism-polyglot@1.0** is generated by node-q from Exercism's MIT-licensed problem specifications and C++, Go, Java, JavaScript, Python and Rust tracks. Its task directories are Apache-2.0, with Exercism's notices and Catch2's licence in its NOTICE. -- **swebench-verified@1.0, swebench-lite@1.0 and swebench-multilingual@1.0** are generated by node-q +- **swebench-verified@2.0, swebench-lite@1.0 and swebench-multilingual@1.0** are generated by node-q from SWE-bench's data, and **multi-swe-bench@1.0** from Multi-SWE-bench's (CC0, its harness Apache-2.0), keeping only instances from permissively licensed repositories. node-q's files are Apache-2.0; the issue texts and patches keep their repositories' licences, which each NOTICE diff --git a/datasets/swebench-verified/1.0/LICENSE b/datasets/swebench-verified/2.0/LICENSE similarity index 100% rename from datasets/swebench-verified/1.0/LICENSE rename to datasets/swebench-verified/2.0/LICENSE diff --git a/datasets/swebench-verified/1.0/NOTICE b/datasets/swebench-verified/2.0/NOTICE similarity index 99% rename from datasets/swebench-verified/1.0/NOTICE rename to datasets/swebench-verified/2.0/NOTICE index 8e83f36..41003ad 100644 --- a/datasets/swebench-verified/1.0/NOTICE +++ b/datasets/swebench-verified/2.0/NOTICE @@ -1,4 +1,4 @@ -SWE-bench Verified 1.0 +SWE-bench Verified 2.0 ====================== This dataset is 456 task directories that node-q generates from SWE-bench Verified, one per instance diff --git a/datasets/swebench-verified/1.0/generated.lock.json b/datasets/swebench-verified/2.0/generated.lock.json similarity index 100% rename from datasets/swebench-verified/1.0/generated.lock.json rename to datasets/swebench-verified/2.0/generated.lock.json diff --git a/datasets/swebench-verified/1.0/images.lock.json b/datasets/swebench-verified/2.0/images.lock.json similarity index 100% rename from datasets/swebench-verified/1.0/images.lock.json rename to datasets/swebench-verified/2.0/images.lock.json diff --git a/datasets/swebench-verified/1.0/source.yaml b/datasets/swebench-verified/2.0/source.yaml similarity index 99% rename from datasets/swebench-verified/1.0/source.yaml rename to datasets/swebench-verified/2.0/source.yaml index 57bceab..cdc5bfe 100644 --- a/datasets/swebench-verified/1.0/source.yaml +++ b/datasets/swebench-verified/2.0/source.yaml @@ -1,7 +1,7 @@ schema_version: 1 name: swebench-verified -version: "1.0" -title: SWE-bench Verified 1.0 +version: "2.0" +title: SWE-bench Verified 2.0 description: >- 456 real GitHub issues from Python repositories under permissive licences: the instances of SWE-bench Verified, the human-validated 500-instance subset of SWE-bench, whose repository's licence diff --git a/docs/licensing.md b/docs/licensing.md index 2e6a889..880b2f4 100644 --- a/docs/licensing.md +++ b/docs/licensing.md @@ -15,7 +15,7 @@ upstream revision instead. | `mbpp-plus@1.0` | 378 | [MBPP+], Apache-2.0, from [MBPP], CC-BY-4.0 | Generated by node-q from the upstream data, Apache-2.0 | **Configured** in [`datasets/mbpp-plus/1.0`](../datasets/mbpp-plus/1.0); clear to publish once its image lock is filled | | `cruxeval-input@1.0` | 800 | [CRUXEval], MIT | Generated by node-q from the upstream data, Apache-2.0 | **Configured** in [`datasets/cruxeval-input/1.0`](../datasets/cruxeval-input/1.0); clear to publish once its image lock is filled | | `exercism-polyglot@1.0` | 210 | [Exercism] exercises, MIT; Catch2 in the C++ exercises, BSL-1.0 | Generated by node-q from Exercism's problem specifications and six tracks, Apache-2.0 | **Configured** in [`datasets/exercism-polyglot/1.0`](../datasets/exercism-polyglot/1.0); clear to publish once its six images are built | -| `swebench-verified@1.0` | 456 | [SWE-bench Verified]: no licence on its card; [SWE-bench] is MIT; issues and patches under their repositories' licences | Generated by node-q from the upstream data, Apache-2.0; only permissively licensed repositories | **Configured** in [`datasets/swebench-verified/1.0`](../datasets/swebench-verified/1.0); environments built locally, never redistributed | +| `swebench-verified@2.0` | 456 | [SWE-bench Verified]: no licence on its card; [SWE-bench] is MIT; issues and patches under their repositories' licences | Generated by node-q from the upstream data, Apache-2.0; only permissively licensed repositories | **Configured** in [`datasets/swebench-verified/2.0`](../datasets/swebench-verified/2.0); environments built locally, never redistributed | | `swebench-lite@1.0` | 270 | [SWE-bench Lite]: no licence on its card; [SWE-bench] is MIT; issues and patches under their repositories' licences | Generated by node-q from the upstream data, Apache-2.0; only permissively licensed repositories | **Configured** in [`datasets/swebench-lite/1.0`](../datasets/swebench-lite/1.0); environments built locally, never redistributed | | `swebench-multilingual@1.0` | 283 | [SWE-bench Multilingual], MIT on its card; issues and patches under their repositories' licences | Generated by node-q from the upstream data, Apache-2.0; only permissively licensed repositories | **Configured** in [`datasets/swebench-multilingual/1.0`](../datasets/swebench-multilingual/1.0); environments built locally, never redistributed | | `multi-swe-bench@1.0` | 1594 | [Multi-SWE-bench], CC0 subject to Bytedance's rights, harness Apache-2.0; issues and patches under their repositories' licences | Generated by node-q from the upstream data, Apache-2.0; only permissively licensed repositories | **Configured** in [`datasets/multi-swe-bench/1.0`](../datasets/multi-swe-bench/1.0); environments built locally, never redistributed | @@ -150,7 +150,7 @@ licence. The images are not part of the dataset's OCI artifact. ## SWE-bench Verified, Lite and Multilingual -The task directories of `swebench-verified@1.0`, `swebench-lite@1.0` and `swebench-multilingual@1.0` +The task directories of `swebench-verified@2.0`, `swebench-lite@1.0` and `swebench-multilingual@1.0` are node-q's: `adapters/swebench` writes them from SWE-bench's own dataset files on Hugging Face, each pinned by revision and sha256. node-q's files in them (instructions, `task.toml`, environment Dockerfiles, the history-removal script, the evaluation wrapper, the oracle script and the grading @@ -351,8 +351,9 @@ Until then, none of them has a descriptor. ### SWE-bench Verified -The third-party task directories are not republished. `swebench-verified@1.0` in this repository is -node-q's own, generated from SWE-bench's data; see +The third-party task directories are not republished. node-q's own task directories, generated +from SWE-bench's data, are `swebench-verified@2.0`: a new version, because `swebench-verified@1.0` +already names the third-party ones elsewhere. See [SWE-bench Verified, Lite and Multilingual](#swe-bench-verified-lite-and-multilingual). ### Aider Polyglot From be198eb3be3b0baddb372113953d5c05bd4aa9b8 Mon Sep 17 00:00:00 2001 From: rzonyx <77541804+rzonyx@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:23:53 +0700 Subject: [PATCH 2/2] SWE-bench history check: count PR mentions only since the PR was opened The check counted every commit message with "#" as naming the pull request. Django's messages cite Trac tickets the same way, and ticket 15315 (2011) is not pull request 15315 (2022), so django__django-15315 failed with a base history that is correctly pruned. A commit from before the pull request was opened cannot refer to it, so the probe now passes the task's created_at to git log --since. Multi-SWE-bench tasks carry no created_at and keep counting every message. --- adapters/swebench/verification.py | 14 ++++--- tests/test_swebench_verification.py | 59 +++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 tests/test_swebench_verification.py diff --git a/adapters/swebench/verification.py b/adapters/swebench/verification.py index a5fac66..6f18ef1 100644 --- a/adapters/swebench/verification.py +++ b/adapters/swebench/verification.py @@ -35,7 +35,7 @@ "$(git rev-list --count --all)" \\ "$(git cat-file --batch-all-objects --batch-check='%(objecttype)' | grep -c '^commit$')" \\ "$(git tag --no-merged HEAD | wc -l)" \\ - "$(git log --all --format=%B | grep -cE "#$2([^0-9]|$)")" + "$(git log --all ${3:+--since="$3"} --format=%B | grep -cE "#$2([^0-9]|$)")" """ HISTORY_FIELDS = ("in_history", "reachable", "stored", "later_tags", "naming_pull_request") @@ -115,9 +115,12 @@ def built_image(task, workdir): return environments[0] if len(environments) == 1 else None -def history(image, workdir, pull_request_number): +def history(image, workdir, pull_request_number, opened_at=""): probed = subprocess.run( - ["docker", "run", "--rm", "--entrypoint", "/bin/bash", image, "-c", HISTORY_PROBE, "probe", workdir, pull_request_number], + [ + "docker", "run", "--rm", "--entrypoint", "/bin/bash", image, + "-c", HISTORY_PROBE, "probe", workdir, pull_request_number, opened_at, + ], capture_output=True, text=True, timeout=PROBE_TIMEOUT_SECONDS, @@ -174,8 +177,9 @@ def verified_task(hbench, task, kept_jobs_directory=None): if image is None: problems.append("cannot find the image hbench built") else: - summary["built_history"] = history(image, workdir, pull_request) - summary["upstream_history"] = history(metadata["base_image"], workdir, pull_request) + opened_at = metadata.get("created_at", "") + summary["built_history"] = history(image, workdir, pull_request, opened_at) + summary["upstream_history"] = history(metadata["base_image"], workdir, pull_request, opened_at) problems += history_problems(summary["built_history"]) summary["problems"] = problems return summary diff --git a/tests/test_swebench_verification.py b/tests/test_swebench_verification.py new file mode 100644 index 0000000..dd466d5 --- /dev/null +++ b/tests/test_swebench_verification.py @@ -0,0 +1,59 @@ +import os +import pathlib +import subprocess +import tempfile +import unittest + +from adapters.swebench import verification + +OPENED_AT = "2022-01-13T18:51:58Z" + + +def commit(repository, message, date): + environment = { + **os.environ, + "GIT_AUTHOR_NAME": "Test", + "GIT_AUTHOR_EMAIL": "test@example.com", + "GIT_AUTHOR_DATE": date, + "GIT_COMMITTER_NAME": "Test", + "GIT_COMMITTER_EMAIL": "test@example.com", + "GIT_COMMITTER_DATE": date, + } + subprocess.run( + ["git", "-C", str(repository), "commit", "--quiet", "--allow-empty", "-m", message], + check=True, + env=environment, + ) + + +def probe(repository, pull_request_number, opened_at): + completed = subprocess.run( + ["bash", "-c", verification.HISTORY_PROBE, "probe", str(repository), pull_request_number, opened_at], + check=True, + capture_output=True, + text=True, + ) + return dict(zip(verification.HISTORY_FIELDS, (int(value) for value in completed.stdout.split()))) + + +class HistoryProbeTest(unittest.TestCase): + def setUp(self): + self.repository = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory())) + subprocess.run(["git", "-C", str(self.repository), "init", "--quiet", "--initial-branch=main"], check=True) + commit(self.repository, "Fixed #15315 -- Added the widget argument to modelform_factory.", "2011-08-23T04:08:24Z") + + def test_a_ticket_older_than_the_pull_request_does_not_name_it(self): + counts = probe(self.repository, "15315", OPENED_AT) + self.assertEqual(counts["naming_pull_request"], 0) + self.assertEqual(verification.history_problems(counts), []) + + def test_a_message_since_the_pull_request_was_opened_names_it(self): + commit(self.repository, "Fixed #33412 -- Fixed the widget's attrs (#15315).", "2022-01-20T09:00:00Z") + self.assertEqual(probe(self.repository, "15315", OPENED_AT)["naming_pull_request"], 1) + + def test_without_an_opening_date_every_message_counts(self): + self.assertEqual(probe(self.repository, "15315", "")["naming_pull_request"], 1) + + +if __name__ == "__main__": + unittest.main()