diff --git a/README.md b/README.md index b59f1c9..cb32a32 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ new version is a new tag. Each dataset is also mirrored to a second registry, no | `mbpp-plus@1.0` | 378 | Apache-2.0 | Generated; waits for its image lock | | `cruxeval-input@1.0` | 800 | Apache-2.0 | Generated; waits for its image lock | | `exercism-polyglot@1.0` | 210 | Apache-2.0 | Generated, 35 tasks in each of six languages; waits for its six images to be built | -| `swebench-verified@1.0` | 456 | Apache-2.0, with its repositories' licences | Generated; environments built locally | +| `swebench-verified@2.0` | 456 | Apache-2.0, with its repositories' licences | Generated; environments built locally | | `swebench-lite@1.0` | 270 | Apache-2.0, with its repositories' licences | Generated; environments built locally | | `swebench-multilingual@1.0` | 283 | Apache-2.0, with its repositories' licences | Generated; environments built locally | | `multi-swe-bench@1.0` | 1594 | Apache-2.0, with its repositories' licences | Generated; environments built locally | @@ -248,9 +248,11 @@ runners, and reports every task whose oracle or `nop` result is wrong. ### SWE-bench -`swebench-verified@1.0`, `swebench-lite@1.0` and `swebench-multilingual@1.0` are generated by +`swebench-verified@2.0`, `swebench-lite@1.0` and `swebench-multilingual@1.0` are generated by `adapters/swebench` from SWE-bench's own dataset files, and keep only the instances from repositories under a permissive licence (see [`docs/licensing.md`](docs/licensing.md#swe-bench-verified-lite-and-multilingual)). +Verified starts at 2.0 because `swebench-verified@1.0` already names a third-party set of 500 task +directories in other registries, with different bytes. Each task is named by its instance id, which `[metadata].instance_id` repeats, beside the repository, its licence, the base commit and the pull request. @@ -328,7 +330,7 @@ licence, in its directory and published with it. - **exercism-polyglot@1.0** is generated by node-q from Exercism's MIT-licensed problem specifications and C++, Go, Java, JavaScript, Python and Rust tracks. Its task directories are Apache-2.0, with Exercism's notices and Catch2's licence in its NOTICE. -- **swebench-verified@1.0, swebench-lite@1.0 and swebench-multilingual@1.0** are generated by node-q +- **swebench-verified@2.0, swebench-lite@1.0 and swebench-multilingual@1.0** are generated by node-q from SWE-bench's data, and **multi-swe-bench@1.0** from Multi-SWE-bench's (CC0, its harness Apache-2.0), keeping only instances from permissively licensed repositories. node-q's files are Apache-2.0; the issue texts and patches keep their repositories' licences, which each NOTICE diff --git a/adapters/swebench/verification.py b/adapters/swebench/verification.py index a5fac66..6f18ef1 100644 --- a/adapters/swebench/verification.py +++ b/adapters/swebench/verification.py @@ -35,7 +35,7 @@ "$(git rev-list --count --all)" \\ "$(git cat-file --batch-all-objects --batch-check='%(objecttype)' | grep -c '^commit$')" \\ "$(git tag --no-merged HEAD | wc -l)" \\ - "$(git log --all --format=%B | grep -cE "#$2([^0-9]|$)")" + "$(git log --all ${3:+--since="$3"} --format=%B | grep -cE "#$2([^0-9]|$)")" """ HISTORY_FIELDS = ("in_history", "reachable", "stored", "later_tags", "naming_pull_request") @@ -115,9 +115,12 @@ def built_image(task, workdir): return environments[0] if len(environments) == 1 else None -def history(image, workdir, pull_request_number): +def history(image, workdir, pull_request_number, opened_at=""): probed = subprocess.run( - ["docker", "run", "--rm", "--entrypoint", "/bin/bash", image, "-c", HISTORY_PROBE, "probe", workdir, pull_request_number], + [ + "docker", "run", "--rm", "--entrypoint", "/bin/bash", image, + "-c", HISTORY_PROBE, "probe", workdir, pull_request_number, opened_at, + ], capture_output=True, text=True, timeout=PROBE_TIMEOUT_SECONDS, @@ -174,8 +177,9 @@ def verified_task(hbench, task, kept_jobs_directory=None): if image is None: problems.append("cannot find the image hbench built") else: - summary["built_history"] = history(image, workdir, pull_request) - summary["upstream_history"] = history(metadata["base_image"], workdir, pull_request) + opened_at = metadata.get("created_at", "") + summary["built_history"] = history(image, workdir, pull_request, opened_at) + summary["upstream_history"] = history(metadata["base_image"], workdir, pull_request, opened_at) problems += history_problems(summary["built_history"]) summary["problems"] = problems return summary diff --git a/datasets/swebench-verified/1.0/LICENSE b/datasets/swebench-verified/2.0/LICENSE similarity index 100% rename from datasets/swebench-verified/1.0/LICENSE rename to datasets/swebench-verified/2.0/LICENSE diff --git a/datasets/swebench-verified/1.0/NOTICE b/datasets/swebench-verified/2.0/NOTICE similarity index 99% rename from datasets/swebench-verified/1.0/NOTICE rename to datasets/swebench-verified/2.0/NOTICE index 8e83f36..41003ad 100644 --- a/datasets/swebench-verified/1.0/NOTICE +++ b/datasets/swebench-verified/2.0/NOTICE @@ -1,4 +1,4 @@ -SWE-bench Verified 1.0 +SWE-bench Verified 2.0 ====================== This dataset is 456 task directories that node-q generates from SWE-bench Verified, one per instance diff --git a/datasets/swebench-verified/1.0/generated.lock.json b/datasets/swebench-verified/2.0/generated.lock.json similarity index 100% rename from datasets/swebench-verified/1.0/generated.lock.json rename to datasets/swebench-verified/2.0/generated.lock.json diff --git a/datasets/swebench-verified/1.0/images.lock.json b/datasets/swebench-verified/2.0/images.lock.json similarity index 100% rename from datasets/swebench-verified/1.0/images.lock.json rename to datasets/swebench-verified/2.0/images.lock.json diff --git a/datasets/swebench-verified/1.0/source.yaml b/datasets/swebench-verified/2.0/source.yaml similarity index 99% rename from datasets/swebench-verified/1.0/source.yaml rename to datasets/swebench-verified/2.0/source.yaml index 57bceab..cdc5bfe 100644 --- a/datasets/swebench-verified/1.0/source.yaml +++ b/datasets/swebench-verified/2.0/source.yaml @@ -1,7 +1,7 @@ schema_version: 1 name: swebench-verified -version: "1.0" -title: SWE-bench Verified 1.0 +version: "2.0" +title: SWE-bench Verified 2.0 description: >- 456 real GitHub issues from Python repositories under permissive licences: the instances of SWE-bench Verified, the human-validated 500-instance subset of SWE-bench, whose repository's licence diff --git a/docs/licensing.md b/docs/licensing.md index 2e6a889..880b2f4 100644 --- a/docs/licensing.md +++ b/docs/licensing.md @@ -15,7 +15,7 @@ upstream revision instead. | `mbpp-plus@1.0` | 378 | [MBPP+], Apache-2.0, from [MBPP], CC-BY-4.0 | Generated by node-q from the upstream data, Apache-2.0 | **Configured** in [`datasets/mbpp-plus/1.0`](../datasets/mbpp-plus/1.0); clear to publish once its image lock is filled | | `cruxeval-input@1.0` | 800 | [CRUXEval], MIT | Generated by node-q from the upstream data, Apache-2.0 | **Configured** in [`datasets/cruxeval-input/1.0`](../datasets/cruxeval-input/1.0); clear to publish once its image lock is filled | | `exercism-polyglot@1.0` | 210 | [Exercism] exercises, MIT; Catch2 in the C++ exercises, BSL-1.0 | Generated by node-q from Exercism's problem specifications and six tracks, Apache-2.0 | **Configured** in [`datasets/exercism-polyglot/1.0`](../datasets/exercism-polyglot/1.0); clear to publish once its six images are built | -| `swebench-verified@1.0` | 456 | [SWE-bench Verified]: no licence on its card; [SWE-bench] is MIT; issues and patches under their repositories' licences | Generated by node-q from the upstream data, Apache-2.0; only permissively licensed repositories | **Configured** in [`datasets/swebench-verified/1.0`](../datasets/swebench-verified/1.0); environments built locally, never redistributed | +| `swebench-verified@2.0` | 456 | [SWE-bench Verified]: no licence on its card; [SWE-bench] is MIT; issues and patches under their repositories' licences | Generated by node-q from the upstream data, Apache-2.0; only permissively licensed repositories | **Configured** in [`datasets/swebench-verified/2.0`](../datasets/swebench-verified/2.0); environments built locally, never redistributed | | `swebench-lite@1.0` | 270 | [SWE-bench Lite]: no licence on its card; [SWE-bench] is MIT; issues and patches under their repositories' licences | Generated by node-q from the upstream data, Apache-2.0; only permissively licensed repositories | **Configured** in [`datasets/swebench-lite/1.0`](../datasets/swebench-lite/1.0); environments built locally, never redistributed | | `swebench-multilingual@1.0` | 283 | [SWE-bench Multilingual], MIT on its card; issues and patches under their repositories' licences | Generated by node-q from the upstream data, Apache-2.0; only permissively licensed repositories | **Configured** in [`datasets/swebench-multilingual/1.0`](../datasets/swebench-multilingual/1.0); environments built locally, never redistributed | | `multi-swe-bench@1.0` | 1594 | [Multi-SWE-bench], CC0 subject to Bytedance's rights, harness Apache-2.0; issues and patches under their repositories' licences | Generated by node-q from the upstream data, Apache-2.0; only permissively licensed repositories | **Configured** in [`datasets/multi-swe-bench/1.0`](../datasets/multi-swe-bench/1.0); environments built locally, never redistributed | @@ -150,7 +150,7 @@ licence. The images are not part of the dataset's OCI artifact. ## SWE-bench Verified, Lite and Multilingual -The task directories of `swebench-verified@1.0`, `swebench-lite@1.0` and `swebench-multilingual@1.0` +The task directories of `swebench-verified@2.0`, `swebench-lite@1.0` and `swebench-multilingual@1.0` are node-q's: `adapters/swebench` writes them from SWE-bench's own dataset files on Hugging Face, each pinned by revision and sha256. node-q's files in them (instructions, `task.toml`, environment Dockerfiles, the history-removal script, the evaluation wrapper, the oracle script and the grading @@ -351,8 +351,9 @@ Until then, none of them has a descriptor. ### SWE-bench Verified -The third-party task directories are not republished. `swebench-verified@1.0` in this repository is -node-q's own, generated from SWE-bench's data; see +The third-party task directories are not republished. node-q's own task directories, generated +from SWE-bench's data, are `swebench-verified@2.0`: a new version, because `swebench-verified@1.0` +already names the third-party ones elsewhere. See [SWE-bench Verified, Lite and Multilingual](#swe-bench-verified-lite-and-multilingual). ### Aider Polyglot diff --git a/tests/test_swebench_verification.py b/tests/test_swebench_verification.py new file mode 100644 index 0000000..dd466d5 --- /dev/null +++ b/tests/test_swebench_verification.py @@ -0,0 +1,59 @@ +import os +import pathlib +import subprocess +import tempfile +import unittest + +from adapters.swebench import verification + +OPENED_AT = "2022-01-13T18:51:58Z" + + +def commit(repository, message, date): + environment = { + **os.environ, + "GIT_AUTHOR_NAME": "Test", + "GIT_AUTHOR_EMAIL": "test@example.com", + "GIT_AUTHOR_DATE": date, + "GIT_COMMITTER_NAME": "Test", + "GIT_COMMITTER_EMAIL": "test@example.com", + "GIT_COMMITTER_DATE": date, + } + subprocess.run( + ["git", "-C", str(repository), "commit", "--quiet", "--allow-empty", "-m", message], + check=True, + env=environment, + ) + + +def probe(repository, pull_request_number, opened_at): + completed = subprocess.run( + ["bash", "-c", verification.HISTORY_PROBE, "probe", str(repository), pull_request_number, opened_at], + check=True, + capture_output=True, + text=True, + ) + return dict(zip(verification.HISTORY_FIELDS, (int(value) for value in completed.stdout.split()))) + + +class HistoryProbeTest(unittest.TestCase): + def setUp(self): + self.repository = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory())) + subprocess.run(["git", "-C", str(self.repository), "init", "--quiet", "--initial-branch=main"], check=True) + commit(self.repository, "Fixed #15315 -- Added the widget argument to modelform_factory.", "2011-08-23T04:08:24Z") + + def test_a_ticket_older_than_the_pull_request_does_not_name_it(self): + counts = probe(self.repository, "15315", OPENED_AT) + self.assertEqual(counts["naming_pull_request"], 0) + self.assertEqual(verification.history_problems(counts), []) + + def test_a_message_since_the_pull_request_was_opened_names_it(self): + commit(self.repository, "Fixed #33412 -- Fixed the widget's attrs (#15315).", "2022-01-20T09:00:00Z") + self.assertEqual(probe(self.repository, "15315", OPENED_AT)["naming_pull_request"], 1) + + def test_without_an_opening_date_every_message_counts(self): + self.assertEqual(probe(self.repository, "15315", "")["naming_pull_request"], 1) + + +if __name__ == "__main__": + unittest.main()