diff --git a/adapters/swebench/verification.py b/adapters/swebench/verification.py index a5fac66..6f18ef1 100644 --- a/adapters/swebench/verification.py +++ b/adapters/swebench/verification.py @@ -35,7 +35,7 @@ "$(git rev-list --count --all)" \\ "$(git cat-file --batch-all-objects --batch-check='%(objecttype)' | grep -c '^commit$')" \\ "$(git tag --no-merged HEAD | wc -l)" \\ - "$(git log --all --format=%B | grep -cE "#$2([^0-9]|$)")" + "$(git log --all ${3:+--since="$3"} --format=%B | grep -cE "#$2([^0-9]|$)")" """ HISTORY_FIELDS = ("in_history", "reachable", "stored", "later_tags", "naming_pull_request") @@ -115,9 +115,12 @@ def built_image(task, workdir): return environments[0] if len(environments) == 1 else None -def history(image, workdir, pull_request_number): +def history(image, workdir, pull_request_number, opened_at=""): probed = subprocess.run( - ["docker", "run", "--rm", "--entrypoint", "/bin/bash", image, "-c", HISTORY_PROBE, "probe", workdir, pull_request_number], + [ + "docker", "run", "--rm", "--entrypoint", "/bin/bash", image, + "-c", HISTORY_PROBE, "probe", workdir, pull_request_number, opened_at, + ], capture_output=True, text=True, timeout=PROBE_TIMEOUT_SECONDS, @@ -174,8 +177,9 @@ def verified_task(hbench, task, kept_jobs_directory=None): if image is None: problems.append("cannot find the image hbench built") else: - summary["built_history"] = history(image, workdir, pull_request) - summary["upstream_history"] = history(metadata["base_image"], workdir, pull_request) + opened_at = metadata.get("created_at", "") + summary["built_history"] = history(image, workdir, pull_request, opened_at) + summary["upstream_history"] = history(metadata["base_image"], workdir, pull_request, opened_at) problems += history_problems(summary["built_history"]) summary["problems"] = problems return summary diff --git a/tests/test_swebench_verification.py b/tests/test_swebench_verification.py new file mode 100644 index 0000000..dd466d5 --- /dev/null +++ b/tests/test_swebench_verification.py @@ -0,0 +1,59 @@ +import os +import pathlib +import subprocess +import tempfile +import unittest + +from adapters.swebench import verification + +OPENED_AT = "2022-01-13T18:51:58Z" + + +def commit(repository, message, date): + environment = { + **os.environ, + "GIT_AUTHOR_NAME": "Test", + "GIT_AUTHOR_EMAIL": "test@example.com", + "GIT_AUTHOR_DATE": date, + "GIT_COMMITTER_NAME": "Test", + "GIT_COMMITTER_EMAIL": "test@example.com", + "GIT_COMMITTER_DATE": date, + } + subprocess.run( + ["git", "-C", str(repository), "commit", "--quiet", "--allow-empty", "-m", message], + check=True, + env=environment, + ) + + +def probe(repository, pull_request_number, opened_at): + completed = subprocess.run( + ["bash", "-c", verification.HISTORY_PROBE, "probe", str(repository), pull_request_number, opened_at], + check=True, + capture_output=True, + text=True, + ) + return dict(zip(verification.HISTORY_FIELDS, (int(value) for value in completed.stdout.split()))) + + +class HistoryProbeTest(unittest.TestCase): + def setUp(self): + self.repository = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory())) + subprocess.run(["git", "-C", str(self.repository), "init", "--quiet", "--initial-branch=main"], check=True) + commit(self.repository, "Fixed #15315 -- Added the widget argument to modelform_factory.", "2011-08-23T04:08:24Z") + + def test_a_ticket_older_than_the_pull_request_does_not_name_it(self): + counts = probe(self.repository, "15315", OPENED_AT) + self.assertEqual(counts["naming_pull_request"], 0) + self.assertEqual(verification.history_problems(counts), []) + + def test_a_message_since_the_pull_request_was_opened_names_it(self): + commit(self.repository, "Fixed #33412 -- Fixed the widget's attrs (#15315).", "2022-01-20T09:00:00Z") + self.assertEqual(probe(self.repository, "15315", OPENED_AT)["naming_pull_request"], 1) + + def test_without_an_opening_date_every_message_counts(self): + self.assertEqual(probe(self.repository, "15315", "")["naming_pull_request"], 1) + + +if __name__ == "__main__": + unittest.main()