From eec1f43980df072eec095cf315f39c681217ca3c Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Tue, 17 Nov 2026 14:48:46 -0500 Subject: [PATCH] Stop the sandbox when the student process reports a timeout Co-Authored-By: Claude Opus 5.5 --- src/pytest_prairielearn_grader/fixture.py | 8 ++++-- tests/test_subtests.py | 31 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 4160cd1..9b3e015 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -399,16 +399,20 @@ def query_function_raw( try: data: StudentFunctionResponse = self._read_query_response() except TimeoutError as error: - self._timed_out_call = f"{format_call(function_name, args, kwargs)} timed out after {query_timeout} seconds" data = { "status": FunctionStatusCode.TIMEOUT, "value": None, "stdout": "", "stderr": "", "exception_name": type(error).__name__, - "exception_message": f"{self._timed_out_call}.", + "exception_message": None, "traceback": None, } + # The student process also times out the call, and its reply can arrive before the socket + # read times out. Either way, the function may still be running, so stop the sandbox. + if data["status"] == FunctionStatusCode.TIMEOUT: + self._timed_out_call = f"{format_call(function_name, args, kwargs)} timed out after {query_timeout} seconds" + data["exception_message"] = f"{self._timed_out_call}." if data.get("stdout"): self._accumulated_stdout.append(data["stdout"]) return data diff --git a/tests/test_subtests.py b/tests/test_subtests.py index 6df6857..b8b46f7 100644 --- a/tests/test_subtests.py +++ b/tests/test_subtests.py @@ -195,3 +195,34 @@ def test_second(module_sandbox): assert results["first"]["message"] == "double(4) timed out after 0.2 seconds." assert results["second"]["message"] == "This check didn't run because double(4) timed out after 0.2 seconds." assert results["second"]["points_frac"] == 0 + + +def test_timeout_reported_by_student_process_stops_sandbox(pytester: pytest.Pytester) -> None: + # The socket waits a second longer than the call's timeout, so the student process's own + # timeout reply always arrives first. + result = run_grader( + pytester, + """ + class SlowSocket: + def __init__(self, sock): + self.sock = sock + + def settimeout(self, timeout): + self.sock.settimeout(timeout + 1) + + def __getattr__(self, name): + return getattr(self.sock, name) + + + @pytest.mark.grading_data(name="case", points=1) + def test_case(sandbox, subtests, monkeypatch): + monkeypatch.setattr(sandbox, "student_socket", SlowSocket(sandbox.student_socket)) + for x in [1, 4, 2, 5]: + with subtests.test(): + assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x, query_timeout=0.2) + """, + )["case"] + + assert result["points_frac"] == pytest.approx(0.25) + assert "double(4) timed out after 0.2 seconds." in result["message"] + assert "The remaining 2 checks didn't run because double(4) timed out after 0.2 seconds." in result["message"]