From a31411759b80df65c50414452a7cfdd574cc7a8f Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Thu, 5 Nov 2026 13:12:56 -0500 Subject: [PATCH] Return serialization failures from query_raw normally When a student variable couldn't be serialized, the sandbox sent a failure message without the usual response fields and closed the connection. query then failed with KeyError: 'value', and later queries crashed with socket errors. The sandbox now serializes inside a try and sends a normal response with QueryStatusCode.NOT_SERIALIZABLE and an error naming the variable and its type, and keeps the connection open. query raises a RuntimeError with that message. Co-Authored-By: Claude Opus 5.5 --- quick_start.md | 2 +- .../_student_code_runner.py | 23 ++++-- src/pytest_prairielearn_grader/fixture.py | 8 ++- src/pytest_prairielearn_grader/utils.py | 2 + .../test_query_not_serializable/__init__.py | 0 .../expected_outcome.json | 72 +++++++++++++++++++ .../test_query_not_serializable/scenario.py | 58 +++++++++++++++ .../student_code.py | 9 +++ 8 files changed, 164 insertions(+), 10 deletions(-) create mode 100644 tests/scenario_root/test_query_not_serializable/__init__.py create mode 100644 tests/scenario_root/test_query_not_serializable/expected_outcome.json create mode 100644 tests/scenario_root/test_query_not_serializable/scenario.py create mode 100644 tests/scenario_root/test_query_not_serializable/student_code.py diff --git a/quick_start.md b/quick_start.md index f7dfe3b..594cbee 100644 --- a/quick_start.md +++ b/quick_start.md @@ -150,7 +150,7 @@ value = sandbox.query("variable_name") ``` Retrieves the value of a variable defined in the student code or `setup_code.py`. -Raises a `RuntimeError` if the variable doesn't exist. +Raises a `NameError` if the variable doesn't exist, and a `RuntimeError` with a clear message if its value can't be sent back to the grader. Later queries on the sandbox still work after either error. **Example:** diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index db954ae..2672270 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -529,14 +529,23 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit else: raise KeyError(var_to_query) - query_response: StudentQueryResponse = { - "status": QueryStatusCode.SUCCESS, - "value": to_json(value), - } except (KeyError, AttributeError, ImportError): - query_response = {"status": QueryStatusCode.NOT_FOUND, "value": ""} - - writer.write((json.dumps(query_response) + os.linesep).encode()) + not_found_response: StudentQueryResponse = {"status": QueryStatusCode.NOT_FOUND, "value": "", "error": None} + encoded_query_response = json.dumps(not_found_response) + else: + try: + query_response: StudentQueryResponse = {"status": QueryStatusCode.SUCCESS, "value": to_json(value), "error": None} + encoded_query_response = json.dumps(query_response) + except Exception as e: + not_serializable_response: StudentQueryResponse = { + "status": QueryStatusCode.NOT_SERIALIZABLE, + "value": None, + "error": f"Variable '{var_to_query}' has a value of type {type(value).__name__}, " + f"which can't be sent back to the grader: {e}", + } + encoded_query_response = json.dumps(not_serializable_response) + + writer.write((encoded_query_response + os.linesep).encode()) elif msg_type == "query_function": assert student_code_vars is not None diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 4c9975f..6c3e1ee 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -14,6 +14,7 @@ from .utils import ProcessStartRequest from .utils import ProcessStartResponse from .utils import ProcessStatusCode +from .utils import QueryStatusCode from .utils import SetupQueryRequest from .utils import SetupQueryResponse from .utils import StudentFunctionRequest @@ -243,8 +244,11 @@ def query_raw(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT def query(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> Any: """Query a variable from the student sandbox by name.""" response = self.query_raw(var_to_query, query_timeout=query_timeout) - if response["status"] == "not_found": - raise NameError(f"Query for '{var_to_query}' failed") + match response["status"]: + case QueryStatusCode.NOT_FOUND: + raise NameError(f"Query for '{var_to_query}' failed") + case QueryStatusCode.NOT_SERIALIZABLE: + raise RuntimeError(response["error"]) return from_json(response["value"]) def query_function_raw( diff --git a/src/pytest_prairielearn_grader/utils.py b/src/pytest_prairielearn_grader/utils.py index 74a6392..cf0e16c 100644 --- a/src/pytest_prairielearn_grader/utils.py +++ b/src/pytest_prairielearn_grader/utils.py @@ -25,6 +25,7 @@ class QueryStatusCode(StrEnum): SUCCESS = "success" NOT_FOUND = "not_found" + NOT_SERIALIZABLE = "not_serializable" class FunctionStatusCode(StrEnum): @@ -81,6 +82,7 @@ class StudentQueryResponse(TypedDict): # This is meant to be deserialized into a Python object status: QueryStatusCode value: Any + error: str | None # Function query dict types diff --git a/tests/scenario_root/test_query_not_serializable/__init__.py b/tests/scenario_root/test_query_not_serializable/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_query_not_serializable/expected_outcome.json b/tests/scenario_root/test_query_not_serializable/expected_outcome.json new file mode 100644 index 0000000..e917fdf --- /dev/null +++ b/tests/scenario_root/test_query_not_serializable/expected_outcome.json @@ -0,0 +1,72 @@ +{ + "expected_data_object": { + "score": 0.875, + "tests": [ + { + "max_points": 1, + "name": "raw_response", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_not_serializable.py::test_raw_response_has_all_fields[student_code]" + }, + { + "max_points": 1, + "name": "success_response", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_not_serializable.py::test_success_response_has_no_error[student_code]" + }, + { + "max_points": 1, + "name": "other_values", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_not_serializable.py::test_other_unserializable_values[student_code-nested-list]" + }, + { + "max_points": 1, + "name": "other_values", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_not_serializable.py::test_other_unserializable_values[student_code-self_reference-list]" + }, + { + "max_points": 1, + "name": "query", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_not_serializable.py::test_query_has_clear_error[student_code]" + }, + { + "max_points": 1, + "name": "sandbox_still_works", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_not_serializable.py::test_later_queries_still_work[student_code]" + }, + { + "max_points": 1, + "name": "not_found", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_not_serializable.py::test_missing_variable[student_code]" + }, + { + "max_points": 1, + "name": "assert_equal", + "points": 0.0, + "points_frac": 0.0, + "outcome": "failed", + "test_id": "test_query_not_serializable.py::test_assert_equal_shows_clear_error[student_code]", + "message": "Variable 'x' has a value of type Thing, which can't be sent back to the grader" + } + ] + } +} diff --git a/tests/scenario_root/test_query_not_serializable/scenario.py b/tests/scenario_root/test_query_not_serializable/scenario.py new file mode 100644 index 0000000..f2bdd78 --- /dev/null +++ b/tests/scenario_root/test_query_not_serializable/scenario.py @@ -0,0 +1,58 @@ +import pytest + +from pytest_prairielearn_grader.assertions import assert_equal +from pytest_prairielearn_grader.fixture import StudentFixture +from pytest_prairielearn_grader.utils import QueryStatusCode + +EXPECTED_MESSAGE = "Variable 'x' has a value of type Thing, which can't be sent back to the grader" + + +@pytest.mark.grading_data(name="raw_response", points=1) +def test_raw_response_has_all_fields(sandbox: StudentFixture) -> None: + response = sandbox.query_raw("x") + + assert response["status"] == QueryStatusCode.NOT_SERIALIZABLE + assert response["value"] is None + assert response["error"] is not None + assert EXPECTED_MESSAGE in response["error"] + + +@pytest.mark.grading_data(name="success_response", points=1) +def test_success_response_has_no_error(sandbox: StudentFixture) -> None: + assert sandbox.query_raw("y") == {"status": QueryStatusCode.SUCCESS, "value": 5, "error": None} + + +@pytest.mark.grading_data(name="other_values", points=1) +@pytest.mark.parametrize(("name", "type_name"), [("nested", "list"), ("self_reference", "list")]) +def test_other_unserializable_values(sandbox: StudentFixture, name: str, type_name: str) -> None: + with pytest.raises(RuntimeError, match=f"Variable '{name}' has a value of type {type_name}, which can't be sent back to the grader"): + sandbox.query(name) + + +@pytest.mark.grading_data(name="query", points=1) +def test_query_has_clear_error(sandbox: StudentFixture) -> None: + with pytest.raises(RuntimeError, match=EXPECTED_MESSAGE): + sandbox.query("x") + + +@pytest.mark.grading_data(name="sandbox_still_works", points=1) +def test_later_queries_still_work(sandbox: StudentFixture) -> None: + with pytest.raises(RuntimeError, match=EXPECTED_MESSAGE): + sandbox.query("x") + + assert sandbox.query("y") == 5 + assert sandbox.query_raw("x")["status"] == QueryStatusCode.NOT_SERIALIZABLE + assert sandbox.query_raw("y")["value"] == 5 + + +@pytest.mark.grading_data(name="not_found", points=1) +def test_missing_variable(sandbox: StudentFixture) -> None: + assert sandbox.query_raw("missing")["status"] == QueryStatusCode.NOT_FOUND + with pytest.raises(NameError, match="Query for 'missing' failed"): + sandbox.query("missing") + + +@pytest.mark.output(level="friendly") +@pytest.mark.grading_data(name="assert_equal", points=1) +def test_assert_equal_shows_clear_error(sandbox: StudentFixture) -> None: + assert_equal(sandbox.query("x"), 25, description="x") diff --git a/tests/scenario_root/test_query_not_serializable/student_code.py b/tests/scenario_root/test_query_not_serializable/student_code.py new file mode 100644 index 0000000..67c5a4f --- /dev/null +++ b/tests/scenario_root/test_query_not_serializable/student_code.py @@ -0,0 +1,9 @@ +class Thing: + pass + + +x = Thing() +y = 5 +nested = [1, {"thing": Thing()}] +self_reference = [1] +self_reference.append(self_reference)