From 777d42932ae711ca557d165857a805f1c282e4a4 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Wed, 4 Nov 2026 17:01:15 -0500 Subject: [PATCH 1/7] Stop deserializing student-influenced dill in query_setup query_setup asked the sandbox for dill-encoded values after student code had run, so student code could forge the data the grader then unpickled. Encode setup variables once, after setup code and before student code, and wait for the grader's ack before starting student code. query_setup now reads only that stored snapshot, so functions and other dill-only values keep working. Values dill can't encode raise a clear TypeError. Also add plot-serializer to the full image, and correct the docs to say plot grading needs MatplotlibSerializer rather than a plain Figure. Co-Authored-By: Claude Opus 5.5 --- .github/copilot-instructions.md | 7 +- docker/requirements-full.txt | 1 + quick_start.md | 51 +++++++++++++-- .../_student_code_runner.py | 62 +++++++++++------- src/pytest_prairielearn_grader/fixture.py | 65 ++++++++++++++----- src/pytest_prairielearn_grader/utils.py | 18 +++-- .../test_query_setup_snapshot/__init__.py | 0 .../test_query_setup_snapshot/data.json | 12 ++++ .../expected_outcome.json | 47 ++++++++++++++ .../test_query_setup_snapshot/scenario.py | 34 ++++++++++ .../test_query_setup_snapshot/setup_code.py | 11 ++++ .../test_query_setup_snapshot/student_code.py | 26 ++++++++ 12 files changed, 280 insertions(+), 54 deletions(-) create mode 100644 tests/scenario_root/test_query_setup_snapshot/__init__.py create mode 100644 tests/scenario_root/test_query_setup_snapshot/data.json create mode 100644 tests/scenario_root/test_query_setup_snapshot/expected_outcome.json create mode 100644 tests/scenario_root/test_query_setup_snapshot/scenario.py create mode 100644 tests/scenario_root/test_query_setup_snapshot/setup_code.py create mode 100644 tests/scenario_root/test_query_setup_snapshot/student_code.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 2175719..c5535e2 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -143,8 +143,9 @@ result = sandbox.query_function("student_func", arg1, arg2, timeout=5) ### 5. Serialization -- Uses `dill` for complex Python objects (numpy arrays, pandas DataFrames, matplotlib plots) -- Base64-encoded in JSON for transport +- Student-to-grader values (`query`, `query_function`) are JSON only, through `to_json` / `from_json` +- Grader-to-student function arguments use base64-encoded `dill` +- `query_setup` values use base64-encoded `dill`, but they're encoded and sent before student code starts, and the grader never asks the sandbox for them again. The grader must not deserialize `dill` from anything sent after that point - See [src/pytest_prairielearn_grader/json_utils.py](src/pytest_prairielearn_grader/json_utils.py) for custom serialization ## Important Integration Points @@ -169,7 +170,7 @@ result = sandbox.query_function("student_func", arg1, arg2, timeout=5) | "Student code server process terminated" | Check for unhandled exceptions in setup_code or timeout too short | | Function not found errors | Ensure function is defined in student_code.py, not just setup_code.py | | Timeout errors on network tests | Increase timeout via `@pytest.mark.sandbox_timeout(N)` | -| Serialization fails | Use `dill`-compatible types (check json_utils.py for supported types) | +| Serialization fails | Check json_utils.py for types supported from student code; `query_setup` needs `dill` | | Module sandbox with multiple student codes | Use regular `sandbox` fixture instead; module_sandbox requires single student_code.py | ## Files to Reference When Extending diff --git a/docker/requirements-full.txt b/docker/requirements-full.txt index 68e332d..5340f2c 100644 --- a/docker/requirements-full.txt +++ b/docker/requirements-full.txt @@ -10,6 +10,7 @@ numpy==2.5.4 openpyxl==3.1.5 pandas==3.1.0 Pillow==13.0.0 +plot-serializer==0.2.3 PuLP==4.2.0 Pygments==2.21.0 pytest==9.2.0 diff --git a/quick_start.md b/quick_start.md index d045a30..0b3fa1d 100644 --- a/quick_start.md +++ b/quick_start.md @@ -264,7 +264,30 @@ result = sandbox.query_function("slow_computation", data, query_timeout=2.0) **Note**: Student code must define the queried symbols, and return values must be JSON-serializable. Supported types include: `int`, `float`, `str`, `list`, `dict`, `bool`, `None`, numpy arrays, pandas DataFrames, -and matplotlib figures. +and plots built with `MatplotlibSerializer` (see [Testing Matplotlib Plots](#testing-matplotlib-plots)). + +### 5. Query Setup Code Values + +```python +value = sandbox.query_setup("variable_name") +``` + +Returns a variable from `setup_code.py`, including ones that aren't passed to the student in `names_for_user`. +Values can be any object `dill` can serialize, including functions: + +```python +@pytest.mark.grading_data(name="Check Answer", points=2) +def test_answer(sandbox: StudentFixture) -> None: + expected = sandbox.query_setup("compute_expected")(sandbox.query_setup("x")) + assert sandbox.query("answer") == expected +``` + +- **Values are captured before student code runs.** You get the value as it was when setup code finished, and student code can't change it. +- **Each call returns a new copy.** Changing it doesn't affect the sandbox or later calls. +- **Missing variables** raise a `NameError`. +- **Values `dill` can't serialize**, such as generators, raise a `TypeError` that names the variable. Other setup variables still work. + +Because every setup variable is serialized for each sandbox, keep large or slow-to-serialize values out of the top level of `setup_code.py` when you can. ## Partial Credit and Feedback @@ -950,9 +973,27 @@ since students can learn from reading full tracebacks. ### Testing Matplotlib Plots -The grader supports automatic serialization and deserialization of matplotlib figures: +To grade a plot, student code builds it with `MatplotlibSerializer` from the `plot-serializer` package and returns the serializer. The test receives a matplotlib `Figure`. + +`plot-serializer` is included in the `:latest` image but not in `:lite`. Returning a plain matplotlib `Figure` from student code isn't supported. + +```python +# Student code +from plot_serializer.matplotlib.serializer import MatplotlibSerializer + + +def create_plot(data): + serializer = MatplotlibSerializer() + fig, ax = serializer.subplots() + ax.plot(data) + ax.set_xlabel("Time") + ax.set_ylabel("Value") + ax.set_title("Data Visualization") + return serializer +``` ```python +# Test code import matplotlib matplotlib.use("Agg") # Non-GUI backend for server environments @@ -965,7 +1006,7 @@ from pytest_prairielearn_grader.fixture import StudentFixture @pytest.mark.grading_data(name="Plot Test", points=5) def test_student_plot(sandbox: StudentFixture) -> None: """Test that student creates a correct plot.""" - # Student function returns a matplotlib figure + # Student function returns a MatplotlibSerializer, which arrives here as a Figure plot = sandbox.query_function("create_plot", data) assert isinstance(plot, Figure) @@ -984,7 +1025,7 @@ def test_student_plot(sandbox: StudentFixture) -> None: The grader automatically serializes/deserializes: -- Matplotlib figures +- Plots built with `MatplotlibSerializer` (received as a `Figure`) - NumPy arrays - Pandas DataFrames - Standard Python types (int, float, str, list, dict, bool, None) @@ -1019,7 +1060,7 @@ Two image variants are published. Pick the smallest one that includes what your | Tag | Size (approx.) | Includes | Best for | | --------- | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| `:latest` | ~400–500 MB | Full scientific stack: `numpy`, `pandas`, `scipy`, `matplotlib` + `matplotcheck2`, `seaborn`, `scikit-learn`, `sympy`, `networkx`, `nltk`, `nbformat`, and more | Any course that needs scientific computing, data science, plotting, or Jupyter notebook grading | +| `:latest` | ~400–500 MB | Full scientific stack: `numpy`, `pandas`, `scipy`, `matplotlib` + `matplotcheck2` + `plot-serializer`, `seaborn`, `scikit-learn`, `sympy`, `networkx`, `nltk`, `nbformat`, and more | Any course that needs scientific computing, data science, plotting, or Jupyter notebook grading | | `:lite` | ~60–80 MB | Grader core only: `pytest-prairielearn-grader`, `Pygments`, `beautifulsoup4`, `defusedxml`, `requests` | Introductory CS courses where student code uses only the Python standard library | To use the lite image, change the `image` field: diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 020891d..0b0833b 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -10,6 +10,7 @@ import sys import traceback import types +from collections.abc import Awaitable from collections.abc import Callable from collections.abc import Mapping from collections.abc import Sequence @@ -28,8 +29,8 @@ from pytest_prairielearn_grader.utils import ProcessStartResponse from pytest_prairielearn_grader.utils import ProcessStatusCode from pytest_prairielearn_grader.utils import QueryStatusCode -from pytest_prairielearn_grader.utils import SetupQueryRequest from pytest_prairielearn_grader.utils import SetupQueryResponse +from pytest_prairielearn_grader.utils import SetupSnapshotMessage from pytest_prairielearn_grader.utils import StudentFunctionRequest from pytest_prairielearn_grader.utils import StudentFunctionResponse from pytest_prairielearn_grader.utils import StudentQueryRequest @@ -142,6 +143,22 @@ def student_function_temp() -> Any: return function_response +def encode_setup_variables(local_vars: dict[str, Any]) -> dict[str, SetupQueryResponse]: + """ + Dill-encode each setup variable for the grader. + + Each variable is encoded on its own so that one value dill can't handle only + fails a query for that value, not the whole sandbox. + """ + encoded: dict[str, SetupQueryResponse] = {} + for name, value in local_vars.items(): + try: + encoded[name] = {"status": QueryStatusCode.SUCCESS, "value_encoded": serialize_object_unsafe(value), "error": None} + except Exception as e: + encoded[name] = {"status": QueryStatusCode.NOT_SERIALIZABLE, "value_encoded": "", "error": f"{type(e).__name__}: {e}"} + return encoded + + def get_custom_importer( import_whitelist: list[str] | None, import_blacklist: list[str] | None, @@ -213,7 +230,8 @@ async def student_code_runner( starting_vars: dict[str, Any] | None, builtin_whitelist: list[str] | None, names_for_user_list: list[str] | None, -) -> tuple[dict[str, Any], dict[str, Any], ProcessStartResponse]: + on_setup_complete: Callable[[dict[str, Any]], Awaitable[None]], +) -> tuple[dict[str, Any], ProcessStartResponse]: stdout_capture = io.StringIO() stderr_capture = io.StringIO() execution_error: Exception | None = None @@ -259,6 +277,10 @@ async def student_code_runner( # If not in local_vars (setup_code), try getting from starting_vars student_code_vars[var_name] = deepcopy(starting_vars[var_name]) + # This has to finish before any student code runs: it's what guarantees the dill data + # the grader deserializes contains nothing the student defined or changed. + await on_setup_complete(local_vars if execution_error is None else {}) + if execution_error is None: try: # Next, compile student code. Make sure to handle errors in this later @@ -297,7 +319,7 @@ async def student_code_runner( "execution_traceback": str(exception_traceback), } - return local_vars, student_code_vars, result_dict + return student_code_vars, result_dict async def workspace_runner( @@ -310,7 +332,7 @@ async def workspace_runner( starting_vars: dict[str, Any] | None, builtin_whitelist: list[str] | None, names_for_user_list: list[str] | None, -) -> tuple[dict[str, Any], dict[str, Any], ProcessStartResponse]: +) -> tuple[dict[str, Any], ProcessStartResponse]: """ Sets up a workspace sandbox by inserting workspace_dir at the front of sys.path so that student modules can be imported via their dotted names. Optionally @@ -395,7 +417,7 @@ async def workspace_runner( "execution_traceback": str(exception_traceback), } - return local_vars, student_code_vars, result_dict + return student_code_vars, result_dict async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: @@ -425,7 +447,6 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit try: student_code_vars: None | dict = None - local_vars: None | dict = None workspace_mode: bool = False async for line_bytes in reader: @@ -440,7 +461,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit ws_json_message: WorkspaceStartRequest = json_message workspace_mode = True - local_vars, student_code_vars, start_response = await workspace_runner( + student_code_vars, start_response = await workspace_runner( workspace_dir=ws_json_message["workspace_dir"], exec_entry=ws_json_message["exec_entry"], setup_code=ws_json_message["setup_code"], @@ -470,7 +491,16 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit populate_linecache(student_code, student_file_name) - local_vars, student_code_vars, start_response = await student_code_runner( + async def send_setup_snapshot(setup_vars: dict[str, Any]) -> None: + snapshot: SetupSnapshotMessage = {"message_type": "setup_snapshot", "variables": encode_setup_variables(setup_vars)} + writer.write((json.dumps(snapshot) + os.linesep).encode()) + await writer.drain() + # drain() can return with data still buffered in this process, where student code + # could reach it. Only the grader's ack proves it has the whole snapshot. + if not await reader.readline(): + raise ConnectionError("Grader disconnected before acknowledging the setup snapshot.") + + student_code_vars, start_response = await student_code_runner( setup_code=setup_code, student_code=student_code, student_file_name=student_file_name, @@ -480,25 +510,11 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit starting_vars=starting_vars, builtin_whitelist=builtin_whitelist, names_for_user_list=names_for_user_list, + on_setup_complete=send_setup_snapshot, ) writer.write((json.dumps(start_response) + os.linesep).encode()) - elif msg_type == "query_setup": - assert local_vars is not None - query_setup_json_message: SetupQueryRequest = json_message - - var_to_query = query_setup_json_message["var"] - if var_to_query in local_vars: - setup_query_response: SetupQueryResponse = { - "status": QueryStatusCode.SUCCESS, - "value_encoded": serialize_object_unsafe(local_vars[var_to_query]), - } - else: - setup_query_response = {"status": QueryStatusCode.NOT_FOUND, "value_encoded": ""} - - writer.write((json.dumps(setup_query_response) + os.linesep).encode()) - elif msg_type == "query": assert student_code_vars is not None query_json_message: StudentQueryRequest = json_message diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 8adc3b2..606e0b0 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -14,8 +14,10 @@ from .utils import ProcessStartRequest from .utils import ProcessStartResponse from .utils import ProcessStatusCode -from .utils import SetupQueryRequest +from .utils import QueryStatusCode from .utils import SetupQueryResponse +from .utils import SetupSnapshotAck +from .utils import SetupSnapshotMessage from .utils import StudentFunctionRequest from .utils import StudentFunctionResponse from .utils import StudentQueryRequest @@ -80,6 +82,17 @@ class StudentFiles(NamedTuple): setup_code_file: Path +def _no_response(error: Exception) -> ProcessStartResponse: + return { + "status": ProcessStatusCode.NO_RESPONSE, + "execution_error": type(error).__name__, + "execution_message": str(error), + "execution_traceback": "", + "stdout": "", + "stderr": "", + } + + class FeedbackFixture: """ A fixture to handle feedback from the student code. @@ -221,14 +234,7 @@ def _read_start_response(self) -> ProcessStartResponse: if res.get("stdout"): self._accumulated_stdout.append(res["stdout"]) except Exception as e: - res = { - "status": ProcessStatusCode.NO_RESPONSE, - "execution_error": type(e).__name__, - "execution_message": str(e), - "execution_traceback": "", - "stdout": "", - "stderr": "", - } + res = _no_response(e) return res def query_raw(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> StudentQueryResponse: @@ -317,6 +323,7 @@ class StudentFixture(_SandboxBase): trailing_file: Path student_code_file: Path setup_code_file: Path + _setup_snapshot: dict[str, SetupQueryResponse] | None def __init__( self, @@ -335,6 +342,7 @@ def __init__( self.student_code_file = file_names.student_code_file self.setup_code_file = file_names.setup_code_file self.notebook_cell_tag = notebook_cell_tag + self._setup_snapshot = None def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_TIMEOUT) -> ProcessStartResponse: if self.worker_username is not None: @@ -377,21 +385,42 @@ def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_T names_for_user_list=self.names_for_user_list, ) self._send_json_object(json_message) + try: + self._receive_setup_snapshot() + except Exception as e: + return _no_response(e) return self._read_start_response() + def _receive_setup_snapshot(self) -> None: + """Store the setup variables and release the subprocess to start running student code.""" + snapshot: SetupSnapshotMessage = json.loads(self._read_from_socket().decode()) + if snapshot.get("message_type") != "setup_snapshot": + raise RuntimeError(f"Expected the setup snapshot from the sandbox, but got: {snapshot}") + self._setup_snapshot = snapshot["variables"] + ack: SetupSnapshotAck = {"message_type": "setup_snapshot_ack"} + self._send_json_object(ack) + def query_setup_raw(self, var_to_query: str) -> SetupQueryResponse: - self._assert_process_running() - json_message: SetupQueryRequest = {"message_type": "query_setup", "var": var_to_query} - assert self.student_socket is not None, "Student socket is not connected. Please start the student code server first." - self._send_json_object(json_message) - data: SetupQueryResponse = json.loads(self._read_from_socket().decode()) - return data + """Look up a setup variable in the snapshot taken before student code ran.""" + assert self._setup_snapshot is not None, "Setup variables are not available. Please start the student code server first." + response = self._setup_snapshot.get(var_to_query) + if response is None: + return {"status": QueryStatusCode.NOT_FOUND, "value_encoded": "", "error": None} + return response def query_setup(self, var_to_query: str) -> Any: - """Queries a variable from the setup code and returns its value.""" + """ + Return a value from the setup code, as it was when setup code finished. + + This doesn't contact the sandbox. Values are dill-encoded before student code + starts, so they can include functions and hold nothing from the student. + """ response = self.query_setup_raw(var_to_query) - if response["status"] == "not_found": - raise NameError(f"Query for setup variable '{var_to_query}' failed") + match response["status"]: + case QueryStatusCode.NOT_FOUND: + raise NameError(f"Query for setup variable '{var_to_query}' failed") + case QueryStatusCode.NOT_SERIALIZABLE: + raise TypeError(f"Setup variable '{var_to_query}' could not be serialized with dill: {response['error']}") return deserialize_object_unsafe(response["value_encoded"]) # TODO add functions that let instructors use the student fixture diff --git a/src/pytest_prairielearn_grader/utils.py b/src/pytest_prairielearn_grader/utils.py index 6ee5f8b..fead843 100644 --- a/src/pytest_prairielearn_grader/utils.py +++ b/src/pytest_prairielearn_grader/utils.py @@ -25,6 +25,7 @@ class QueryStatusCode(StrEnum): SUCCESS = "success" NOT_FOUND = "not_found" + NOT_SERIALIZABLE = "not_serializable" class FunctionStatusCode(StrEnum): @@ -56,14 +57,21 @@ class ProcessStatusCode(StrEnum): # TODO use some inheritance on the query and response types -class SetupQueryRequest(TypedDict): - message_type: Literal["query_setup"] - var: str - - class SetupQueryResponse(TypedDict): status: QueryStatusCode value_encoded: str + error: str | None + + +class SetupSnapshotMessage(TypedDict): + # Sent once per sandbox, after setup code and before student code, so its dill + # payloads hold nothing from the student. Nothing sent later may be deserialized with dill. + message_type: Literal["setup_snapshot"] + variables: dict[str, SetupQueryResponse] + + +class SetupSnapshotAck(TypedDict): + message_type: Literal["setup_snapshot_ack"] # Variable query dict types diff --git a/tests/scenario_root/test_query_setup_snapshot/__init__.py b/tests/scenario_root/test_query_setup_snapshot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_query_setup_snapshot/data.json b/tests/scenario_root/test_query_setup_snapshot/data.json new file mode 100644 index 0000000..9883336 --- /dev/null +++ b/tests/scenario_root/test_query_setup_snapshot/data.json @@ -0,0 +1,12 @@ +{ + "params": { + "import_blacklist": [], + "names_for_user": [ + { + "name": "shared", + "type": "list", + "description": "" + } + ] + } +} diff --git a/tests/scenario_root/test_query_setup_snapshot/expected_outcome.json b/tests/scenario_root/test_query_setup_snapshot/expected_outcome.json new file mode 100644 index 0000000..18dcd82 --- /dev/null +++ b/tests/scenario_root/test_query_setup_snapshot/expected_outcome.json @@ -0,0 +1,47 @@ +{ + "expected_data_object": { + "score": 1.0, + "tests": [ + { + "max_points": 1, + "name": "data_values", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_setup_values_ignore_student_changes[student_code]" + }, + { + "max_points": 1, + "name": "functions", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_setup_functions_are_returned[student_code]" + }, + { + "max_points": 1, + "name": "function_globals", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_setup_function_globals_exclude_student_values[student_code]" + }, + { + "max_points": 1, + "name": "unserializable", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_unserializable_setup_value_has_clear_error[student_code]" + }, + { + "max_points": 1, + "name": "not_found", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_missing_setup_value[student_code]" + } + ] + } +} diff --git a/tests/scenario_root/test_query_setup_snapshot/scenario.py b/tests/scenario_root/test_query_setup_snapshot/scenario.py new file mode 100644 index 0000000..5520943 --- /dev/null +++ b/tests/scenario_root/test_query_setup_snapshot/scenario.py @@ -0,0 +1,34 @@ +import builtins + +import pytest + +from pytest_prairielearn_grader.fixture import StudentFixture + + +@pytest.mark.grading_data(name="data_values", points=1) +def test_setup_values_ignore_student_changes(sandbox: StudentFixture) -> None: + assert sandbox.query_setup("value") == 42 + assert not hasattr(builtins, "grader_compromised") + + +@pytest.mark.grading_data(name="functions", points=1) +def test_setup_functions_are_returned(sandbox: StudentFixture) -> None: + assert sandbox.query_setup("add_twenty")(10) == 30 + + +@pytest.mark.grading_data(name="function_globals", points=1) +def test_setup_function_globals_exclude_student_values(sandbox: StudentFixture) -> None: + assert sandbox.query("shared") == "from student" + assert sandbox.query_setup("read_shared")() == [1, 2, 3] + + +@pytest.mark.grading_data(name="unserializable", points=1) +def test_unserializable_setup_value_has_clear_error(sandbox: StudentFixture) -> None: + with pytest.raises(TypeError, match="Setup variable 'unserializable' could not be serialized with dill"): + sandbox.query_setup("unserializable") + + +@pytest.mark.grading_data(name="not_found", points=1) +def test_missing_setup_value(sandbox: StudentFixture) -> None: + with pytest.raises(NameError, match="Query for setup variable 'missing' failed"): + sandbox.query_setup("missing") diff --git a/tests/scenario_root/test_query_setup_snapshot/setup_code.py b/tests/scenario_root/test_query_setup_snapshot/setup_code.py new file mode 100644 index 0000000..b908214 --- /dev/null +++ b/tests/scenario_root/test_query_setup_snapshot/setup_code.py @@ -0,0 +1,11 @@ +value = 42 +shared = [1, 2, 3] +unserializable = (i for i in range(3)) + + +def read_shared(): + return shared + + +def add_twenty(x): + return x + 20 diff --git a/tests/scenario_root/test_query_setup_snapshot/student_code.py b/tests/scenario_root/test_query_setup_snapshot/student_code.py new file mode 100644 index 0000000..cf1bbdb --- /dev/null +++ b/tests/scenario_root/test_query_setup_snapshot/student_code.py @@ -0,0 +1,26 @@ +import base64 +import builtins +import gc +import pickle +import sys + + +class Exploit: + def __reduce__(self): + return (builtins.exec, ("import builtins; builtins.grader_compromised = True",)) + + +payload = base64.b64encode(pickle.dumps(Exploit())).decode() + +# Forge whatever the runner would send if it serialized setup values on request. +sys.modules["__main__"].serialize_object_unsafe = lambda obj: payload +sys.modules["pytest_prairielearn_grader.utils"].serialize_object_unsafe = lambda obj: payload + +# Find the setup namespace and replace its values. +for obj in gc.get_objects(): + if isinstance(obj, dict) and "add_twenty" in obj and "value" in obj: + obj["value"] = Exploit() + obj["add_twenty"] = Exploit() + +# Rebind a global that setup functions look up when they're called. +shared = "from student" From c2d01dc025b0efc2d1fdf30c83f7a9549996c802 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Thu, 5 Nov 2026 10:35:50 -0500 Subject: [PATCH 2/7] Load PrairieLearn's data.json in the grader images PrairieLearn writes /grade/data/data.json, but the data_json fixture reads data.json next to the test file and run.sh never copied it there, so tests using data_json failed with a NoneType error. Copy it in run.sh. data.json can hold correct_answers, so make both copies root-owned with owner-only modes before student code runs, and stop grading if that fails. Root ownership matters because the student uid can match the host owner of mounted files, and owner-only modes matter because student code keeps root's supplementary groups. The fixture now raises a clear error for a data.json it can't parse or read, instead of treating it like a missing file. Co-Authored-By: Claude Opus 5.5 --- docker/run.sh | 33 ++++- quick_start.md | 2 + src/pytest_prairielearn_grader/plugin.py | 18 +-- .../test_docker_layout/scenario.py | 2 +- .../test_invalid_data_json/__init__.py | 0 .../test_invalid_data_json/data.json | 1 + .../expected_outcome.json | 15 +++ .../test_invalid_data_json/scenario.py | 5 + .../test_invalid_data_json/student_code.py | 1 + tests/test_run_script.py | 115 ++++++++++++++++++ 10 files changed, 178 insertions(+), 14 deletions(-) create mode 100644 tests/scenario_root/test_invalid_data_json/__init__.py create mode 100644 tests/scenario_root/test_invalid_data_json/data.json create mode 100644 tests/scenario_root/test_invalid_data_json/expected_outcome.json create mode 100644 tests/scenario_root/test_invalid_data_json/scenario.py create mode 100644 tests/scenario_root/test_invalid_data_json/student_code.py create mode 100644 tests/test_run_script.py diff --git a/docker/run.sh b/docker/run.sh index 2fbc49a..c8e0dd2 100644 --- a/docker/run.sh +++ b/docker/run.sh @@ -4,17 +4,19 @@ # INIT ########################## -if [ ! -d /grade ]; then - echo "ERROR: /grade not found! Mounting may have failed." +# the parent directory containing everything about this grading job +export JOB_DIR="${JOB_DIR:-/grade}" + +if [ ! -d "$JOB_DIR" ]; then + echo "ERROR: $JOB_DIR not found! Mounting may have failed." exit 1 fi -# the parent directory containing everything about this grading job -export JOB_DIR='/grade' # the job subdirectories STUDENT_DIR=$JOB_DIR'/student' TEST_DIR=$JOB_DIR'/tests' OUT_DIR=$JOB_DIR'/results' +DATA_DIR=$JOB_DIR'/data' # where we will copy everything export MERGE_DIR=$JOB_DIR'/run' @@ -33,6 +35,29 @@ mv "$STUDENT_DIR" "$MERGE_DIR/test_student/student" mv $TEST_DIR/* $MERGE_DIR/test_student/ mv $MERGE_DIR/test_student/test_student.py $MERGE_DIR/test_student.py +# PrairieLearn writes data.json to the data directory, but the data_json fixture reads it from +# the directory next to the test file. It can hold correct_answers, so both copies are for the +# grader only. They must be owned by root with owner-only modes: student code keeps root's +# supplementary groups, and its uid may match the owner of files mounted from the host. +deliver_data_json() { + if [ "$(id -u)" = 0 ]; then + chown -R root:root "$DATA_DIR" || return 1 + fi + chmod 700 "$DATA_DIR" || return 1 + chmod 600 "$DATA_DIR/data.json" || return 1 + ( + umask 077 + rm -f "$MERGE_DIR/test_student/data.json" + cp "$DATA_DIR/data.json" "$MERGE_DIR/test_student/data.json" + ) +} + +if [ -f "$DATA_DIR/data.json" ] && ! deliver_data_json; then + echo '{"succeeded": false, "score": 0.0, "message": "The autograder could not protect its grading data. Please contact course staff and have them check the logs for this submission."}' > "$OUT_DIR/results.json" + echo "ERROR: could not protect $DATA_DIR/data.json" + exit 1 +fi + # Do not allow ag user to modify, rename, or delete any existing files #chmod -R 755 "$MERGE_DIR" #chmod 1777 "$MERGE_DIR" diff --git a/quick_start.md b/quick_start.md index 0b3fa1d..c2940d5 100644 --- a/quick_start.md +++ b/quick_start.md @@ -626,6 +626,8 @@ def generate(data): These parameters are automatically passed to the autograder via `/grade/data/data.json` and enforced during student code execution. +In the grader images, `run.sh` copies `data.json` next to your test file, where the `data_json` fixture loads it. It also makes both copies readable only by root, so student code can't open the file, which can contain `correct_answers`. The sandbox still receives `params` through `__data_params`, so don't rely on values in `params` staying hidden from student code. + #### Testing Import/Builtin Restrictions Locally To test your restrictions in the test scenarios: diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 00b15c5..03a28a5 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -69,17 +69,17 @@ def get_datadir(test_module: ModuleType) -> Path | None: @pytest.fixture(scope="module") def data_json(request: pytest.FixtureRequest) -> dict[str, Any] | None: + datadir = get_datadir(request.module) + assert datadir is not None + data_file = datadir / "data.json" + logger.debug(f"Loading data.json from {data_file}") try: - datadir = get_datadir(request.module) - assert datadir is not None - data_file = datadir / "data.json" - logger.debug(f"Loading data.json from {data_file}") return json.loads(data_file.read_text(encoding="utf-8")) - except Exception as e: - logger.debug(f"No data.json found or failed to load: {e}") - - # If the data file is not found or cannot be read, return None - return None + except FileNotFoundError: + logger.debug(f"No data.json found at {data_file}") + return None + except (OSError, ValueError) as e: + raise RuntimeError(f"Could not load {data_file.name}: {e}") from e # if datadir is None or not datadir.is_dir(): # raise ValueError(f"Data directory '{datadir}' not found or is not a directory.") diff --git a/tests/scenario_root/test_docker_layout/scenario.py b/tests/scenario_root/test_docker_layout/scenario.py index 22b17df..a117f6e 100644 --- a/tests/scenario_root/test_docker_layout/scenario.py +++ b/tests/scenario_root/test_docker_layout/scenario.py @@ -10,7 +10,7 @@ student/ student_code.py <- moved from /grade/student/ by run.sh setup_code.py <- from /grade/tests/ (not present here) - data.json <- from /grade/tests/ (not present here) + data.json <- from /grade/data/ (not present here) The plugin's _find_student_files fallback discovers student_code.py inside the student/ subdirectory automatically when no files match the pattern at the root. diff --git a/tests/scenario_root/test_invalid_data_json/__init__.py b/tests/scenario_root/test_invalid_data_json/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_invalid_data_json/data.json b/tests/scenario_root/test_invalid_data_json/data.json new file mode 100644 index 0000000..42e2ff7 --- /dev/null +++ b/tests/scenario_root/test_invalid_data_json/data.json @@ -0,0 +1 @@ +{"params": diff --git a/tests/scenario_root/test_invalid_data_json/expected_outcome.json b/tests/scenario_root/test_invalid_data_json/expected_outcome.json new file mode 100644 index 0000000..2762715 --- /dev/null +++ b/tests/scenario_root/test_invalid_data_json/expected_outcome.json @@ -0,0 +1,15 @@ +{ + "expected_data_object": { + "score": 0.0, + "tests": [ + { + "max_points": 1, + "points_frac": 0.0, + "points": 0.0, + "test_id": "test_invalid_data_json.py::test_query[student_code]", + "outcome": "error", + "message": "Could not load data.json: Expecting value" + } + ] + } +} diff --git a/tests/scenario_root/test_invalid_data_json/scenario.py b/tests/scenario_root/test_invalid_data_json/scenario.py new file mode 100644 index 0000000..1950ffa --- /dev/null +++ b/tests/scenario_root/test_invalid_data_json/scenario.py @@ -0,0 +1,5 @@ +from pytest_prairielearn_grader.fixture import StudentFixture + + +def test_query(sandbox: StudentFixture) -> None: + assert sandbox.query("ANSWER") == 42 diff --git a/tests/scenario_root/test_invalid_data_json/student_code.py b/tests/scenario_root/test_invalid_data_json/student_code.py new file mode 100644 index 0000000..b0f8b2d --- /dev/null +++ b/tests/scenario_root/test_invalid_data_json/student_code.py @@ -0,0 +1 @@ +ANSWER = 42 diff --git a/tests/test_run_script.py b/tests/test_run_script.py new file mode 100644 index 0000000..c06f86a --- /dev/null +++ b/tests/test_run_script.py @@ -0,0 +1,115 @@ +"""Runs docker/run.sh against a temporary job directory, with the layout PrairieLearn creates.""" + +import json +import os +import stat +import subprocess +import sys +from pathlib import Path + +import pytest + +RUN_SH = Path(__file__).parents[1] / "docker" / "run.sh" + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="run.sh only runs on Unix systems") + +TEST_FILE = """ +from pytest_prairielearn_grader.fixture import DataFixture, StudentFixture + + +def test_params(sandbox: StudentFixture, data_json: DataFixture) -> None: + assert data_json["params"]["expected"] == 42 + assert sandbox.query("ANSWER") == 42 + assert sandbox.query_setup("from_setup") == 42 +""" + +SETUP_CODE = """ +from_setup = __data_params["expected"] +""" + +# Stands in for `uv run`, and drops the option that needs root and an 'ag' user. +FAKE_UV = f"""#!{sys.executable} +import os +import sys + +args = sys.argv[2:] +index = args.index("--worker-username") +del args[index : index + 2] +os.execv(sys.executable, [sys.executable, "-m", *args]) +""" + + +def mode(path: Path) -> int: + return stat.S_IMODE(path.stat().st_mode) + + +@pytest.fixture +def job_dir(tmp_path: Path) -> Path: + job = tmp_path / "grade" + (job / "tests").mkdir(parents=True) + (job / "student").mkdir() + (job / "tests" / "test_student.py").write_text(TEST_FILE) + (job / "tests" / "setup_code.py").write_text(SETUP_CODE) + (job / "student" / "student_code.py").write_text("ANSWER = 42\n") + return job + + +def write_data_json(job: Path, content: str) -> Path: + data_file = job / "data" / "data.json" + data_file.parent.mkdir() + data_file.write_text(content) + data_file.chmod(0o644) + return data_file + + +def run_grader(job: Path, tmp_path: Path) -> subprocess.CompletedProcess[str]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + fake_uv = bin_dir / "uv" + fake_uv.write_text(FAKE_UV) + fake_uv.chmod(0o755) + env = {**os.environ, "JOB_DIR": str(job), "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}"} + return subprocess.run(["/bin/sh", str(RUN_SH)], env=env, capture_output=True, text=True, check=False) + + +def test_data_json_reaches_grader_and_sandbox(job_dir: Path, tmp_path: Path) -> None: + content = json.dumps({"params": {"expected": 42}, "correct_answers": {"x": 1}}) + write_data_json(job_dir, content) + + run_grader(job_dir, tmp_path) + + assert json.loads((job_dir / "results" / "results.json").read_text())["score"] == 1.0 + assert (job_dir / "run" / "test_student" / "data.json").read_text() == content + + +def test_data_json_copies_are_owner_only(job_dir: Path, tmp_path: Path) -> None: + data_file = write_data_json(job_dir, json.dumps({"params": {"expected": 42}})) + + run_grader(job_dir, tmp_path) + + assert mode(job_dir / "run" / "test_student" / "data.json") == 0o600 + assert mode(data_file) == 0o600 + assert mode(data_file.parent) == 0o700 + + +def test_prairielearn_data_json_replaces_one_from_tests(job_dir: Path, tmp_path: Path) -> None: + stale = job_dir / "tests" / "data.json" + stale.write_text(json.dumps({"params": {"expected": 0}})) + stale.chmod(0o644) + write_data_json(job_dir, json.dumps({"params": {"expected": 42}})) + + run_grader(job_dir, tmp_path) + + delivered = job_dir / "run" / "test_student" / "data.json" + assert json.loads(delivered.read_text())["params"]["expected"] == 42 + assert mode(delivered) == 0o600 + + +def test_missing_data_json_still_grades(job_dir: Path, tmp_path: Path) -> None: + (job_dir / "tests" / "test_student.py").write_text("def test_runs() -> None:\n assert True\n") + (job_dir / "tests" / "setup_code.py").unlink() + + run_grader(job_dir, tmp_path) + + assert json.loads((job_dir / "results" / "results.json").read_text())["score"] == 1.0 + assert not (job_dir / "run" / "test_student" / "data.json").exists() From 17763ce70069212e4b2deb1812b675300a2cb781 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Thu, 5 Nov 2026 10:49:50 -0500 Subject: [PATCH 3/7] Read data.json from the job's data directory The data_json fixture now reads $JOB_DIR/data/data.json when run.sh sets JOB_DIR, and otherwise falls back to data.json next to the test file. This removes the copy from run.sh, so there's only one file holding answers to protect, and it stays out of a directory student code can read. The image's package and run.sh now depend on each other: this run.sh with package 1.1.0 finds no data.json. Co-Authored-By: Claude Opus 5.5 --- docker/run.sh | 19 +++----- quick_start.md | 2 +- src/pytest_prairielearn_grader/plugin.py | 17 +++++-- .../test_docker_layout/scenario.py | 1 - tests/test_run_script.py | 44 +++++++++++++------ 5 files changed, 51 insertions(+), 32 deletions(-) diff --git a/docker/run.sh b/docker/run.sh index c8e0dd2..a5a3da1 100644 --- a/docker/run.sh +++ b/docker/run.sh @@ -35,24 +35,17 @@ mv "$STUDENT_DIR" "$MERGE_DIR/test_student/student" mv $TEST_DIR/* $MERGE_DIR/test_student/ mv $MERGE_DIR/test_student/test_student.py $MERGE_DIR/test_student.py -# PrairieLearn writes data.json to the data directory, but the data_json fixture reads it from -# the directory next to the test file. It can hold correct_answers, so both copies are for the -# grader only. They must be owned by root with owner-only modes: student code keeps root's -# supplementary groups, and its uid may match the owner of files mounted from the host. -deliver_data_json() { +# The data_json fixture reads data.json from the data directory. It can hold correct_answers, so +# it's for the grader only. It must be owned by root with owner-only modes: student code keeps +# root's supplementary groups, and its uid may match the owner of files mounted from the host. +protect_data_json() { if [ "$(id -u)" = 0 ]; then chown -R root:root "$DATA_DIR" || return 1 fi - chmod 700 "$DATA_DIR" || return 1 - chmod 600 "$DATA_DIR/data.json" || return 1 - ( - umask 077 - rm -f "$MERGE_DIR/test_student/data.json" - cp "$DATA_DIR/data.json" "$MERGE_DIR/test_student/data.json" - ) + chmod 700 "$DATA_DIR" && chmod 600 "$DATA_DIR/data.json" } -if [ -f "$DATA_DIR/data.json" ] && ! deliver_data_json; then +if [ -f "$DATA_DIR/data.json" ] && ! protect_data_json; then echo '{"succeeded": false, "score": 0.0, "message": "The autograder could not protect its grading data. Please contact course staff and have them check the logs for this submission."}' > "$OUT_DIR/results.json" echo "ERROR: could not protect $DATA_DIR/data.json" exit 1 diff --git a/quick_start.md b/quick_start.md index c2940d5..29c38c0 100644 --- a/quick_start.md +++ b/quick_start.md @@ -626,7 +626,7 @@ def generate(data): These parameters are automatically passed to the autograder via `/grade/data/data.json` and enforced during student code execution. -In the grader images, `run.sh` copies `data.json` next to your test file, where the `data_json` fixture loads it. It also makes both copies readable only by root, so student code can't open the file, which can contain `correct_answers`. The sandbox still receives `params` through `__data_params`, so don't rely on values in `params` staying hidden from student code. +In the grader images, the `data_json` fixture loads this file directly, and `run.sh` makes it readable only by root, so student code can't open it. The file can contain `correct_answers`. When running outside the images, the fixture reads `data.json` next to your test file instead. The sandbox still receives `params` through `__data_params`, so don't rely on values in `params` staying hidden from student code. #### Testing Import/Builtin Restrictions Locally diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 03a28a5..44c2c70 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -67,11 +67,22 @@ def get_datadir(test_module: ModuleType) -> Path | None: return data_dir +def _find_data_json(module: Any) -> Path: + """Prefer the file PrairieLearn wrote for this grading job, which run.sh announces with JOB_DIR.""" + job_dir = os.environ.get("JOB_DIR") + if job_dir: + job_data_file = Path(job_dir) / "data" / "data.json" + if job_data_file.is_file(): + return job_data_file + + datadir = get_datadir(module) + assert datadir is not None + return datadir / "data.json" + + @pytest.fixture(scope="module") def data_json(request: pytest.FixtureRequest) -> dict[str, Any] | None: - datadir = get_datadir(request.module) - assert datadir is not None - data_file = datadir / "data.json" + data_file = _find_data_json(request.module) logger.debug(f"Loading data.json from {data_file}") try: return json.loads(data_file.read_text(encoding="utf-8")) diff --git a/tests/scenario_root/test_docker_layout/scenario.py b/tests/scenario_root/test_docker_layout/scenario.py index a117f6e..ed52db4 100644 --- a/tests/scenario_root/test_docker_layout/scenario.py +++ b/tests/scenario_root/test_docker_layout/scenario.py @@ -10,7 +10,6 @@ student/ student_code.py <- moved from /grade/student/ by run.sh setup_code.py <- from /grade/tests/ (not present here) - data.json <- from /grade/data/ (not present here) The plugin's _find_student_files fallback discovers student_code.py inside the student/ subdirectory automatically when no files match the pattern at the root. diff --git a/tests/test_run_script.py b/tests/test_run_script.py index c06f86a..1abfe86 100644 --- a/tests/test_run_script.py +++ b/tests/test_run_script.py @@ -72,37 +72,53 @@ def run_grader(job: Path, tmp_path: Path) -> subprocess.CompletedProcess[str]: return subprocess.run(["/bin/sh", str(RUN_SH)], env=env, capture_output=True, text=True, check=False) +def score(job: Path) -> float: + return json.loads((job / "results" / "results.json").read_text())["score"] + + def test_data_json_reaches_grader_and_sandbox(job_dir: Path, tmp_path: Path) -> None: - content = json.dumps({"params": {"expected": 42}, "correct_answers": {"x": 1}}) - write_data_json(job_dir, content) + write_data_json(job_dir, json.dumps({"params": {"expected": 42}, "correct_answers": {"x": 1}})) run_grader(job_dir, tmp_path) - assert json.loads((job_dir / "results" / "results.json").read_text())["score"] == 1.0 - assert (job_dir / "run" / "test_student" / "data.json").read_text() == content + assert score(job_dir) == 1.0 -def test_data_json_copies_are_owner_only(job_dir: Path, tmp_path: Path) -> None: +def test_data_json_is_owner_only_and_not_copied(job_dir: Path, tmp_path: Path) -> None: data_file = write_data_json(job_dir, json.dumps({"params": {"expected": 42}})) run_grader(job_dir, tmp_path) - assert mode(job_dir / "run" / "test_student" / "data.json") == 0o600 assert mode(data_file) == 0o600 assert mode(data_file.parent) == 0o700 + assert not (job_dir / "run" / "test_student" / "data.json").exists() -def test_prairielearn_data_json_replaces_one_from_tests(job_dir: Path, tmp_path: Path) -> None: - stale = job_dir / "tests" / "data.json" - stale.write_text(json.dumps({"params": {"expected": 0}})) - stale.chmod(0o644) +def test_prairielearn_data_json_takes_precedence_over_tests(job_dir: Path, tmp_path: Path) -> None: + (job_dir / "tests" / "data.json").write_text(json.dumps({"params": {"expected": 0}})) write_data_json(job_dir, json.dumps({"params": {"expected": 42}})) run_grader(job_dir, tmp_path) - delivered = job_dir / "run" / "test_student" / "data.json" - assert json.loads(delivered.read_text())["params"]["expected"] == 42 - assert mode(delivered) == 0o600 + assert score(job_dir) == 1.0 + + +def test_data_json_from_tests_used_without_prairielearn_file(job_dir: Path, tmp_path: Path) -> None: + (job_dir / "tests" / "data.json").write_text(json.dumps({"params": {"expected": 42}})) + + run_grader(job_dir, tmp_path) + + assert score(job_dir) == 1.0 + + +def test_invalid_prairielearn_data_json_is_an_error(job_dir: Path, tmp_path: Path) -> None: + write_data_json(job_dir, '{"params": ') + + run_grader(job_dir, tmp_path) + + results = json.loads((job_dir / "results" / "results.json").read_text()) + assert results["score"] == 0.0 + assert "Could not load data.json" in results["tests"][0]["message"] def test_missing_data_json_still_grades(job_dir: Path, tmp_path: Path) -> None: @@ -111,5 +127,5 @@ def test_missing_data_json_still_grades(job_dir: Path, tmp_path: Path) -> None: run_grader(job_dir, tmp_path) - assert json.loads((job_dir / "results" / "results.json").read_text())["score"] == 1.0 + assert score(job_dir) == 1.0 assert not (job_dir / "run" / "test_student" / "data.json").exists() From d9f71b7d824851b531427248f3ceb103ae7f3b2a Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Thu, 5 Nov 2026 11:07:55 -0500 Subject: [PATCH 4/7] Pass the data.json path as a pytest option Replace the fixture's JOB_DIR lookup with --data-json-path, next to --worker-username, so the plugin no longer depends on the environment or the job directory layout. run.sh now takes the job directory as an optional argument and no longer exports JOB_DIR or MERGE_DIR, which student subprocesses would inherit. The option has to use the --data-json-path=VALUE form: pytest's first pass over the arguments would treat a separate value as a test path, which moves rootdir and loses the results file. Co-Authored-By: Claude Opus 5.5 --- docker/run.sh | 15 +++++++++------ quick_start.md | 2 +- src/pytest_prairielearn_grader/plugin.py | 24 ++++++++++++++---------- tests/test_run_script.py | 4 ++-- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/docker/run.sh b/docker/run.sh index a5a3da1..c7f7709 100644 --- a/docker/run.sh +++ b/docker/run.sh @@ -5,7 +5,7 @@ ########################## # the parent directory containing everything about this grading job -export JOB_DIR="${JOB_DIR:-/grade}" +JOB_DIR="${1:-/grade}" if [ ! -d "$JOB_DIR" ]; then echo "ERROR: $JOB_DIR not found! Mounting may have failed." @@ -19,7 +19,7 @@ OUT_DIR=$JOB_DIR'/results' DATA_DIR=$JOB_DIR'/data' # where we will copy everything -export MERGE_DIR=$JOB_DIR'/run' +MERGE_DIR=$JOB_DIR'/run' # now set up the stuff so that our run.sh can work mkdir $MERGE_DIR @@ -35,9 +35,10 @@ mv "$STUDENT_DIR" "$MERGE_DIR/test_student/student" mv $TEST_DIR/* $MERGE_DIR/test_student/ mv $MERGE_DIR/test_student/test_student.py $MERGE_DIR/test_student.py -# The data_json fixture reads data.json from the data directory. It can hold correct_answers, so -# it's for the grader only. It must be owned by root with owner-only modes: student code keeps -# root's supplementary groups, and its uid may match the owner of files mounted from the host. +# PrairieLearn writes data.json to the data directory, and pytest is told to read it from there. +# It can hold correct_answers, so it's for the grader only. It must be owned by root with +# owner-only modes: student code keeps root's supplementary groups, and its uid may match the owner +# of files mounted from the host. protect_data_json() { if [ "$(id -u)" = 0 ]; then chown -R root:root "$DATA_DIR" || return 1 @@ -62,8 +63,10 @@ fi echo "[run] starting autograder" # run the autograder as a limited user called ag +# Pass --data-json-path with '=': pytest's first pass over the arguments, before plugin options are +# registered, would treat a separate value as a test path. # Use --worker-username to drop privileges to the 'ag' user in the student code sandbox -uv run pytest -p prairielearn-grader --color=no --log-cli-level=DEBUG --worker-username ag "$MERGE_DIR" +uv run pytest -p prairielearn-grader --color=no --log-cli-level=DEBUG --worker-username ag --data-json-path="$DATA_DIR/data.json" "$MERGE_DIR" # TODO change the default output name mv "$MERGE_DIR/autograder_results.json" "$OUT_DIR/results.json" diff --git a/quick_start.md b/quick_start.md index 29c38c0..a6e215d 100644 --- a/quick_start.md +++ b/quick_start.md @@ -626,7 +626,7 @@ def generate(data): These parameters are automatically passed to the autograder via `/grade/data/data.json` and enforced during student code execution. -In the grader images, the `data_json` fixture loads this file directly, and `run.sh` makes it readable only by root, so student code can't open it. The file can contain `correct_answers`. When running outside the images, the fixture reads `data.json` next to your test file instead. The sandbox still receives `params` through `__data_params`, so don't rely on values in `params` staying hidden from student code. +In the grader images, `run.sh` passes this path to pytest with `--data-json-path` and makes the file readable only by root, so student code can't open it. The file can contain `correct_answers`. Without that option, for example when running locally, the fixture reads `data.json` next to your test file. The sandbox still receives `params` through `__data_params`, so don't rely on values in `params` staying hidden from student code. #### Testing Import/Builtin Restrictions Locally diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 44c2c70..8cb96fe 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -67,22 +67,20 @@ def get_datadir(test_module: ModuleType) -> Path | None: return data_dir -def _find_data_json(module: Any) -> Path: - """Prefer the file PrairieLearn wrote for this grading job, which run.sh announces with JOB_DIR.""" - job_dir = os.environ.get("JOB_DIR") - if job_dir: - job_data_file = Path(job_dir) / "data" / "data.json" - if job_data_file.is_file(): - return job_data_file - - datadir = get_datadir(module) +def _find_data_json(request: pytest.FixtureRequest) -> Path: + """Prefer the file named by --data-json-path, and otherwise use data.json next to the test file.""" + data_json_path = request.config.getoption("--data-json-path") + if data_json_path is not None and Path(data_json_path).is_file(): + return Path(data_json_path) + + datadir = get_datadir(request.module) assert datadir is not None return datadir / "data.json" @pytest.fixture(scope="module") def data_json(request: pytest.FixtureRequest) -> dict[str, Any] | None: - data_file = _find_data_json(request.module) + data_file = _find_data_json(request) logger.debug(f"Loading data.json from {data_file}") try: return json.loads(data_file.read_text(encoding="utf-8")) @@ -688,6 +686,12 @@ def pytest_addoption(parser: pytest.Parser) -> None: """ group = parser.getgroup("pytest_pl_autograder", "Prairielearn Python Autograder Options") + group.addoption( + "--data-json-path", + action="store", + default=None, + help="Path to the data.json file for this grading job. If it doesn't exist, data.json next to the test file is used.", + ) group.addoption( "--worker-username", action="store", diff --git a/tests/test_run_script.py b/tests/test_run_script.py index 1abfe86..b7df0fc 100644 --- a/tests/test_run_script.py +++ b/tests/test_run_script.py @@ -68,8 +68,8 @@ def run_grader(job: Path, tmp_path: Path) -> subprocess.CompletedProcess[str]: fake_uv = bin_dir / "uv" fake_uv.write_text(FAKE_UV) fake_uv.chmod(0o755) - env = {**os.environ, "JOB_DIR": str(job), "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}"} - return subprocess.run(["/bin/sh", str(RUN_SH)], env=env, capture_output=True, text=True, check=False) + env = {**os.environ, "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}"} + return subprocess.run(["/bin/sh", str(RUN_SH), str(job)], env=env, capture_output=True, text=True, check=False) def score(job: Path) -> float: From b065f641699cd3112c03cfba6263161264b6d5ce Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Thu, 5 Nov 2026 11:15:19 -0500 Subject: [PATCH 5/7] Send query_setup data as JSON on demand Encoding every setup variable at startup slows every sandbox when setup code holds large values. Values JSON can represent exactly now stay in the sandbox and are sent as JSON only when queried. Values it can't, such as functions and tuples, are still dill-encoded once before student code starts, and only those pre-student payloads are ever unpickled. Function payloads leave out globals the function doesn't use. Student code could change a JSON value before it's queried, but a forged response can't run code in the grader. A value that can't be serialized when queried raises a clear TypeError. Co-Authored-By: Claude Opus 5.5 --- .github/copilot-instructions.md | 4 +- quick_start.md | 15 ++-- .../_student_code_runner.py | 54 ++++++++++-- src/pytest_prairielearn_grader/fixture.py | 43 ++++++---- src/pytest_prairielearn_grader/json_utils.py | 35 ++++++++ src/pytest_prairielearn_grader/utils.py | 23 ++++- .../expected_outcome.json | 30 ++++++- .../test_query_setup_snapshot/scenario.py | 34 +++++++- .../test_query_setup_snapshot/setup_code.py | 8 ++ .../test_query_setup_snapshot/student_code.py | 12 ++- tests/test_setup_query.py | 85 +++++++++++++++++++ 11 files changed, 298 insertions(+), 45 deletions(-) create mode 100644 tests/test_setup_query.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c5535e2..af3b8ef 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -145,7 +145,7 @@ result = sandbox.query_function("student_func", arg1, arg2, timeout=5) - Student-to-grader values (`query`, `query_function`) are JSON only, through `to_json` / `from_json` - Grader-to-student function arguments use base64-encoded `dill` -- `query_setup` values use base64-encoded `dill`, but they're encoded and sent before student code starts, and the grader never asks the sandbox for them again. The grader must not deserialize `dill` from anything sent after that point +- `query_setup` sends JSON-representable values as JSON when queried. Other values use base64-encoded `dill`, encoded and sent once before student code starts. The grader must not deserialize `dill` from anything sent after that point - See [src/pytest_prairielearn_grader/json_utils.py](src/pytest_prairielearn_grader/json_utils.py) for custom serialization ## Important Integration Points @@ -170,7 +170,7 @@ result = sandbox.query_function("student_func", arg1, arg2, timeout=5) | "Student code server process terminated" | Check for unhandled exceptions in setup_code or timeout too short | | Function not found errors | Ensure function is defined in student_code.py, not just setup_code.py | | Timeout errors on network tests | Increase timeout via `@pytest.mark.sandbox_timeout(N)` | -| Serialization fails | Check json_utils.py for types supported from student code; `query_setup` needs `dill` | +| Serialization fails | Check json_utils.py for types supported; `query_setup` also accepts `dill` values | | Module sandbox with multiple student codes | Use regular `sandbox` fixture instead; module_sandbox requires single student_code.py | ## Files to Reference When Extending diff --git a/quick_start.md b/quick_start.md index a6e215d..6ed18ba 100644 --- a/quick_start.md +++ b/quick_start.md @@ -273,7 +273,7 @@ value = sandbox.query_setup("variable_name") ``` Returns a variable from `setup_code.py`, including ones that aren't passed to the student in `names_for_user`. -Values can be any object `dill` can serialize, including functions: +Values can be functions and other objects, as long as JSON or `dill` can serialize them: ```python @pytest.mark.grading_data(name="Check Answer", points=2) @@ -282,12 +282,17 @@ def test_answer(sandbox: StudentFixture) -> None: assert sandbox.query("answer") == expected ``` -- **Values are captured before student code runs.** You get the value as it was when setup code finished, and student code can't change it. +How a value reaches your test depends on its type: + +| Value | How it's sent | Can student code change it? | +| --- | --- | --- | +| Numbers, strings, `bool`, `None`, lists and string-keyed dicts of these, numeric NumPy arrays and scalars, pandas DataFrames | As JSON, only when queried | It could, so don't rely on these values if students are working around the sandbox. It can't run code in the grader this way | +| Everything else, such as functions, classes, tuples, sets and other objects | Encoded with `dill` before student code runs | No | + +- **Large values:** prefer the JSON types for large data, because they add nothing to sandbox start time. Other values are encoded every time a sandbox starts, whether or not they're queried. Pass a longer `query_timeout` when querying large values, for example `sandbox.query_setup("data", query_timeout=10.0)`. The default is 1 second. - **Each call returns a new copy.** Changing it doesn't affect the sandbox or later calls. - **Missing variables** raise a `NameError`. -- **Values `dill` can't serialize**, such as generators, raise a `TypeError` that names the variable. Other setup variables still work. - -Because every setup variable is serialized for each sandbox, keep large or slow-to-serialize values out of the top level of `setup_code.py` when you can. +- **Values that can't be serialized**, such as generators, raise a `TypeError` that names the variable. Other setup variables still work. ## Partial Credit and Feedback diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 0b0833b..999bf8a 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -20,6 +20,7 @@ from typing import Any from pytest_prairielearn_grader.json_utils import from_server_json +from pytest_prairielearn_grader.json_utils import is_json_transportable # TODO make it so that other files in this package cannot import from this one # ask Gemini how to do it @@ -29,7 +30,9 @@ from pytest_prairielearn_grader.utils import ProcessStartResponse from pytest_prairielearn_grader.utils import ProcessStatusCode from pytest_prairielearn_grader.utils import QueryStatusCode +from pytest_prairielearn_grader.utils import SetupQueryRequest from pytest_prairielearn_grader.utils import SetupQueryResponse +from pytest_prairielearn_grader.utils import SetupSnapshotEntry from pytest_prairielearn_grader.utils import SetupSnapshotMessage from pytest_prairielearn_grader.utils import StudentFunctionRequest from pytest_prairielearn_grader.utils import StudentFunctionResponse @@ -143,17 +146,25 @@ def student_function_temp() -> Any: return function_response -def encode_setup_variables(local_vars: dict[str, Any]) -> dict[str, SetupQueryResponse]: +def encode_setup_variables(local_vars: dict[str, Any]) -> dict[str, SetupSnapshotEntry]: """ - Dill-encode each setup variable for the grader. + Dill-encode the setup variables that JSON can't represent, such as functions. - Each variable is encoded on its own so that one value dill can't handle only - fails a query for that value, not the whole sandbox. + Other variables aren't encoded here, so large data doesn't slow down starting the sandbox. + They're sent as JSON if they're queried. Each variable is encoded on its own so that one + value dill can't handle only fails a query for that value, not the whole sandbox. """ - encoded: dict[str, SetupQueryResponse] = {} + encoded: dict[str, SetupSnapshotEntry] = {} for name, value in local_vars.items(): + if is_json_transportable(value): + continue try: - encoded[name] = {"status": QueryStatusCode.SUCCESS, "value_encoded": serialize_object_unsafe(value), "error": None} + # recurse leaves out globals a function doesn't use, which can include large data. + encoded[name] = { + "status": QueryStatusCode.SUCCESS, + "value_encoded": serialize_object_unsafe(value, recurse=True), + "error": None, + } except Exception as e: encoded[name] = {"status": QueryStatusCode.NOT_SERIALIZABLE, "value_encoded": "", "error": f"{type(e).__name__}: {e}"} return encoded @@ -231,7 +242,7 @@ async def student_code_runner( builtin_whitelist: list[str] | None, names_for_user_list: list[str] | None, on_setup_complete: Callable[[dict[str, Any]], Awaitable[None]], -) -> tuple[dict[str, Any], ProcessStartResponse]: +) -> tuple[dict[str, Any], dict[str, Any], ProcessStartResponse]: stdout_capture = io.StringIO() stderr_capture = io.StringIO() execution_error: Exception | None = None @@ -319,7 +330,7 @@ async def student_code_runner( "execution_traceback": str(exception_traceback), } - return student_code_vars, result_dict + return local_vars, student_code_vars, result_dict async def workspace_runner( @@ -447,6 +458,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit try: student_code_vars: None | dict = None + local_vars: None | dict = None workspace_mode: bool = False async for line_bytes in reader: @@ -500,7 +512,7 @@ async def send_setup_snapshot(setup_vars: dict[str, Any]) -> None: if not await reader.readline(): raise ConnectionError("Grader disconnected before acknowledging the setup snapshot.") - student_code_vars, start_response = await student_code_runner( + local_vars, student_code_vars, start_response = await student_code_runner( setup_code=setup_code, student_code=student_code, student_file_name=student_file_name, @@ -515,6 +527,30 @@ async def send_setup_snapshot(setup_vars: dict[str, Any]) -> None: writer.write((json.dumps(start_response) + os.linesep).encode()) + elif msg_type == "query_setup": + assert local_vars is not None + query_setup_json_message: SetupQueryRequest = json_message + + var_to_query = query_setup_json_message["var"] + setup_query_response: SetupQueryResponse + if var_to_query not in local_vars: + setup_query_response = {"status": QueryStatusCode.NOT_FOUND, "value": None, "error": None} + encoded_response = json.dumps(setup_query_response) + else: + setup_query_response = {"status": QueryStatusCode.SUCCESS, "value": None, "error": None} + try: + setup_query_response["value"] = to_json(local_vars[var_to_query]) + encoded_response = json.dumps(setup_query_response) + except Exception as e: + setup_query_response = { + "status": QueryStatusCode.NOT_SERIALIZABLE, + "value": None, + "error": f"{type(e).__name__}: {e}", + } + encoded_response = json.dumps(setup_query_response) + + writer.write((encoded_response + os.linesep).encode()) + elif msg_type == "query": assert student_code_vars is not None query_json_message: StudentQueryRequest = json_message diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 606e0b0..fc40e3e 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -15,8 +15,10 @@ from .utils import ProcessStartResponse from .utils import ProcessStatusCode from .utils import QueryStatusCode +from .utils import SetupQueryRequest from .utils import SetupQueryResponse from .utils import SetupSnapshotAck +from .utils import SetupSnapshotEntry from .utils import SetupSnapshotMessage from .utils import StudentFunctionRequest from .utils import StudentFunctionResponse @@ -323,7 +325,7 @@ class StudentFixture(_SandboxBase): trailing_file: Path student_code_file: Path setup_code_file: Path - _setup_snapshot: dict[str, SetupQueryResponse] | None + _setup_snapshot: dict[str, SetupSnapshotEntry] | None def __init__( self, @@ -400,28 +402,39 @@ def _receive_setup_snapshot(self) -> None: ack: SetupSnapshotAck = {"message_type": "setup_snapshot_ack"} self._send_json_object(ack) - def query_setup_raw(self, var_to_query: str) -> SetupQueryResponse: - """Look up a setup variable in the snapshot taken before student code ran.""" - assert self._setup_snapshot is not None, "Setup variables are not available. Please start the student code server first." - response = self._setup_snapshot.get(var_to_query) - if response is None: - return {"status": QueryStatusCode.NOT_FOUND, "value_encoded": "", "error": None} - return response + def query_setup_raw(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> SetupQueryResponse: + """Ask the sandbox for a setup variable as JSON. Never deserialize dill from this response.""" + self._assert_process_running() + json_message: SetupQueryRequest = {"message_type": "query_setup", "var": var_to_query} + assert self.student_socket is not None, "Student socket is not connected. Please start the student code server first." + self.student_socket.settimeout(query_timeout) + self._send_json_object(json_message) + data: SetupQueryResponse = json.loads(self._read_from_socket().decode()) + return data - def query_setup(self, var_to_query: str) -> Any: + def query_setup(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> Any: """ - Return a value from the setup code, as it was when setup code finished. + Return a value from the setup code. - This doesn't contact the sandbox. Values are dill-encoded before student code - starts, so they can include functions and hold nothing from the student. + Values JSON can't represent, such as functions, were dill-encoded before student code + started, so they hold nothing from the student and are read from that stored copy. + Other values stay in the sandbox and are sent as JSON when queried. Student code could + change these, but they can't run code in the grader. """ - response = self.query_setup_raw(var_to_query) + assert self._setup_snapshot is not None, "Setup variables are not available. Please start the student code server first." + snapshot_entry = self._setup_snapshot.get(var_to_query) + if snapshot_entry is not None: + if snapshot_entry["status"] == QueryStatusCode.NOT_SERIALIZABLE: + raise TypeError(f"Setup variable '{var_to_query}' could not be serialized with dill: {snapshot_entry['error']}") + return deserialize_object_unsafe(snapshot_entry["value_encoded"]) + + response = self.query_setup_raw(var_to_query, query_timeout=query_timeout) match response["status"]: case QueryStatusCode.NOT_FOUND: raise NameError(f"Query for setup variable '{var_to_query}' failed") case QueryStatusCode.NOT_SERIALIZABLE: - raise TypeError(f"Setup variable '{var_to_query}' could not be serialized with dill: {response['error']}") - return deserialize_object_unsafe(response["value_encoded"]) + raise TypeError(f"Setup variable '{var_to_query}' could not be serialized as JSON: {response['error']}") + return from_json(response["value"]) # TODO add functions that let instructors use the student fixture # use the stuff pete set up here: https://github.com/reteps/pytest-autograder-prototype diff --git a/src/pytest_prairielearn_grader/json_utils.py b/src/pytest_prairielearn_grader/json_utils.py index 8f84265..36834de 100644 --- a/src/pytest_prairielearn_grader/json_utils.py +++ b/src/pytest_prairielearn_grader/json_utils.py @@ -61,6 +61,41 @@ def to_json(v: Any) -> Any: return v +def is_json_transportable(v: Any) -> bool: + """ + Return True if to_json and from_json can carry this value without changing what it is. + + This is deliberately strict: tuples, sets, non-string dict keys and subclasses would change + type or fail in JSON, so they return False. to_json only converts NumPy and pandas values + at the top level, so they return False when nested. Keep this in step with to_json. + """ + module = type(v).__module__.split(".")[0] + if module == "numpy": + import numpy as np + + if isinstance(v, np.ndarray): + return v.dtype.kind in "biufc" + return isinstance(v, np.number | np.bool_) + if module == "pandas": + import pandas as pd + + return isinstance(v, pd.DataFrame) + try: + return _is_plain_json(v) + except RecursionError: + return False + + +def _is_plain_json(v: Any) -> bool: + if type(v) in (type(None), bool, int, float, str): + return True + if type(v) is list: + return all(_is_plain_json(item) for item in v) + if type(v) is dict: + return all(type(key) is str and _is_plain_json(item) for key, item in v.items()) + return False + + def _has_value_fields(v, fields: list[str]) -> bool: """Return True if all fields in the '_value' dictionary are present.""" return "_value" in v and isinstance(v["_value"], dict) and all(field in v["_value"] for field in fields) diff --git a/src/pytest_prairielearn_grader/utils.py b/src/pytest_prairielearn_grader/utils.py index fead843..00c865d 100644 --- a/src/pytest_prairielearn_grader/utils.py +++ b/src/pytest_prairielearn_grader/utils.py @@ -57,7 +57,19 @@ class ProcessStatusCode(StrEnum): # TODO use some inheritance on the query and response types +class SetupQueryRequest(TypedDict): + message_type: Literal["query_setup"] + var: str + + class SetupQueryResponse(TypedDict): + # Student code may have run by the time this is sent, so it's JSON only. + status: QueryStatusCode + value: Any + error: str | None + + +class SetupSnapshotEntry(TypedDict): status: QueryStatusCode value_encoded: str error: str | None @@ -66,8 +78,9 @@ class SetupQueryResponse(TypedDict): class SetupSnapshotMessage(TypedDict): # Sent once per sandbox, after setup code and before student code, so its dill # payloads hold nothing from the student. Nothing sent later may be deserialized with dill. + # Holds only the setup variables that JSON can't represent. message_type: Literal["setup_snapshot"] - variables: dict[str, SetupQueryResponse] + variables: dict[str, SetupSnapshotEntry] class SetupSnapshotAck(TypedDict): @@ -145,16 +158,20 @@ class ProcessStartResponse(TypedDict): execution_traceback: str -def serialize_object_unsafe(obj: object) -> str: +def serialize_object_unsafe(obj: object, *, recurse: bool = False) -> str: """ Serializes an arbitrary Python object to a JSON string. The object is first serialized using dill, then base64 encoded. + Args: + obj: The object to serialize. + recurse: Include only the globals a function refers to, rather than all of them. + Returns: A JSON string representing the serialized object. """ # 1. Serialize the object using dill - dilled_bytes = dill.dumps(obj) + dilled_bytes = dill.dumps(obj, recurse=recurse) # 2. Base64 encode the byte stream base64_encoded_bytes = base64.b64encode(dilled_bytes) diff --git a/tests/scenario_root/test_query_setup_snapshot/expected_outcome.json b/tests/scenario_root/test_query_setup_snapshot/expected_outcome.json index 18dcd82..b202265 100644 --- a/tests/scenario_root/test_query_setup_snapshot/expected_outcome.json +++ b/tests/scenario_root/test_query_setup_snapshot/expected_outcome.json @@ -4,11 +4,19 @@ "tests": [ { "max_points": 1, - "name": "data_values", + "name": "json_values", "points": 1.0, "points_frac": 1.0, "outcome": "passed", - "test_id": "test_query_setup_snapshot.py::test_setup_values_ignore_student_changes[student_code]" + "test_id": "test_query_setup_snapshot.py::test_json_values_sent_on_demand[student_code]" + }, + { + "max_points": 1, + "name": "dill_values", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_values_json_would_change_keep_their_types[student_code]" }, { "max_points": 1, @@ -16,7 +24,7 @@ "points": 1.0, "points_frac": 1.0, "outcome": "passed", - "test_id": "test_query_setup_snapshot.py::test_setup_functions_are_returned[student_code]" + "test_id": "test_query_setup_snapshot.py::test_setup_functions_ignore_student_changes[student_code]" }, { "max_points": 1, @@ -26,6 +34,22 @@ "outcome": "passed", "test_id": "test_query_setup_snapshot.py::test_setup_function_globals_exclude_student_values[student_code]" }, + { + "max_points": 1, + "name": "forged_json", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_forged_json_response_cannot_run_code[student_code]" + }, + { + "max_points": 1, + "name": "replaced_value", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_value_replaced_with_object_has_clear_error[student_code]" + }, { "max_points": 1, "name": "unserializable", diff --git a/tests/scenario_root/test_query_setup_snapshot/scenario.py b/tests/scenario_root/test_query_setup_snapshot/scenario.py index 5520943..0fae9c1 100644 --- a/tests/scenario_root/test_query_setup_snapshot/scenario.py +++ b/tests/scenario_root/test_query_setup_snapshot/scenario.py @@ -1,19 +1,32 @@ import builtins +import numpy as np import pytest from pytest_prairielearn_grader.fixture import StudentFixture -@pytest.mark.grading_data(name="data_values", points=1) -def test_setup_values_ignore_student_changes(sandbox: StudentFixture) -> None: - assert sandbox.query_setup("value") == 42 +def assert_grader_not_compromised() -> None: assert not hasattr(builtins, "grader_compromised") +@pytest.mark.grading_data(name="json_values", points=1) +def test_json_values_sent_on_demand(sandbox: StudentFixture) -> None: + assert sandbox.query_setup("value") == 42 + np.testing.assert_array_equal(sandbox.query_setup("array_value"), np.array([1.0, 2.0])) + assert len(sandbox.query_setup("large_value", query_timeout=10.0)) == 200000 + + +@pytest.mark.grading_data(name="dill_values", points=1) +def test_values_json_would_change_keep_their_types(sandbox: StudentFixture) -> None: + assert sandbox.query_setup("tuple_value") == (1, 2) + assert sandbox.query_setup("int_keys_value") == {1: "one"} + + @pytest.mark.grading_data(name="functions", points=1) -def test_setup_functions_are_returned(sandbox: StudentFixture) -> None: +def test_setup_functions_ignore_student_changes(sandbox: StudentFixture) -> None: assert sandbox.query_setup("add_twenty")(10) == 30 + assert_grader_not_compromised() @pytest.mark.grading_data(name="function_globals", points=1) @@ -22,6 +35,19 @@ def test_setup_function_globals_exclude_student_values(sandbox: StudentFixture) assert sandbox.query_setup("read_shared")() == [1, 2, 3] +@pytest.mark.grading_data(name="forged_json", points=1) +def test_forged_json_response_cannot_run_code(sandbox: StudentFixture) -> None: + assert isinstance(sandbox.query_setup("forged_value"), str) + assert_grader_not_compromised() + + +@pytest.mark.grading_data(name="replaced_value", points=1) +def test_value_replaced_with_object_has_clear_error(sandbox: StudentFixture) -> None: + with pytest.raises(TypeError, match="Setup variable 'replaced_value' could not be serialized as JSON"): + sandbox.query_setup("replaced_value") + assert_grader_not_compromised() + + @pytest.mark.grading_data(name="unserializable", points=1) def test_unserializable_setup_value_has_clear_error(sandbox: StudentFixture) -> None: with pytest.raises(TypeError, match="Setup variable 'unserializable' could not be serialized with dill"): diff --git a/tests/scenario_root/test_query_setup_snapshot/setup_code.py b/tests/scenario_root/test_query_setup_snapshot/setup_code.py index b908214..f1a32f6 100644 --- a/tests/scenario_root/test_query_setup_snapshot/setup_code.py +++ b/tests/scenario_root/test_query_setup_snapshot/setup_code.py @@ -1,5 +1,13 @@ +import numpy as np + value = 42 +array_value = np.array([1.0, 2.0]) +large_value = list(range(200000)) +tuple_value = (1, 2) +int_keys_value = {1: "one"} shared = [1, 2, 3] +replaced_value = 1 +forged_value = 2 unserializable = (i for i in range(3)) diff --git a/tests/scenario_root/test_query_setup_snapshot/student_code.py b/tests/scenario_root/test_query_setup_snapshot/student_code.py index cf1bbdb..8bb3c30 100644 --- a/tests/scenario_root/test_query_setup_snapshot/student_code.py +++ b/tests/scenario_root/test_query_setup_snapshot/student_code.py @@ -11,16 +11,20 @@ def __reduce__(self): payload = base64.b64encode(pickle.dumps(Exploit())).decode() +runner = sys.modules["__main__"] +real_to_json = runner.to_json # Forge whatever the runner would send if it serialized setup values on request. -sys.modules["__main__"].serialize_object_unsafe = lambda obj: payload -sys.modules["pytest_prairielearn_grader.utils"].serialize_object_unsafe = lambda obj: payload +runner.serialize_object_unsafe = lambda *args, **kwargs: payload +sys.modules["pytest_prairielearn_grader.utils"].serialize_object_unsafe = lambda *args, **kwargs: payload +runner.to_json = lambda v: payload if isinstance(v, int) and v == 999 else real_to_json(v) # Find the setup namespace and replace its values. for obj in gc.get_objects(): - if isinstance(obj, dict) and "add_twenty" in obj and "value" in obj: - obj["value"] = Exploit() + if isinstance(obj, dict) and "add_twenty" in obj and "forged_value" in obj: obj["add_twenty"] = Exploit() + obj["replaced_value"] = Exploit() + obj["forged_value"] = 999 # Rebind a global that setup functions look up when they're called. shared = "from student" diff --git a/tests/test_setup_query.py b/tests/test_setup_query.py new file mode 100644 index 0000000..8623d37 --- /dev/null +++ b/tests/test_setup_query.py @@ -0,0 +1,85 @@ +from typing import Any + +import numpy as np +import pandas as pd +import pytest + +from pytest_prairielearn_grader._student_code_runner import encode_setup_variables +from pytest_prairielearn_grader.json_utils import from_json +from pytest_prairielearn_grader.json_utils import is_json_transportable +from pytest_prairielearn_grader.json_utils import to_json +from pytest_prairielearn_grader.utils import QueryStatusCode +from pytest_prairielearn_grader.utils import deserialize_object_unsafe + + +@pytest.mark.parametrize( + "value", + [None, True, 1, 1.5, "text", [1, [2]], {"a": {"b": [1]}}, np.array([1, 2]), np.array([1 + 2j]), np.float64(1.5), np.bool_(True)], +) +def test_json_transportable_values_round_trip(value: Any) -> None: + assert is_json_transportable(value) + np.testing.assert_equal(from_json(to_json(value)), value) + + +def test_dataframe_is_json_transportable() -> None: + value = pd.DataFrame({"a": [1, 2]}) + assert is_json_transportable(value) + pd.testing.assert_frame_equal(from_json(to_json(value)), value) + + +class ListSubclass(list): + pass + + +@pytest.mark.parametrize( + "value", + [ + (1, 2), + {1, 2}, + {1: "one"}, + [(1, 2)], + {"a": (1, 2)}, + [np.array([1])], + ListSubclass(), + len, + np.array(["a"]), + np.array([object()]), + pd.Series([1]), + ], +) +def test_values_json_would_change_are_not_transportable(value: Any) -> None: + assert not is_json_transportable(value) + + +def test_self_referencing_list_is_not_transportable() -> None: + value: list[Any] = [] + value.append(value) + assert not is_json_transportable(value) + + +def test_snapshot_leaves_out_json_values() -> None: + def function() -> int: + return 1 + + encoded = encode_setup_variables({"large": list(range(1000)), "params": {"a": 1}, "function": function}) + + assert set(encoded) == {"function"} + assert encoded["function"]["status"] == QueryStatusCode.SUCCESS + + +def test_snapshot_records_values_dill_cannot_encode() -> None: + encoded = encode_setup_variables({"generator": (i for i in range(3))}) + + assert encoded["generator"]["status"] == QueryStatusCode.NOT_SERIALIZABLE + assert encoded["generator"]["error"] + + +def test_function_snapshot_leaves_out_unused_globals() -> None: + namespace: dict[str, Any] = {"unused": "x" * 1_000_000, "used": 1} + exec("def function():\n return used\n", namespace) # noqa: S102 + + encoded = encode_setup_variables({"function": namespace["function"]}) + + assert len(encoded["function"]["value_encoded"]) < 10_000 + function = deserialize_object_unsafe(encoded["function"]["value_encoded"]) + assert function() == 1 # type: ignore[operator] From 083cdbefed4037cdbad66dd89faeff87fef69f07 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Thu, 5 Nov 2026 11:52:37 -0500 Subject: [PATCH 6/7] Give the matplotlib scenario a longer sandbox timeout Its first test failed in CI on Python 3.11 with "No response from initialization with timeout 1.0" while the other tests passed. The first sandbox imports matplotlib and plot_serializer, and those imports took about 8s locally without cached bytecode, against about 0.9s with it. The failure didn't reproduce locally, so this cause is likely but not confirmed. Co-Authored-By: Claude Opus 5.5 --- tests/scenario_root/test_matplotlib_plots/scenario.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/scenario_root/test_matplotlib_plots/scenario.py b/tests/scenario_root/test_matplotlib_plots/scenario.py index dcaa940..b66658c 100644 --- a/tests/scenario_root/test_matplotlib_plots/scenario.py +++ b/tests/scenario_root/test_matplotlib_plots/scenario.py @@ -8,6 +8,9 @@ from pytest_prairielearn_grader.fixture import StudentFixture +# In a fresh environment, the first sandbox's imports can take longer than the default timeout. +sandbox_timeout = 30.0 + @pytest.mark.grading_data(name="line_plot_basic", points=2) def test_line_plot_basic(sandbox: StudentFixture) -> None: From 24186eaeff4a06e1d8d908b0a71ecd13be87b0e3 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Wed, 18 Nov 2026 18:40:48 -0500 Subject: [PATCH 7/7] Port the setup-snapshot data protection to the pipe transport Re-implements #27's query_setup hardening on top of the merged multiprocessing pipe: setup variables JSON can't represent are dill-encoded once before student code runs and acked over the pipe; the grader reads those from the snapshot and only ever gets JSON from a live query_setup, never dill. Points the snapshot exploit scenario at the runner's real module name under spawn. --- src/pytest_prairielearn_grader/_student_code_runner.py | 6 +++++- .../scenario_root/test_query_setup_snapshot/student_code.py | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 2add944..1f9da29 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -823,7 +823,11 @@ def send_setup_snapshot(setup_vars: dict[str, Any]) -> None: "error": None, } except Exception as e: - setup_query_response = {"status": QueryStatusCode.NOT_SERIALIZABLE, "value": None, "error": f"{type(e).__name__}: {e}"} + setup_query_response = { + "status": QueryStatusCode.NOT_SERIALIZABLE, + "value": None, + "error": f"{type(e).__name__}: {e}", + } conn.send(setup_query_response) diff --git a/tests/scenario_root/test_query_setup_snapshot/student_code.py b/tests/scenario_root/test_query_setup_snapshot/student_code.py index 8bb3c30..2569314 100644 --- a/tests/scenario_root/test_query_setup_snapshot/student_code.py +++ b/tests/scenario_root/test_query_setup_snapshot/student_code.py @@ -11,7 +11,9 @@ def __reduce__(self): payload = base64.b64encode(pickle.dumps(Exploit())).decode() -runner = sys.modules["__main__"] +# The sandbox runs this module as a spawned child, so the runner is imported under its package name +# rather than as __main__. Reach it the way a real attacker on this runtime would. +runner = sys.modules["pytest_prairielearn_grader._student_code_runner"] real_to_json = runner.to_json # Forge whatever the runner would send if it serialized setup values on request.