diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index b364ca5..9bcd826 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -152,9 +152,9 @@ the originals through the `tests_dir` fixture. A missing listed file makes the s ### 5. Serialization -- Student values reach the grader as JSON through `to_json` and `from_json`, including numpy arrays, pandas DataFrames and matplotlib plots (see [src/pytest_prairielearn_grader/matplotlib_json.py](src/pytest_prairielearn_grader/matplotlib_json.py)) -- Uses `dill` for grader-to-student function arguments and `query_setup` values -- Base64-encoded in JSON for transport +- Student-to-grader values (`query`, `query_function`) are JSON only, through `to_json` / `from_json`, including numpy arrays, pandas DataFrames and matplotlib plots (see [src/pytest_prairielearn_grader/matplotlib_json.py](src/pytest_prairielearn_grader/matplotlib_json.py)) +- Grader-to-student function arguments use base64-encoded `dill` +- `query_setup` sends JSON-representable values as JSON when queried. Other values use base64-encoded `dill`, encoded and sent once before student code starts. The grader must not deserialize `dill` from anything sent after that point - See [src/pytest_prairielearn_grader/json_utils.py](src/pytest_prairielearn_grader/json_utils.py) for custom serialization ## Important Integration Points @@ -179,7 +179,7 @@ the originals through the `tests_dir` fixture. A missing listed file makes the s | "Student code server process terminated" | Check for unhandled exceptions in setup_code or timeout too short | | Function not found errors | Ensure function is defined in student_code.py, not just setup_code.py | | Timeout errors on network tests | Increase timeout via `@pytest.mark.sandbox_timeout(N)` | -| Serialization fails | Use `dill`-compatible types (check json_utils.py for supported types) | +| Serialization fails | Check json_utils.py for types supported; `query_setup` also accepts `dill` values | | Module sandbox with multiple student codes | Use regular `sandbox` fixture instead; module_sandbox requires single student_code.py | ## Files to Reference When Extending diff --git a/quick_start.md b/quick_start.md index 6cdb017..58c5bbd 100644 --- a/quick_start.md +++ b/quick_start.md @@ -383,17 +383,34 @@ So a timeout fails the rest of the test, and with a [module-scoped sandbox](#mod the later tests that use it too. In a [subtests loop](#checking-many-inputs), the inputs before the timeout keep their credit. -#### 5. Query Setup Variables +### 5. Query Setup Code Values ```python -x = sandbox.query_setup("x") +value = sandbox.query_setup("variable_name") ``` -Retrieves a variable defined by `setup_code.py`, whether or not it's in `names_for_user`. The value -comes from the same sandbox as the student's code, so a random value matches the one the student's -code received. +Returns a variable from `setup_code.py`, including ones that aren't passed to the student in `names_for_user`. +Values can be functions and other objects, as long as JSON or `dill` can serialize them: + +```python +@pytest.mark.grading_data(name="Check Answer", points=2) +def test_answer(sandbox: StudentFixture) -> None: + expected = sandbox.query_setup("compute_expected")(sandbox.query_setup("x")) + assert sandbox.query("answer") == expected +``` + +How a value reaches your test depends on its type: + +| Value | How it's sent | Can student code change it? | +| --- | --- | --- | +| Numbers, strings, `bool`, `None`, lists and string-keyed dicts of these, numeric NumPy arrays and scalars, pandas DataFrames | As JSON, only when queried | It could, so don't rely on these values if students are working around the sandbox. It can't run code in the grader this way | +| Everything else, such as functions, classes, tuples, sets and other objects | Encoded with `dill` before student code runs | No | + +- **Large values:** prefer the JSON types for large data, because they add nothing to sandbox start time. Other values are encoded every time a sandbox starts, whether or not they're queried. Pass a longer `query_timeout` when querying large values, for example `sandbox.query_setup("data", query_timeout=10.0)`. The default is 1 second. +- **Each call returns a new copy.** Changing it doesn't affect the sandbox or later calls. +- **Missing variables** raise a `NameError`. +- **Values that can't be serialized**, such as generators, raise a `TypeError` that names the variable. Other setup variables still work. -**Note**: Values sent back from the sandbox must be serializable. Supported types include: `int`, `float`, `str`, `list`, `dict`, `bool`, `None`, numpy arrays, pandas DataFrames, and matplotlib plots (see [Testing Matplotlib Plots](#testing-matplotlib-plots)). ### Checking Answers diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index a8fa0b8..1f9da29 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -20,6 +20,7 @@ from typing import Any from pytest_prairielearn_grader.json_utils import from_server_json +from pytest_prairielearn_grader.json_utils import is_json_transportable # TODO make it so that other files in this package cannot import from this one # ask Gemini how to do it @@ -34,6 +35,8 @@ from pytest_prairielearn_grader.utils import ServerStatusCode from pytest_prairielearn_grader.utils import SetupQueryRequest from pytest_prairielearn_grader.utils import SetupQueryResponse +from pytest_prairielearn_grader.utils import SetupSnapshotEntry +from pytest_prairielearn_grader.utils import SetupSnapshotMessage from pytest_prairielearn_grader.utils import StudentFunctionRequest from pytest_prairielearn_grader.utils import StudentFunctionResponse from pytest_prairielearn_grader.utils import StudentQueryRequest @@ -200,6 +203,30 @@ def student_function_temp() -> Any: return not_serializable_response +def encode_setup_variables(local_vars: dict[str, Any]) -> dict[str, SetupSnapshotEntry]: + """ + Dill-encode the setup variables that JSON can't represent, such as functions. + + Other variables aren't encoded here, so large data doesn't slow down starting the sandbox. + They're sent as JSON if they're queried. Each variable is encoded on its own so that one + value dill can't handle only fails a query for that value, not the whole sandbox. + """ + encoded: dict[str, SetupSnapshotEntry] = {} + for name, value in local_vars.items(): + if is_json_transportable(value): + continue + try: + # recurse leaves out globals a function doesn't use, which can include large data. + encoded[name] = { + "status": QueryStatusCode.SUCCESS, + "value_encoded": serialize_object_unsafe(value, recurse=True), + "error": None, + } + except Exception as e: + encoded[name] = {"status": QueryStatusCode.NOT_SERIALIZABLE, "value_encoded": "", "error": f"{type(e).__name__}: {e}"} + return encoded + + def _find_blacklisted(name: str, fromlist: Sequence[str], import_blacklist: list[str]) -> str | None: """ Returns the module that an import would load in spite of the blacklist, or None. @@ -466,6 +493,7 @@ def _run_sandbox( workspace_dir: str | None, load_main: Callable[[], tuple[str, str]] | None, main_timeout_message: str, + on_setup_complete: Callable[[dict[str, Any]], None] | None = None, ) -> tuple[dict[str, Any], dict[str, Any], ProcessStartResponse]: """ Shared implementation behind student_code_runner and workspace_runner. @@ -477,6 +505,10 @@ def _run_sandbox( real builtins.__import__ and the directory is put at the front of sys.path, so that student modules loaded via importlib.import_module() go through the same restrictions and resolve against the workspace. + + *on_setup_complete*, if given, is called with the setup namespace after setup code finishes and + before any main/student code runs. It must complete before the main step: it's what lets the + grader capture a snapshot of setup values that nothing the student defined or changed can reach. """ stdout_capture = io.StringIO() stderr_capture = io.StringIO() @@ -540,6 +572,11 @@ def _run_sandbox( # If not in local_vars (setup_code), try getting from starting_vars student_code_vars[var_name] = deepcopy(starting_vars[var_name]) + # This has to finish before any student code runs: it's what guarantees the dill data the + # grader deserializes contains nothing the student defined or changed. + if on_setup_complete is not None: + on_setup_complete(local_vars if execution_error is None else {}) + if execution_error is None and load_main is not None: try: # Compile and run the main code (student submission or workspace entry point). @@ -589,6 +626,7 @@ def student_code_runner( builtin_whitelist: list[str] | None, names_for_user_list: list[str] | None, helper_modules: dict[str, str] | None, + on_setup_complete: Callable[[dict[str, Any]], None] | None = None, ) -> tuple[dict[str, Any], dict[str, Any], ProcessStartResponse]: return _run_sandbox( setup_code=setup_code, @@ -602,6 +640,7 @@ def student_code_runner( workspace_dir=None, load_main=lambda: (student_code, student_file_name), main_timeout_message="Student code execution timed out", + on_setup_complete=on_setup_complete, ) @@ -743,6 +782,13 @@ def _serve(conn: Connection, worker_username: str | None, data_files_dir: str | populate_linecache(student_code, student_file_name) + def send_setup_snapshot(setup_vars: dict[str, Any]) -> None: + snapshot: SetupSnapshotMessage = {"message_type": "setup_snapshot", "variables": encode_setup_variables(setup_vars)} + conn.send(snapshot) + # Wait for the grader to acknowledge before student code runs. Only the ack proves + # the grader has the whole snapshot; a recv failure means it's gone, so stop. + conn.recv() + local_vars, student_code_vars, start_response = student_code_runner( setup_code=setup_code, student_code=student_code, @@ -754,6 +800,7 @@ def _serve(conn: Connection, worker_username: str | None, data_files_dir: str | builtin_whitelist=builtin_whitelist, names_for_user_list=names_for_user_list, helper_modules=start_json_message["helper_modules"], + on_setup_complete=send_setup_snapshot, ) conn.send(start_response) @@ -763,13 +810,24 @@ def _serve(conn: Connection, worker_username: str | None, data_files_dir: str | query_setup_json_message: SetupQueryRequest = json_message var_to_query = query_setup_json_message["var"] - if var_to_query in local_vars: - setup_query_response: SetupQueryResponse = { - "status": QueryStatusCode.SUCCESS, - "value_encoded": serialize_object_unsafe(local_vars[var_to_query]), - } + # Student code may have run by now, so a setup value is only ever sent as JSON here; + # the grader never deserializes dill from this response. Values JSON can't represent + # come from the snapshot taken before student code ran, not from this path. + if var_to_query not in local_vars: + setup_query_response: SetupQueryResponse = {"status": QueryStatusCode.NOT_FOUND, "value": None, "error": None} else: - setup_query_response = {"status": QueryStatusCode.NOT_FOUND, "value_encoded": ""} + try: + setup_query_response = { + "status": QueryStatusCode.SUCCESS, + "value": json.dumps(to_json(local_vars[var_to_query])), + "error": None, + } + except Exception as e: + setup_query_response = { + "status": QueryStatusCode.NOT_SERIALIZABLE, + "value": None, + "error": f"{type(e).__name__}: {e}", + } conn.send(setup_query_response) diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 4f68ff0..9205f5e 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -26,6 +26,9 @@ from .utils import ServerStatusCode from .utils import SetupQueryRequest from .utils import SetupQueryResponse +from .utils import SetupSnapshotAck +from .utils import SetupSnapshotEntry +from .utils import SetupSnapshotMessage from .utils import StudentFunctionRequest from .utils import StudentFunctionResponse from .utils import StudentQueryRequest @@ -146,6 +149,17 @@ class StudentFiles(NamedTuple): setup_code_file: Path +def _no_response(error: Exception) -> ProcessStartResponse: + return { + "status": ProcessStatusCode.NO_RESPONSE, + "execution_error": type(error).__name__, + "execution_message": str(error), + "execution_traceback": "", + "stdout": "", + "stderr": "", + } + + class FeedbackFixture: """ A fixture to handle feedback from the student code. @@ -262,11 +276,28 @@ def __init__( self._server_failure = None self._timed_out_call = None - def _start_server(self, initialization_timeout: float, build_start_message: Callable[[], Any]) -> ProcessStartResponse: - """Spawn the child process, send the start message it builds, and read the start response.""" + def _start_server( + self, + initialization_timeout: float, + build_start_message: Callable[[], Any], + after_send: Callable[[], None] | None = None, + ) -> ProcessStartResponse: + """Spawn the child process, send the start message it builds, and read the start response. + + *after_send*, if given, runs between sending the start message and reading the start + response. The student fixture uses it to receive the setup snapshot and release the sandbox. + """ self._start_subprocess() json_message = build_start_message() + # The snapshot and the start response both arrive after setup/student code, so allow the + # full initialization window for each read. + self._read_timeout = initialization_timeout + START_RESPONSE_GRACE self._send_json_object(json_message) + if after_send is not None: + try: + after_send() + except Exception as e: + return _no_response(e) return self._read_start_response(initialization_timeout) def _assert_process_running(self) -> None: @@ -351,14 +382,7 @@ def _read_start_response(self, initialization_timeout: float) -> ProcessStartRes if res.get("stdout"): self._accumulated_stdout.append(res["stdout"]) except Exception as e: - res = { - "status": ProcessStatusCode.NO_RESPONSE, - "execution_error": type(e).__name__, - "execution_message": str(e), - "execution_traceback": "", - "stdout": "", - "stderr": "", - } + res = _no_response(e) return res def query_raw(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> StudentQueryResponse: @@ -465,6 +489,7 @@ class StudentFixture(_SandboxBase): trailing_file: Path student_code_file: Path setup_code_file: Path + _setup_snapshot: dict[str, SetupSnapshotEntry] | None def __init__( self, @@ -494,6 +519,7 @@ def __init__( self.student_code_file = file_names.student_code_file self.setup_code_file = file_names.setup_code_file self.notebook_cell_tag = notebook_cell_tag + self._setup_snapshot = None def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_TIMEOUT) -> ProcessStartResponse: if self.worker_username is not None: @@ -534,22 +560,50 @@ def build_start_message() -> ProcessStartRequest: helper_modules=self.helper_modules, ) - return self._start_server(initialization_timeout, build_start_message) + return self._start_server(initialization_timeout, build_start_message, after_send=self._receive_setup_snapshot) - def query_setup_raw(self, var_to_query: str) -> SetupQueryResponse: + def _receive_setup_snapshot(self) -> None: + """Store the setup snapshot and acknowledge it, which releases the sandbox to run student code.""" + snapshot: SetupSnapshotMessage = self._recv_message() + if snapshot.get("message_type") != "setup_snapshot": + raise RuntimeError(f"Expected the setup snapshot from the sandbox, but got: {snapshot}") + self._setup_snapshot = snapshot["variables"] + ack: SetupSnapshotAck = {"message_type": "setup_snapshot_ack"} + self._send_json_object(ack) + + def query_setup_raw(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> SetupQueryResponse: + """Ask the sandbox for a setup variable as JSON. Never deserialize dill from this response.""" self._assert_no_server_failure() self._assert_process_running() json_message: SetupQueryRequest = {"message_type": MessageType.QUERY_SETUP, "var": var_to_query} + self._read_timeout = query_timeout self._send_json_object(json_message) - data: SetupQueryResponse = self._recv_message() + data: SetupQueryResponse = self._read_query_response() return data - def query_setup(self, var_to_query: str) -> Any: - """Queries a variable from the setup code and returns its value.""" - response = self.query_setup_raw(var_to_query) - if response["status"] == "not_found": - raise NameError(f"Query for setup variable '{var_to_query}' failed") - return deserialize_object_unsafe(response["value_encoded"]) + def query_setup(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> Any: + """ + Return a value from the setup code. + + Values JSON can't represent, such as functions, were dill-encoded before student code + started, so they hold nothing from the student and are read from that stored snapshot. + Other values stay in the sandbox and are sent as JSON when queried. Student code that gets + around the sandbox could change these, but it can't run code in the grader this way. + """ + assert self._setup_snapshot is not None, "Setup variables are not available. Please start the student code server first." + snapshot_entry = self._setup_snapshot.get(var_to_query) + if snapshot_entry is not None: + if snapshot_entry["status"] == QueryStatusCode.NOT_SERIALIZABLE: + raise TypeError(f"Setup variable '{var_to_query}' could not be serialized with dill: {snapshot_entry['error']}") + return deserialize_object_unsafe(snapshot_entry["value_encoded"]) + + response = self.query_setup_raw(var_to_query, query_timeout=query_timeout) + match response["status"]: + case QueryStatusCode.NOT_FOUND: + raise NameError(f"Query for setup variable '{var_to_query}' failed") + case QueryStatusCode.NOT_SERIALIZABLE: + raise TypeError(f"Setup variable '{var_to_query}' could not be serialized as JSON: {response['error']}") + return from_json(response["value"]) # TODO add functions that let instructors use the student fixture # use the stuff pete set up here: https://github.com/reteps/pytest-autograder-prototype diff --git a/src/pytest_prairielearn_grader/json_utils.py b/src/pytest_prairielearn_grader/json_utils.py index 3d203b3..efd4078 100644 --- a/src/pytest_prairielearn_grader/json_utils.py +++ b/src/pytest_prairielearn_grader/json_utils.py @@ -62,6 +62,41 @@ def to_json(v: Any) -> Any: return v +def is_json_transportable(v: Any) -> bool: + """ + Return True if to_json and from_json can carry this value without changing what it is. + + This is deliberately strict: tuples, sets, non-string dict keys and subclasses would change + type or fail in JSON, so they return False. to_json only converts NumPy and pandas values + at the top level, so they return False when nested. Keep this in step with to_json. + """ + module = type(v).__module__.split(".")[0] + if module == "numpy": + import numpy as np + + if isinstance(v, np.ndarray): + return v.dtype.kind in "biufc" + return isinstance(v, np.number | np.bool_) + if module == "pandas": + import pandas as pd + + return isinstance(v, pd.DataFrame) + try: + return _is_plain_json(v) + except RecursionError: + return False + + +def _is_plain_json(v: Any) -> bool: + if type(v) in (type(None), bool, int, float, str): + return True + if type(v) is list: + return all(_is_plain_json(item) for item in v) + if type(v) is dict: + return all(type(key) is str and _is_plain_json(item) for key, item in v.items()) + return False + + def _has_value_fields(v, fields: list[str]) -> bool: """Return True if all fields in the '_value' dictionary are present.""" return "_value" in v and isinstance(v["_value"], dict) and all(field in v["_value"] for field in fields) diff --git a/src/pytest_prairielearn_grader/utils.py b/src/pytest_prairielearn_grader/utils.py index b096d49..597b983 100644 --- a/src/pytest_prairielearn_grader/utils.py +++ b/src/pytest_prairielearn_grader/utils.py @@ -109,8 +109,28 @@ class SetupQueryRequest(TypedDict): class SetupQueryResponse(TypedDict): + # Student code may have run by the time this is sent, so it's JSON only. + status: QueryStatusCode + value: Any + error: str | None + + +class SetupSnapshotEntry(TypedDict): status: QueryStatusCode value_encoded: str + error: str | None + + +class SetupSnapshotMessage(TypedDict): + # Sent once per sandbox, after setup code and before student code, so its dill + # payloads hold nothing from the student. Nothing sent later may be deserialized with dill. + # Holds only the setup variables that JSON can't represent. + message_type: Literal["setup_snapshot"] + variables: dict[str, SetupSnapshotEntry] + + +class SetupSnapshotAck(TypedDict): + message_type: Literal["setup_snapshot_ack"] # Variable query dict types @@ -187,16 +207,20 @@ class ProcessStartResponse(TypedDict): execution_traceback: str -def serialize_object_unsafe(obj: object) -> str: +def serialize_object_unsafe(obj: object, *, recurse: bool = False) -> str: """ Serializes an arbitrary Python object to a JSON string. The object is first serialized using dill, then base64 encoded. + Args: + obj: The object to serialize. + recurse: Include only the globals a function refers to, rather than all of them. + Returns: A JSON string representing the serialized object. """ # 1. Serialize the object using dill - dilled_bytes = dill.dumps(obj) + dilled_bytes = dill.dumps(obj, recurse=recurse) # 2. Base64 encode the byte stream base64_encoded_bytes = base64.b64encode(dilled_bytes) diff --git a/tests/scenario_root/test_query_setup_snapshot/__init__.py b/tests/scenario_root/test_query_setup_snapshot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_query_setup_snapshot/data.json b/tests/scenario_root/test_query_setup_snapshot/data.json new file mode 100644 index 0000000..9883336 --- /dev/null +++ b/tests/scenario_root/test_query_setup_snapshot/data.json @@ -0,0 +1,12 @@ +{ + "params": { + "import_blacklist": [], + "names_for_user": [ + { + "name": "shared", + "type": "list", + "description": "" + } + ] + } +} diff --git a/tests/scenario_root/test_query_setup_snapshot/expected_outcome.json b/tests/scenario_root/test_query_setup_snapshot/expected_outcome.json new file mode 100644 index 0000000..b202265 --- /dev/null +++ b/tests/scenario_root/test_query_setup_snapshot/expected_outcome.json @@ -0,0 +1,71 @@ +{ + "expected_data_object": { + "score": 1.0, + "tests": [ + { + "max_points": 1, + "name": "json_values", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_json_values_sent_on_demand[student_code]" + }, + { + "max_points": 1, + "name": "dill_values", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_values_json_would_change_keep_their_types[student_code]" + }, + { + "max_points": 1, + "name": "functions", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_setup_functions_ignore_student_changes[student_code]" + }, + { + "max_points": 1, + "name": "function_globals", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_setup_function_globals_exclude_student_values[student_code]" + }, + { + "max_points": 1, + "name": "forged_json", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_forged_json_response_cannot_run_code[student_code]" + }, + { + "max_points": 1, + "name": "replaced_value", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_value_replaced_with_object_has_clear_error[student_code]" + }, + { + "max_points": 1, + "name": "unserializable", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_unserializable_setup_value_has_clear_error[student_code]" + }, + { + "max_points": 1, + "name": "not_found", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_query_setup_snapshot.py::test_missing_setup_value[student_code]" + } + ] + } +} diff --git a/tests/scenario_root/test_query_setup_snapshot/scenario.py b/tests/scenario_root/test_query_setup_snapshot/scenario.py new file mode 100644 index 0000000..0fae9c1 --- /dev/null +++ b/tests/scenario_root/test_query_setup_snapshot/scenario.py @@ -0,0 +1,60 @@ +import builtins + +import numpy as np +import pytest + +from pytest_prairielearn_grader.fixture import StudentFixture + + +def assert_grader_not_compromised() -> None: + assert not hasattr(builtins, "grader_compromised") + + +@pytest.mark.grading_data(name="json_values", points=1) +def test_json_values_sent_on_demand(sandbox: StudentFixture) -> None: + assert sandbox.query_setup("value") == 42 + np.testing.assert_array_equal(sandbox.query_setup("array_value"), np.array([1.0, 2.0])) + assert len(sandbox.query_setup("large_value", query_timeout=10.0)) == 200000 + + +@pytest.mark.grading_data(name="dill_values", points=1) +def test_values_json_would_change_keep_their_types(sandbox: StudentFixture) -> None: + assert sandbox.query_setup("tuple_value") == (1, 2) + assert sandbox.query_setup("int_keys_value") == {1: "one"} + + +@pytest.mark.grading_data(name="functions", points=1) +def test_setup_functions_ignore_student_changes(sandbox: StudentFixture) -> None: + assert sandbox.query_setup("add_twenty")(10) == 30 + assert_grader_not_compromised() + + +@pytest.mark.grading_data(name="function_globals", points=1) +def test_setup_function_globals_exclude_student_values(sandbox: StudentFixture) -> None: + assert sandbox.query("shared") == "from student" + assert sandbox.query_setup("read_shared")() == [1, 2, 3] + + +@pytest.mark.grading_data(name="forged_json", points=1) +def test_forged_json_response_cannot_run_code(sandbox: StudentFixture) -> None: + assert isinstance(sandbox.query_setup("forged_value"), str) + assert_grader_not_compromised() + + +@pytest.mark.grading_data(name="replaced_value", points=1) +def test_value_replaced_with_object_has_clear_error(sandbox: StudentFixture) -> None: + with pytest.raises(TypeError, match="Setup variable 'replaced_value' could not be serialized as JSON"): + sandbox.query_setup("replaced_value") + assert_grader_not_compromised() + + +@pytest.mark.grading_data(name="unserializable", points=1) +def test_unserializable_setup_value_has_clear_error(sandbox: StudentFixture) -> None: + with pytest.raises(TypeError, match="Setup variable 'unserializable' could not be serialized with dill"): + sandbox.query_setup("unserializable") + + +@pytest.mark.grading_data(name="not_found", points=1) +def test_missing_setup_value(sandbox: StudentFixture) -> None: + with pytest.raises(NameError, match="Query for setup variable 'missing' failed"): + sandbox.query_setup("missing") diff --git a/tests/scenario_root/test_query_setup_snapshot/setup_code.py b/tests/scenario_root/test_query_setup_snapshot/setup_code.py new file mode 100644 index 0000000..f1a32f6 --- /dev/null +++ b/tests/scenario_root/test_query_setup_snapshot/setup_code.py @@ -0,0 +1,19 @@ +import numpy as np + +value = 42 +array_value = np.array([1.0, 2.0]) +large_value = list(range(200000)) +tuple_value = (1, 2) +int_keys_value = {1: "one"} +shared = [1, 2, 3] +replaced_value = 1 +forged_value = 2 +unserializable = (i for i in range(3)) + + +def read_shared(): + return shared + + +def add_twenty(x): + return x + 20 diff --git a/tests/scenario_root/test_query_setup_snapshot/student_code.py b/tests/scenario_root/test_query_setup_snapshot/student_code.py new file mode 100644 index 0000000..2569314 --- /dev/null +++ b/tests/scenario_root/test_query_setup_snapshot/student_code.py @@ -0,0 +1,32 @@ +import base64 +import builtins +import gc +import pickle +import sys + + +class Exploit: + def __reduce__(self): + return (builtins.exec, ("import builtins; builtins.grader_compromised = True",)) + + +payload = base64.b64encode(pickle.dumps(Exploit())).decode() +# The sandbox runs this module as a spawned child, so the runner is imported under its package name +# rather than as __main__. Reach it the way a real attacker on this runtime would. +runner = sys.modules["pytest_prairielearn_grader._student_code_runner"] +real_to_json = runner.to_json + +# Forge whatever the runner would send if it serialized setup values on request. +runner.serialize_object_unsafe = lambda *args, **kwargs: payload +sys.modules["pytest_prairielearn_grader.utils"].serialize_object_unsafe = lambda *args, **kwargs: payload +runner.to_json = lambda v: payload if isinstance(v, int) and v == 999 else real_to_json(v) + +# Find the setup namespace and replace its values. +for obj in gc.get_objects(): + if isinstance(obj, dict) and "add_twenty" in obj and "forged_value" in obj: + obj["add_twenty"] = Exploit() + obj["replaced_value"] = Exploit() + obj["forged_value"] = 999 + +# Rebind a global that setup functions look up when they're called. +shared = "from student" diff --git a/tests/test_setup_query.py b/tests/test_setup_query.py new file mode 100644 index 0000000..8623d37 --- /dev/null +++ b/tests/test_setup_query.py @@ -0,0 +1,85 @@ +from typing import Any + +import numpy as np +import pandas as pd +import pytest + +from pytest_prairielearn_grader._student_code_runner import encode_setup_variables +from pytest_prairielearn_grader.json_utils import from_json +from pytest_prairielearn_grader.json_utils import is_json_transportable +from pytest_prairielearn_grader.json_utils import to_json +from pytest_prairielearn_grader.utils import QueryStatusCode +from pytest_prairielearn_grader.utils import deserialize_object_unsafe + + +@pytest.mark.parametrize( + "value", + [None, True, 1, 1.5, "text", [1, [2]], {"a": {"b": [1]}}, np.array([1, 2]), np.array([1 + 2j]), np.float64(1.5), np.bool_(True)], +) +def test_json_transportable_values_round_trip(value: Any) -> None: + assert is_json_transportable(value) + np.testing.assert_equal(from_json(to_json(value)), value) + + +def test_dataframe_is_json_transportable() -> None: + value = pd.DataFrame({"a": [1, 2]}) + assert is_json_transportable(value) + pd.testing.assert_frame_equal(from_json(to_json(value)), value) + + +class ListSubclass(list): + pass + + +@pytest.mark.parametrize( + "value", + [ + (1, 2), + {1, 2}, + {1: "one"}, + [(1, 2)], + {"a": (1, 2)}, + [np.array([1])], + ListSubclass(), + len, + np.array(["a"]), + np.array([object()]), + pd.Series([1]), + ], +) +def test_values_json_would_change_are_not_transportable(value: Any) -> None: + assert not is_json_transportable(value) + + +def test_self_referencing_list_is_not_transportable() -> None: + value: list[Any] = [] + value.append(value) + assert not is_json_transportable(value) + + +def test_snapshot_leaves_out_json_values() -> None: + def function() -> int: + return 1 + + encoded = encode_setup_variables({"large": list(range(1000)), "params": {"a": 1}, "function": function}) + + assert set(encoded) == {"function"} + assert encoded["function"]["status"] == QueryStatusCode.SUCCESS + + +def test_snapshot_records_values_dill_cannot_encode() -> None: + encoded = encode_setup_variables({"generator": (i for i in range(3))}) + + assert encoded["generator"]["status"] == QueryStatusCode.NOT_SERIALIZABLE + assert encoded["generator"]["error"] + + +def test_function_snapshot_leaves_out_unused_globals() -> None: + namespace: dict[str, Any] = {"unused": "x" * 1_000_000, "used": 1} + exec("def function():\n return used\n", namespace) # noqa: S102 + + encoded = encode_setup_variables({"function": namespace["function"]}) + + assert len(encoded["function"]["value_encoded"]) < 10_000 + function = deserialize_object_unsafe(encoded["function"]["value_encoded"]) + assert function() == 1 # type: ignore[operator]