From 1f00a5969de2d906d548f978b14715c185b71a86 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Tue, 17 Nov 2026 15:18:02 -0500 Subject: [PATCH] Let setup code import server params from pytest_prairielearn_grader.server_data Co-Authored-By: Claude Opus 5.5 --- converting.md | 5 +- quick_start.md | 21 +++++++-- .../_student_code_runner.py | 46 +++++++++++++++++++ src/pytest_prairielearn_grader/server_data.py | 28 +++++++++++ .../test_server_data/__init__.py | 0 .../scenario_root/test_server_data/data.json | 25 ++++++++++ .../test_server_data/expected_outcome.json | 43 +++++++++++++++++ .../test_server_data/scenario.py | 12 +++++ .../test_server_data/setup_code.py | 7 +++ .../test_server_data/student_code.py | 1 + .../test_server_data/student_code_import.py | 3 ++ .../student_code_import_from_package.py | 3 ++ .../student_code_import_package.py | 3 ++ .../test_server_data_workspace/__init__.py | 0 .../test_server_data_workspace/data.json | 5 ++ .../expected_outcome.json | 25 ++++++++++ .../test_server_data_workspace/scenario.py | 16 +++++++ .../test_server_data_workspace/setup_code.py | 3 ++ .../student/reader.py | 4 ++ tests/test_server_data_module.py | 8 ++++ 20 files changed, 251 insertions(+), 7 deletions(-) create mode 100644 src/pytest_prairielearn_grader/server_data.py create mode 100644 tests/scenario_root/test_server_data/__init__.py create mode 100644 tests/scenario_root/test_server_data/data.json create mode 100644 tests/scenario_root/test_server_data/expected_outcome.json create mode 100644 tests/scenario_root/test_server_data/scenario.py create mode 100644 tests/scenario_root/test_server_data/setup_code.py create mode 100644 tests/scenario_root/test_server_data/student_code.py create mode 100644 tests/scenario_root/test_server_data/student_code_import.py create mode 100644 tests/scenario_root/test_server_data/student_code_import_from_package.py create mode 100644 tests/scenario_root/test_server_data/student_code_import_package.py create mode 100644 tests/scenario_root/test_server_data_workspace/__init__.py create mode 100644 tests/scenario_root/test_server_data_workspace/data.json create mode 100644 tests/scenario_root/test_server_data_workspace/expected_outcome.json create mode 100644 tests/scenario_root/test_server_data_workspace/scenario.py create mode 100644 tests/scenario_root/test_server_data_workspace/setup_code.py create mode 100644 tests/scenario_root/test_server_data_workspace/student/reader.py create mode 100644 tests/test_server_data_module.py diff --git a/converting.md b/converting.md index 873217d..7343d58 100644 --- a/converting.md +++ b/converting.md @@ -271,9 +271,10 @@ First, move what you can out of the setup code, as described in [Where code belo If nothing is left, delete `setup_code.py`. Otherwise, update what's left. Setup code runs inside the sandbox, so some things that worked in the legacy grader don't: -- **Parameters.** `data` isn't defined. Use `__data_params`, which is `data["params"]`. Values that +- **Parameters.** `data` isn't defined. Import `params`, which is `data["params"]`, with + `from pytest_prairielearn_grader.server_data import from_server_json, params`. Values that `server.py` stored with `pl.to_json` arrive still encoded: decode them with - `__from_server_json(__data_params["A"])`. The `prairielearn` package isn't available. + `from_server_json(params["A"])`. The `prairielearn` package isn't available. - **Restrictions.** Setup code runs with the same builtins and import rules as student code. For example, it can't call `open` or import `os`. To read a data file, list it in `data_files` and use a library function such as `np.loadtxt`. diff --git a/quick_start.md b/quick_start.md index f0270ea..be4f88c 100644 --- a/quick_start.md +++ b/quick_start.md @@ -193,7 +193,7 @@ in the first place in this table that works: | Inputs to the student's functions, including random ones | The test, passed as arguments. Use `@pytest.mark.parametrize` for several inputs. | | Variables that student code reads and that `server.py` randomizes | `data["params"]` in `server.py`, listed in `names_for_user`. Tests read them from the `data_json` fixture. This works for plain JSON values, not ones stored with `pl.to_json`. | | Fixed variables that student code reads | `starting_vars` in the `ConfigObject`, listed in `names_for_user` | -| Variables that student code reads and that need Python to build | `setup_code.py`, such as numpy arrays or values decoded with `__from_server_json` | +| Variables that student code reads and that need Python to build | `setup_code.py`, such as numpy arrays or values decoded with `from_server_json` | | Blocking a library function | [`import_blacklist`](#blocking-functions), such as `["numpy.linalg.inv"]` | | Imports or definitions that the student's code relies on | The starter code, so that the student sees them. Use `helper_modules` for a module the student imports. Use `leading_code.py` only when neither works. | @@ -896,9 +896,20 @@ With `names_for_user` set to `A`, `b` and `n`, student code gets those three var students from calling a library function such as `la.inv`, use [`import_blacklist`](#blocking-functions) rather than setup code. -- **Parameters.** `__data_params` holds `data["params"]`. Values that `server.py` stored with - `pl.to_json` are still encoded: decode them with `__from_server_json`, for example - `A = __from_server_json(__data_params["A"])`. The `prairielearn` package isn't available. +- **Parameters.** Import `data["params"]` from `pytest_prairielearn_grader.server_data`. Values + that `server.py` stored with `pl.to_json` are still encoded: decode them with `from_server_json`. + The `prairielearn` package isn't available. + + ```python + from pytest_prairielearn_grader.server_data import from_server_json, params + + np.random.seed(params["seed"]) + A = from_server_json(params["A"]) + ``` + + Only setup code can import this module, whatever the import rules. Older setup code can still use + `__data_params` and `__from_server_json`, which hold the same values. + - **Restrictions.** Setup code has the same builtins and import rules as student code. For example, it can't call `open` or import `os`. To read a data file, list it in [`data_files`](#data-files). - **Fresh values.** With the `sandbox` fixture, setup code runs again for each test, so random @@ -1415,7 +1426,7 @@ def generate(data): These parameters are automatically passed to the autograder via `/grade/data/data.json` and enforced during student code execution. -In the grader images, `run.sh` passes this path to pytest with `--data-json-path` and makes the file readable only by root, so student code can't open it. The file can contain `correct_answers`. Without that option, for example when running locally, the fixture reads `data.json` next to your test file. The sandbox still receives `params` through `__data_params`, so don't rely on values in `params` staying hidden from student code. +In the grader images, `run.sh` passes this path to pytest with `--data-json-path` and makes the file readable only by root, so student code can't open it. The file can contain `correct_answers`. Without that option, for example when running locally, the fixture reads `data.json` next to your test file. The sandbox still receives `params` for setup code, so don't rely on values in `params` staying hidden from student code. #### Testing Import/Builtin Restrictions Locally diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 08239cd..53cb6e5 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -239,6 +239,37 @@ def blocked(*args: Any, **kwargs: Any) -> Any: return blocked +SERVER_DATA_PACKAGE = "pytest_prairielearn_grader" +SERVER_DATA_MODULE = f"{SERVER_DATA_PACKAGE}.server_data" + + +class ServerDataImport: + """ + Gives setup code its own copy of the server_data module, holding this sandbox's params. + + The copy is kept out of sys.modules, and it can only be imported until the setup code + finishes, so student code can't read the params. + """ + + def __init__(self, params: dict[str, Any]) -> None: + self.module = types.ModuleType(SERVER_DATA_MODULE) + self.module.params = params # type: ignore[attr-defined] + self.module.from_server_json = from_server_json # type: ignore[attr-defined] + # `import pytest_prairielearn_grader.server_data` and `from pytest_prairielearn_grader + # import server_data` both get the top-level package back from __import__. + self.package = types.ModuleType(SERVER_DATA_PACKAGE) + self.package.server_data = self.module # type: ignore[attr-defined] + self.available = True + + def matches(self, name: str, fromlist: Sequence[str]) -> bool: + return name == SERVER_DATA_MODULE or (name == SERVER_DATA_PACKAGE and "server_data" in fromlist) + + def load(self, name: str, fromlist: Sequence[str]) -> types.ModuleType: + if not self.available: + raise ImportError(f"Module '{SERVER_DATA_MODULE}' can only be imported by setup code.") + return self.module if name == SERVER_DATA_MODULE and fromlist else self.package + + class _HelperModules: """ Loads the modules named by the helper_modules setting from the source sent by the grader. @@ -280,6 +311,7 @@ def get_custom_importer( import_blacklist: list[str] | None, workspace_dir: str | None = None, helper_modules: _HelperModules | None = None, + server_data: ServerDataImport | None = None, ) -> ImportFunction: """ Returns a custom import function that restricts imports based on the provided whitelist and @@ -293,6 +325,9 @@ def get_custom_importer( Modules in *helper_modules* can be imported without being on the whitelist, and they take priority over workspace files with the same name. The grader rejects any that are blacklisted. + + *server_data* provides the server_data module to setup code, whatever the whitelist and + blacklist say. """ original_import = _REAL_IMPORT @@ -328,6 +363,9 @@ def custom_import( if isinstance(importer_file, str) and not importer_file.startswith(_ws_str + os.sep): return original_import(name, globals, locals, fl, level) + if server_data is not None and server_data.matches(name, fl): + return server_data.load(name, fl) + if helper_modules is not None and name in helper_modules: return helper_modules.load(name) @@ -432,10 +470,12 @@ async def student_code_runner( student_code_vars["__builtins__"] = get_builtins(builtin_whitelist) student_code_vars["__builtins__"]["__name__"] = "__main__" # Set __name__ to "__main__" to mimic the main module + server_data = ServerDataImport(local_vars.get("__data_params", {})) student_code_vars["__builtins__"]["__import__"] = get_custom_importer( import_whitelist, import_blacklist, helper_modules=None if helper_modules is None else _HelperModules(helper_modules, student_code_vars["__builtins__"]), + server_data=server_data, ) _block_blacklisted_functions(import_blacklist) @@ -459,6 +499,8 @@ async def student_code_runner( # TODO need to create a different message for setup code errors. This should result # in a different error message reported from the test case. + server_data.available = False + # Only inject variables that are explicitly listed in names_for_user_list # This prevents accidental variable leaking from setup_code or starting_vars if names_for_user_list is not None: @@ -562,11 +604,13 @@ async def workspace_runner( student_code_vars: dict[str, Any] = {} student_code_vars["__builtins__"] = get_builtins(builtin_whitelist) student_code_vars["__builtins__"]["__name__"] = "__main__" + server_data = ServerDataImport(local_vars.get("__data_params", {})) workspace_importer = get_custom_importer( import_whitelist, import_blacklist, workspace_dir, helper_modules=None if helper_modules is None else _HelperModules(helper_modules, student_code_vars["__builtins__"]), + server_data=server_data, ) _block_blacklisted_functions(import_blacklist) student_code_vars["__builtins__"]["__import__"] = workspace_importer @@ -594,6 +638,8 @@ async def workspace_runner( except Exception as e: execution_error = e + server_data.available = False + if names_for_user_list is not None: for var_name in names_for_user_list: if var_name in local_vars: diff --git a/src/pytest_prairielearn_grader/server_data.py b/src/pytest_prairielearn_grader/server_data.py new file mode 100644 index 0000000..9987550 --- /dev/null +++ b/src/pytest_prairielearn_grader/server_data.py @@ -0,0 +1,28 @@ +""" +Values that the question's ``server.py`` prepared, for setup code to import. + +In setup code:: + + from pytest_prairielearn_grader.server_data import from_server_json, params + + np.random.seed(params["seed"]) + A = from_server_json(params["A"]) + +The sandbox gives setup code its own copy of this module, with ``params`` set to +``data["params"]``. Student code can't import it, and outside the sandbox ``params`` isn't set: in +tests, use the ``data_json`` fixture instead. +""" + +from typing import Any + +from pytest_prairielearn_grader.json_utils import from_server_json + +__all__ = ["from_server_json", "params"] + +params: dict[str, Any] + + +def __getattr__(name: str) -> Any: + if name == "params": + raise RuntimeError("pytest_prairielearn_grader.server_data.params is only set in setup code. In tests, use the data_json fixture.") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/tests/scenario_root/test_server_data/__init__.py b/tests/scenario_root/test_server_data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_server_data/data.json b/tests/scenario_root/test_server_data/data.json new file mode 100644 index 0000000..5d26f76 --- /dev/null +++ b/tests/scenario_root/test_server_data/data.json @@ -0,0 +1,25 @@ +{ + "params": { + "seed": 5, + "A": { + "_type": "ndarray", + "_value": [ + [1.0, 2.0], + [3.0, 4.0] + ], + "_dtype": "float64" + }, + "names_for_user": [ + { + "name": "x", + "type": "int", + "description": "" + }, + { + "name": "A", + "type": "ndarray", + "description": "" + } + ] + } +} diff --git a/tests/scenario_root/test_server_data/expected_outcome.json b/tests/scenario_root/test_server_data/expected_outcome.json new file mode 100644 index 0000000..68ab441 --- /dev/null +++ b/tests/scenario_root/test_server_data/expected_outcome.json @@ -0,0 +1,43 @@ +{ + "expected_data_object": { + "score": 0.25, + "tests": [ + { + "max_points": 1, + "name": "total", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_server_data.py::test_total[student_code]", + "message": "" + }, + { + "max_points": 1, + "name": "total", + "outcome": "error", + "points": 0.0, + "points_frac": 0.0, + "test_id": "test_server_data.py::test_total[student_code_import]", + "message": "Module 'pytest_prairielearn_grader.server_data' can only be imported by setup code." + }, + { + "max_points": 1, + "name": "total", + "outcome": "error", + "points": 0.0, + "points_frac": 0.0, + "test_id": "test_server_data.py::test_total[student_code_import_from_package]", + "message": "Module 'pytest_prairielearn_grader.server_data' can only be imported by setup code." + }, + { + "max_points": 1, + "name": "total", + "outcome": "error", + "points": 0.0, + "points_frac": 0.0, + "test_id": "test_server_data.py::test_total[student_code_import_package]", + "message": "Module 'pytest_prairielearn_grader.server_data' can only be imported by setup code." + } + ] + } +} diff --git a/tests/scenario_root/test_server_data/scenario.py b/tests/scenario_root/test_server_data/scenario.py new file mode 100644 index 0000000..3cec288 --- /dev/null +++ b/tests/scenario_root/test_server_data/scenario.py @@ -0,0 +1,12 @@ +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +# The whitelist doesn't need to list the server_data module. +autograder_config = ConfigObject(import_whitelist=["math"]) + + +@pytest.mark.grading_data(name="total", points=1) +def test_total(sandbox: StudentFixture) -> None: + assert sandbox.query("total") == 25 diff --git a/tests/scenario_root/test_server_data/setup_code.py b/tests/scenario_root/test_server_data/setup_code.py new file mode 100644 index 0000000..78f5b1a --- /dev/null +++ b/tests/scenario_root/test_server_data/setup_code.py @@ -0,0 +1,7 @@ +import pytest_prairielearn_grader.server_data +from pytest_prairielearn_grader import server_data +from pytest_prairielearn_grader.server_data import from_server_json +from pytest_prairielearn_grader.server_data import params + +x = params["seed"] + server_data.params["seed"] + pytest_prairielearn_grader.server_data.params["seed"] +A = from_server_json(params["A"]) diff --git a/tests/scenario_root/test_server_data/student_code.py b/tests/scenario_root/test_server_data/student_code.py new file mode 100644 index 0000000..8bc5c3c --- /dev/null +++ b/tests/scenario_root/test_server_data/student_code.py @@ -0,0 +1 @@ +total = x + int(A[0, 0] + A[0, 1] + A[1, 0] + A[1, 1]) diff --git a/tests/scenario_root/test_server_data/student_code_import.py b/tests/scenario_root/test_server_data/student_code_import.py new file mode 100644 index 0000000..eca3ab7 --- /dev/null +++ b/tests/scenario_root/test_server_data/student_code_import.py @@ -0,0 +1,3 @@ +from pytest_prairielearn_grader.server_data import params + +total = params["seed"] diff --git a/tests/scenario_root/test_server_data/student_code_import_from_package.py b/tests/scenario_root/test_server_data/student_code_import_from_package.py new file mode 100644 index 0000000..3aebf90 --- /dev/null +++ b/tests/scenario_root/test_server_data/student_code_import_from_package.py @@ -0,0 +1,3 @@ +from pytest_prairielearn_grader import server_data + +total = server_data.params["seed"] diff --git a/tests/scenario_root/test_server_data/student_code_import_package.py b/tests/scenario_root/test_server_data/student_code_import_package.py new file mode 100644 index 0000000..e74cba0 --- /dev/null +++ b/tests/scenario_root/test_server_data/student_code_import_package.py @@ -0,0 +1,3 @@ +import pytest_prairielearn_grader.server_data + +total = pytest_prairielearn_grader.server_data.params["seed"] diff --git a/tests/scenario_root/test_server_data_workspace/__init__.py b/tests/scenario_root/test_server_data_workspace/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_server_data_workspace/data.json b/tests/scenario_root/test_server_data_workspace/data.json new file mode 100644 index 0000000..cd400d6 --- /dev/null +++ b/tests/scenario_root/test_server_data_workspace/data.json @@ -0,0 +1,5 @@ +{ + "params": { + "seed": 5 + } +} diff --git a/tests/scenario_root/test_server_data_workspace/expected_outcome.json b/tests/scenario_root/test_server_data_workspace/expected_outcome.json new file mode 100644 index 0000000..cbb03f4 --- /dev/null +++ b/tests/scenario_root/test_server_data_workspace/expected_outcome.json @@ -0,0 +1,25 @@ +{ + "expected_data_object": { + "score": 0.5, + "tests": [ + { + "max_points": 1, + "message": "", + "name": "setup code", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_server_data_workspace.py::test_setup_code" + }, + { + "max_points": 1, + "message": "reader.read() raised ImportError: Module 'pytest_prairielearn_grader.server_data' can only be imported by setup code.", + "name": "student code", + "outcome": "failed", + "points": 0.0, + "points_frac": 0.0, + "test_id": "test_server_data_workspace.py::test_student_code" + } + ] + } +} diff --git a/tests/scenario_root/test_server_data_workspace/scenario.py b/tests/scenario_root/test_server_data_workspace/scenario.py new file mode 100644 index 0000000..5e7b470 --- /dev/null +++ b/tests/scenario_root/test_server_data_workspace/scenario.py @@ -0,0 +1,16 @@ +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import WorkspaceFixture + +autograder_config = ConfigObject(workspace_mode=True, names_for_user=["x"], sandbox_timeout=5) + + +@pytest.mark.grading_data(name="setup code", points=1) +def test_setup_code(workspace_sandbox: WorkspaceFixture) -> None: + assert workspace_sandbox.query("x") == 5 + + +@pytest.mark.grading_data(name="student code", points=1) +def test_student_code(workspace_sandbox: WorkspaceFixture) -> None: + workspace_sandbox.query_function("reader.read") diff --git a/tests/scenario_root/test_server_data_workspace/setup_code.py b/tests/scenario_root/test_server_data_workspace/setup_code.py new file mode 100644 index 0000000..0d5d41b --- /dev/null +++ b/tests/scenario_root/test_server_data_workspace/setup_code.py @@ -0,0 +1,3 @@ +from pytest_prairielearn_grader.server_data import params + +x = params["seed"] diff --git a/tests/scenario_root/test_server_data_workspace/student/reader.py b/tests/scenario_root/test_server_data_workspace/student/reader.py new file mode 100644 index 0000000..696ab90 --- /dev/null +++ b/tests/scenario_root/test_server_data_workspace/student/reader.py @@ -0,0 +1,4 @@ +def read(): + from pytest_prairielearn_grader.server_data import params + + return params["seed"] diff --git a/tests/test_server_data_module.py b/tests/test_server_data_module.py new file mode 100644 index 0000000..1dc21b6 --- /dev/null +++ b/tests/test_server_data_module.py @@ -0,0 +1,8 @@ +import pytest + +from pytest_prairielearn_grader import server_data + + +def test_params_are_only_set_in_setup_code() -> None: + with pytest.raises(RuntimeError, match="use the data_json fixture"): + _ = server_data.params