From 16c8007e39ef96bfe9e5ce3158aedd24adac0b26 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Fri, 6 Nov 2026 13:36:51 -0500 Subject: [PATCH 1/5] Let questions name helper modules that student code can import The tests folder isn't on the sandbox's import path, and import_whitelist only permits a name, so helper files there couldn't be imported. The new helper_modules setting names modules in that folder. The grader sends their source to the sandbox, so the folder stays off the import path and other files in it, such as reference answers, still can't be imported. Co-Authored-By: Claude Opus 5.5 --- .github/copilot-instructions.md | 4 + pyproject.toml | 1 + quick_start.md | 34 ++++++ .../_student_code_runner.py | 50 +++++++- src/pytest_prairielearn_grader/config.py | 34 ++++++ src/pytest_prairielearn_grader/fixture.py | 3 + src/pytest_prairielearn_grader/plugin.py | 35 ++++++ src/pytest_prairielearn_grader/utils.py | 1 + .../test_helper_modules/__init__.py | 0 .../scenario_root/test_helper_modules/ans.py | 1 + .../test_helper_modules/expected_outcome.json | 110 ++++++++++++++++++ .../test_helper_modules/helper_function.py | 5 + .../test_helper_modules/leading_code.py | 3 + .../test_helper_modules/scenario.py | 60 ++++++++++ .../test_helper_modules/second_helper.py | 11 ++ .../test_helper_modules/setup_code.py | 3 + .../test_helper_modules/student_code.py | 58 +++++++++ .../__init__.py | 0 .../expected_outcome.json | 13 +++ .../helper_function.py | 5 + .../scenario.py | 13 +++ .../student_code.py | 1 + .../test_helper_modules_missing/__init__.py | 0 .../expected_outcome.json | 13 +++ .../test_helper_modules_missing/scenario.py | 13 +++ .../student_code.py | 1 + .../test_helper_modules_notebook/__init__.py | 0 .../expected_outcome.json | 20 ++++ .../helper_function.py | 5 + .../test_helper_modules_notebook/scenario.py | 17 +++ .../student_code.ipynb | 41 +++++++ .../__init__.py | 0 .../test_helper_modules_restricted/ans.py | 1 + .../blocked_import.py | 3 + .../expected_outcome.json | 65 +++++++++++ .../helper_function.py | 5 + .../scenario.py | 53 +++++++++ .../student_code.py | 33 ++++++ .../uses_open.py | 2 + .../__init__.py | 0 .../expected_outcome.json | 13 +++ .../helper_function.py | 1 + .../scenario.py | 13 +++ .../student_code.py | 1 + tests/test_autograder_scenarios.py | 2 + tests/test_config.py | 41 +++++++ 46 files changed, 787 insertions(+), 1 deletion(-) create mode 100644 tests/scenario_root/test_helper_modules/__init__.py create mode 100644 tests/scenario_root/test_helper_modules/ans.py create mode 100644 tests/scenario_root/test_helper_modules/expected_outcome.json create mode 100644 tests/scenario_root/test_helper_modules/helper_function.py create mode 100644 tests/scenario_root/test_helper_modules/leading_code.py create mode 100644 tests/scenario_root/test_helper_modules/scenario.py create mode 100644 tests/scenario_root/test_helper_modules/second_helper.py create mode 100644 tests/scenario_root/test_helper_modules/setup_code.py create mode 100644 tests/scenario_root/test_helper_modules/student_code.py create mode 100644 tests/scenario_root/test_helper_modules_blacklisted/__init__.py create mode 100644 tests/scenario_root/test_helper_modules_blacklisted/expected_outcome.json create mode 100644 tests/scenario_root/test_helper_modules_blacklisted/helper_function.py create mode 100644 tests/scenario_root/test_helper_modules_blacklisted/scenario.py create mode 100644 tests/scenario_root/test_helper_modules_blacklisted/student_code.py create mode 100644 tests/scenario_root/test_helper_modules_missing/__init__.py create mode 100644 tests/scenario_root/test_helper_modules_missing/expected_outcome.json create mode 100644 tests/scenario_root/test_helper_modules_missing/scenario.py create mode 100644 tests/scenario_root/test_helper_modules_missing/student_code.py create mode 100644 tests/scenario_root/test_helper_modules_notebook/__init__.py create mode 100644 tests/scenario_root/test_helper_modules_notebook/expected_outcome.json create mode 100644 tests/scenario_root/test_helper_modules_notebook/helper_function.py create mode 100644 tests/scenario_root/test_helper_modules_notebook/scenario.py create mode 100644 tests/scenario_root/test_helper_modules_notebook/student_code.ipynb create mode 100644 tests/scenario_root/test_helper_modules_restricted/__init__.py create mode 100644 tests/scenario_root/test_helper_modules_restricted/ans.py create mode 100644 tests/scenario_root/test_helper_modules_restricted/blocked_import.py create mode 100644 tests/scenario_root/test_helper_modules_restricted/expected_outcome.json create mode 100644 tests/scenario_root/test_helper_modules_restricted/helper_function.py create mode 100644 tests/scenario_root/test_helper_modules_restricted/scenario.py create mode 100644 tests/scenario_root/test_helper_modules_restricted/student_code.py create mode 100644 tests/scenario_root/test_helper_modules_restricted/uses_open.py create mode 100644 tests/scenario_root/test_helper_modules_syntax_error/__init__.py create mode 100644 tests/scenario_root/test_helper_modules_syntax_error/expected_outcome.json create mode 100644 tests/scenario_root/test_helper_modules_syntax_error/helper_function.py create mode 100644 tests/scenario_root/test_helper_modules_syntax_error/scenario.py create mode 100644 tests/scenario_root/test_helper_modules_syntax_error/student_code.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 9e88357..9388b89 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -141,6 +141,10 @@ result = sandbox.query_function("student_func", arg1, arg2, timeout=5) # In setup_code.py, define variables; they're injected if in names_for_user ``` +`ConfigObject(helper_modules=["helper_function"])` lets setup, leading and student code import +`helper_function.py` from the tests folder, which isn't otherwise on the import path. Students can read +these modules, so they must not contain answers. It isn't supported in workspace mode. + ### 5. Serialization - Uses `dill` for complex Python objects (numpy arrays, pandas DataFrames, matplotlib plots) diff --git a/pyproject.toml b/pyproject.toml index bfa6e3d..cadd473 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,6 +107,7 @@ prairielearn-grader = "pytest_prairielearn_grader.plugin" exclude = [ "tests/scenario_root/.*(student_code.*|setup_code)\\.py", "tests/scenario_root/.*/student/.*\\.py", + "tests/scenario_root/test_helper_modules[^/]*/(helper_function|second_helper|leading_code)\\.py", ] [[tool.mypy.overrides]] diff --git a/quick_start.md b/quick_start.md index 594cbee..919730c 100644 --- a/quick_start.md +++ b/quick_start.md @@ -347,10 +347,44 @@ All parameters are optional with sensible defaults: - **`import_whitelist`** (list[str] | None): Allowed import modules (whitelist mode) - **`import_blacklist`** (list[str] | None): Blocked import modules (blacklist mode) - **`builtin_whitelist`** (list[str] | None): Allowed builtin functions +- **`helper_modules`** (list[str] | None): Modules from the tests folder that can be imported (see [Helper Modules](#helper-modules)) - **`names_for_user`** (list[str] | None): List of variable names to inject into student code - **`student_code_pattern`** (str, default="student_code\*.py"): Glob pattern for finding student files - **`starting_vars`** (dict[str, Any], default={}): Dictionary of variable values to provide for injection +#### Helper Modules + +The tests folder isn't on the import path in the sandbox, and `import_whitelist` only permits a +name. To let setup code, `leading_code.py` and student code import a file from that folder, name it +in `helper_modules`: + +```python +# tests/test_student.py, with the helper in tests/helper_function.py +autograder_config = ConfigObject(helper_modules=["helper_function"]) +``` + +```python +# tests/leading_code.py, or the student's code +import helper_function as hf +``` + +- **Names:** each entry is a plain module name, without dots, paths or `.py`. Packages aren't supported. +- **Everything else stays out:** other files in the folder, such as `test_student.py`, still can't be + imported, even if they're in `import_whitelist`. +- **Import restrictions:** a listed module doesn't need to be in `import_whitelist`. Listing one that + `import_blacklist` also blocks is a configuration error. +- **Inside a helper module:** it runs with the same builtins and import rules as student code. For + example, it can't use `sum` or import `numpy` unless those are allowed. +- **Configuration errors:** if a listed file is missing or has a syntax error, or the module is + blacklisted, the submission is ungradable. The students don't lose an attempt, and each test's + message says what's wrong. +- **Fixtures:** it works with `sandbox` and `module_sandbox`, for `.py` and `.ipynb` submissions. It + isn't supported in workspace mode, where modules in the student's folder can already be imported. + +> **Warning:** Students can read everything in a helper module, so it must not contain answers. This +> setting also doesn't make the rest of the tests folder private. `importlib` isn't blacklisted by +> default, so student code can load files in that folder by path. + **How variable injection works**: - Only variables listed in `names_for_user` are injected into the student code namespace diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 77b34ea..8de85c6 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -161,10 +161,47 @@ def student_function_temp() -> Any: return json.dumps(not_serializable_response) +class _HelperModules: + """ + Loads the modules named by the helper_modules setting from the source sent by the grader. + + They're kept out of sys.modules so that only the sandbox's importer can find them, and they + run with the given builtins, so importing one doesn't give student code any wider access. + """ + + def __init__(self, sources: dict[str, str], module_builtins: dict[str, Any]) -> None: + self._sources = sources + self._module_builtins = module_builtins + self._loaded: dict[str, types.ModuleType] = {} + + def __contains__(self, name: str) -> bool: + return name in self._sources + + def load(self, name: str) -> types.ModuleType: + if name in self._loaded: + return self._loaded[name] + + source = self._sources[name] + file_name = f"{name}.py" + populate_linecache(source, file_name) + + module = types.ModuleType(name) + module.__dict__["__builtins__"] = self._module_builtins + # Add it before running it so that helper modules can import each other, as in sys.modules. + self._loaded[name] = module + try: + exec(compile(source, file_name, "exec"), module.__dict__) + except Exception as e: + del self._loaded[name] + raise ImportError(f"Helper module '{name}' could not be loaded: {type(e).__name__}: {e}") from e + return module + + def get_custom_importer( import_whitelist: list[str] | None, import_blacklist: list[str] | None, workspace_dir: str | None = None, + helper_modules: _HelperModules | None = None, ) -> ImportFunction: """ Returns a custom import function that restricts imports based on the provided whitelist and @@ -174,6 +211,9 @@ def get_custom_importer( directory inside that directory is unconditionally allowed, so that student modules can import each other without instructors needing to maintain a manual whitelist of their own files. Relative imports (level > 0) are always permitted for the same reason. + + Modules in *helper_modules* can be imported without being on the whitelist, unless they're on + the blacklist. """ original_import = _REAL_IMPORT @@ -209,6 +249,8 @@ def custom_import( # Apply blacklist / whitelist to external imports. if import_blacklist is not None and name in import_blacklist: raise ImportError(f"Module '{name}' is blacklisted and cannot be imported.") + elif helper_modules is not None and name in helper_modules: + return helper_modules.load(name) elif ( (import_whitelist is not None and name in import_whitelist) or name.startswith("_") # Allow Python internal/private modules (_io, _abc, _collections, etc.) @@ -232,6 +274,7 @@ async def student_code_runner( starting_vars: dict[str, Any] | None, builtin_whitelist: list[str] | None, names_for_user_list: list[str] | None, + helper_modules: dict[str, str] | None, ) -> tuple[dict[str, Any], dict[str, Any], ProcessStartResponse]: stdout_capture = io.StringIO() stderr_capture = io.StringIO() @@ -244,7 +287,11 @@ async def student_code_runner( student_code_vars["__builtins__"] = get_builtins(builtin_whitelist) student_code_vars["__builtins__"]["__name__"] = "__main__" # Set __name__ to "__main__" to mimic the main module - student_code_vars["__builtins__"]["__import__"] = get_custom_importer(import_whitelist, import_blacklist) + student_code_vars["__builtins__"]["__import__"] = get_custom_importer( + import_whitelist, + import_blacklist, + helper_modules=None if helper_modules is None else _HelperModules(helper_modules, student_code_vars["__builtins__"]), + ) try: # First, execute the setup code if provided @@ -504,6 +551,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit starting_vars=starting_vars, builtin_whitelist=builtin_whitelist, names_for_user_list=names_for_user_list, + helper_modules=start_json_message["helper_modules"], ) writer.write((json.dumps(start_response) + os.linesep).encode()) diff --git a/src/pytest_prairielearn_grader/config.py b/src/pytest_prairielearn_grader/config.py index f79ffe6..ea00045 100644 --- a/src/pytest_prairielearn_grader/config.py +++ b/src/pytest_prairielearn_grader/config.py @@ -1,5 +1,6 @@ """Configuration dataclass for pytest-prairielearn-grader.""" +import keyword from dataclasses import dataclass from dataclasses import field from typing import Any @@ -64,6 +65,24 @@ def test_student_code(sandbox): Example: ["requests", "socket"] """ + helper_modules: list[str] | None = None + """Modules from the tests folder that setup code, leading code and student code can import. + + Each entry is a plain module name, so "helper_function" loads helper_function.py from + the folder that holds leading_code.py. Packages, dots, paths and the ".py" suffix aren't + accepted. Importing anything else in that folder still fails, because the folder isn't + added to the import path. + + Listed modules don't need to be in import_whitelist. Listing one that import_blacklist + also blocks is a configuration error. A helper module runs in the sandbox with the same + builtins and import rules as student code. + + Students can read everything in a helper module, so it must not contain answers. + Not supported in workspace mode, where the student's folder is on the import path. + + Example: ["helper_function"] + """ + builtin_whitelist: list[str] | None = None """List of allowed Python builtin functions that student code can use. @@ -231,6 +250,21 @@ def __post_init__(self) -> None: if not isinstance(module_name, str) or not module_name.strip(): raise ValueError(f"import_blacklist must contain non-empty strings, got: {module_name!r}") + # Validate helper_modules contains plain module names + if self.helper_modules is not None: + if not isinstance(self.helper_modules, list): + raise TypeError(f"helper_modules must be a list, got {type(self.helper_modules).__name__}") + if not self.helper_modules: + raise ValueError("helper_modules cannot be empty (use None for no helper modules)") + for module_name in self.helper_modules: + if not isinstance(module_name, str) or not module_name.isidentifier() or keyword.iskeyword(module_name): + raise ValueError(f"helper_modules must contain plain module names without dots, paths or '.py', got: {module_name!r}") + if self.workspace_mode: + raise ValueError( + "helper_modules is not supported in workspace mode, where modules in the student's folder can already be imported. " + "Set workspace_mode=False or remove helper_modules." + ) + # Validate builtin_whitelist contains non-empty strings if self.builtin_whitelist is not None: if not isinstance(self.builtin_whitelist, list): diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 3ea59a3..e1901dd 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -355,6 +355,7 @@ def __init__( names_for_user_list: list[str] | None, worker_username: str | None, notebook_cell_tag: str | None = None, + helper_modules: dict[str, str] | None = None, ) -> None: super().__init__(import_whitelist, import_blacklist, starting_vars, builtin_whitelist, names_for_user_list, worker_username) self.leading_file = file_names.leading_file @@ -362,6 +363,7 @@ def __init__( self.student_code_file = file_names.student_code_file self.setup_code_file = file_names.setup_code_file self.notebook_cell_tag = notebook_cell_tag + self.helper_modules = helper_modules def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_TIMEOUT) -> ProcessStartResponse: if self.worker_username is not None: @@ -402,6 +404,7 @@ def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_T starting_vars=self.starting_vars, builtin_whitelist=self.builtin_whitelist, names_for_user_list=self.names_for_user_list, + helper_modules=self.helper_modules, ) self._send_json_object(json_message) return self._read_start_response() diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 81b96da..a235d48 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -99,6 +99,38 @@ def data_json(request: pytest.FixtureRequest) -> dict[str, Any] | None: # raise ValueError(f"Data file '{data_file.name}' not found in '{datadir}'.") +def _load_helper_modules( + request: pytest.FixtureRequest, + names: list[str] | None, + import_blacklist: list[str] | None, +) -> dict[str, str] | None: + """ + Read the source of each module named by the helper_modules setting. + + A problem with these files is the instructor's to fix, so it makes the submission ungradable + instead of costing the student an attempt. + """ + if names is None: + return None + + data_dir = get_datadir(request.module) + sources: dict[str, str] = {} + for name in names: + path = None if data_dir is None else data_dir / f"{name}.py" + try: + if import_blacklist is not None and name in import_blacklist: + raise ValueError("it's also in import_blacklist, so it can't be imported") + if path is None or not path.is_file(): + raise FileNotFoundError(f"{path} doesn't exist") + sources[name] = path.read_text(encoding="utf-8") + compile(sources[name], f"{name}.py", "exec") + except (OSError, ValueError, SyntaxError) as e: + message = f"Autograder configuration error in helper_modules: can't use '{name}': {e}" + request.config.result_collector_plugin.collection_errors.append(message) # type: ignore[attr-defined] + pytest.fail(message, pytrace=False) + return sources + + def _initialize_sandbox_fixture( request: pytest.FixtureRequest, data_json: dict[str, Any] | None, @@ -127,6 +159,7 @@ def _initialize_sandbox_fixture( # Check for module-level ConfigObject that overrides all other configuration config_dict: dict[str, Any] = {} notebook_cell_tag: str | None = None + helper_module_names: list[str] | None = None if hasattr(request, "module") and hasattr(request.module, "autograder_config"): config_object = request.module.autograder_config if not isinstance(config_object, ConfigObject): @@ -137,6 +170,7 @@ def _initialize_sandbox_fixture( initialization_timeout = config_object.sandbox_timeout config_starting_vars = config_object.starting_vars notebook_cell_tag = config_object.notebook_cell_tag + helper_module_names = config_object.helper_modules else: # Check for module-level timeout variable (works for module-scoped fixtures) # This allows setting: sandbox_timeout = 0.5 at module level @@ -198,6 +232,7 @@ def _initialize_sandbox_fixture( names_for_user_list=names_for_user_list, worker_username=request.config.getoption("--worker-username"), notebook_cell_tag=notebook_cell_tag, + helper_modules=_load_helper_modules(request, helper_module_names, import_blacklist), ) return fixture, initialization_timeout diff --git a/src/pytest_prairielearn_grader/utils.py b/src/pytest_prairielearn_grader/utils.py index 32c8acb..6cf7f1c 100644 --- a/src/pytest_prairielearn_grader/utils.py +++ b/src/pytest_prairielearn_grader/utils.py @@ -145,6 +145,7 @@ class ProcessStartRequest(TypedDict): starting_vars: dict[str, Any] | None builtin_whitelist: list[str] | None names_for_user_list: list[str] | None + helper_modules: dict[str, str] | None # Module name to source code class WorkspaceStartRequest(TypedDict): diff --git a/tests/scenario_root/test_helper_modules/__init__.py b/tests/scenario_root/test_helper_modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_helper_modules/ans.py b/tests/scenario_root/test_helper_modules/ans.py new file mode 100644 index 0000000..620af89 --- /dev/null +++ b/tests/scenario_root/test_helper_modules/ans.py @@ -0,0 +1 @@ +ANSWER = "the reference answer" diff --git a/tests/scenario_root/test_helper_modules/expected_outcome.json b/tests/scenario_root/test_helper_modules/expected_outcome.json new file mode 100644 index 0000000..2d42b00 --- /dev/null +++ b/tests/scenario_root/test_helper_modules/expected_outcome.json @@ -0,0 +1,110 @@ +{ + "expected_data_object": { + "score": 1.0, + "tests": [ + { + "max_points": 1, + "message": "", + "name": "setup code", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules.py::test_setup_code[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "query", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules.py::test_query[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "query_function", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules.py::test_query_function[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "module_sandbox", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules.py::test_module_sandbox[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "one copy of each module", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules.py::test_one_copy[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "helper modules can import each other", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules.py::test_second_helper[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "modules that aren't named can't be imported", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules.py::test_not_named[student_code-import_ans]" + }, + { + "max_points": 1, + "message": "", + "name": "modules that aren't named can't be imported", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules.py::test_not_named[student_code-import_test_student]" + }, + { + "max_points": 1, + "message": "", + "name": "modules that aren't named can't be imported", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules.py::test_not_named[student_code-import_setup_code]" + }, + { + "max_points": 1, + "message": "", + "name": "modules that aren't named can't be imported", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules.py::test_not_named[student_code-import_scenario]" + }, + { + "max_points": 1, + "message": "", + "name": "modules that aren't named can't be imported", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules.py::test_not_named[student_code-helper_imports_ans]" + } + ] + }, + "expected_outcome": { + "passed": 11, + "failed": 0 + } +} diff --git a/tests/scenario_root/test_helper_modules/helper_function.py b/tests/scenario_root/test_helper_modules/helper_function.py new file mode 100644 index 0000000..894d01f --- /dev/null +++ b/tests/scenario_root/test_helper_modules/helper_function.py @@ -0,0 +1,5 @@ +SCALE = 3 + + +def triple(x): + return SCALE * x diff --git a/tests/scenario_root/test_helper_modules/leading_code.py b/tests/scenario_root/test_helper_modules/leading_code.py new file mode 100644 index 0000000..83273d4 --- /dev/null +++ b/tests/scenario_root/test_helper_modules/leading_code.py @@ -0,0 +1,3 @@ +import helper_function as hf + +from_leading = hf.triple(3) diff --git a/tests/scenario_root/test_helper_modules/scenario.py b/tests/scenario_root/test_helper_modules/scenario.py new file mode 100644 index 0000000..480a6cb --- /dev/null +++ b/tests/scenario_root/test_helper_modules/scenario.py @@ -0,0 +1,60 @@ +"""Modules named by helper_modules can be imported, and nothing else in the tests folder can.""" + +from pathlib import Path + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +# Written here because the outer test run would collect a file with this name in the scenario folder. +Path(__file__).with_suffix("").joinpath("test_student.py").write_text('ANSWER = "the reference answer"\n') + +autograder_config = ConfigObject( + helper_modules=["helper_function", "second_helper"], + sandbox_timeout=5, +) + + +@pytest.mark.grading_data(name="setup code", points=1) +def test_setup_code(sandbox: StudentFixture) -> None: + assert sandbox.query_setup("from_setup") == 6 + + +@pytest.mark.grading_data(name="query", points=1) +def test_query(sandbox: StudentFixture) -> None: + assert sandbox.query("from_leading") == 9 + assert sandbox.query("from_student") == 12 + assert sandbox.query("from_alias") == 15 + + +@pytest.mark.grading_data(name="query_function", points=1) +def test_query_function(sandbox: StudentFixture) -> None: + assert sandbox.query_function("import_when_called", 7) == 21 + + +@pytest.mark.grading_data(name="module_sandbox", points=1) +def test_module_sandbox(module_sandbox: StudentFixture) -> None: + assert module_sandbox.query_function("uses_helper", 2) == 6 + + +@pytest.mark.grading_data(name="one copy of each module", points=1) +def test_one_copy(sandbox: StudentFixture) -> None: + assert sandbox.query_function("same_module") is True + + +@pytest.mark.grading_data(name="helper modules can import each other", points=1) +def test_second_helper(sandbox: StudentFixture) -> None: + assert sandbox.query_function("through_second_helper", 2) == 7 + + +@pytest.mark.grading_data(name="modules that aren't named can't be imported", points=1) +@pytest.mark.parametrize( + "function_name", + ["import_ans", "import_test_student", "import_setup_code", "import_scenario", "helper_imports_ans"], +) +def test_not_named(sandbox: StudentFixture, function_name: str) -> None: + response = sandbox.query_function_raw(function_name) + assert response["status"] == "exception" + assert response["exception_name"] in ("ModuleNotFoundError", "ImportError") + assert "No module named" in str(response["exception_message"]) diff --git a/tests/scenario_root/test_helper_modules/second_helper.py b/tests/scenario_root/test_helper_modules/second_helper.py new file mode 100644 index 0000000..b78333d --- /dev/null +++ b/tests/scenario_root/test_helper_modules/second_helper.py @@ -0,0 +1,11 @@ +import helper_function + + +def triple_plus_one(x): + return helper_function.triple(x) + 1 + + +def import_ans(): + import ans + + return ans.ANSWER diff --git a/tests/scenario_root/test_helper_modules/setup_code.py b/tests/scenario_root/test_helper_modules/setup_code.py new file mode 100644 index 0000000..3104a64 --- /dev/null +++ b/tests/scenario_root/test_helper_modules/setup_code.py @@ -0,0 +1,3 @@ +import helper_function + +from_setup = helper_function.triple(2) diff --git a/tests/scenario_root/test_helper_modules/student_code.py b/tests/scenario_root/test_helper_modules/student_code.py new file mode 100644 index 0000000..3bd33dc --- /dev/null +++ b/tests/scenario_root/test_helper_modules/student_code.py @@ -0,0 +1,58 @@ +import helper_function +import helper_function as alias +from helper_function import triple + +from_student = helper_function.triple(4) +from_alias = alias.SCALE * 5 + + +def uses_helper(x): + return triple(x) + + +def import_when_called(x): + import helper_function as late + + return late.triple(x) + + +def same_module(): + import helper_function as late + + return late is hf and late is alias + + +def through_second_helper(x): + import second_helper + + return second_helper.triple_plus_one(x) + + +def import_ans(): + import ans + + return ans.ANSWER + + +def import_test_student(): + import test_student + + return test_student.ANSWER + + +def import_setup_code(): + import setup_code + + return setup_code + + +def import_scenario(): + import test_helper_modules + + return test_helper_modules + + +def helper_imports_ans(): + import second_helper + + return second_helper.import_ans() diff --git a/tests/scenario_root/test_helper_modules_blacklisted/__init__.py b/tests/scenario_root/test_helper_modules_blacklisted/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_helper_modules_blacklisted/expected_outcome.json b/tests/scenario_root/test_helper_modules_blacklisted/expected_outcome.json new file mode 100644 index 0000000..e03f8f9 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_blacklisted/expected_outcome.json @@ -0,0 +1,13 @@ +{ + "expected_data_object": { + "gradable": false, + "message": "The autograder encountered an internal error.", + "tests": [ + { + "test_id": "test_helper_modules_blacklisted.py::test_query[student_code]", + "outcome": "error", + "message": "it's also in import_blacklist" + } + ] + } +} diff --git a/tests/scenario_root/test_helper_modules_blacklisted/helper_function.py b/tests/scenario_root/test_helper_modules_blacklisted/helper_function.py new file mode 100644 index 0000000..894d01f --- /dev/null +++ b/tests/scenario_root/test_helper_modules_blacklisted/helper_function.py @@ -0,0 +1,5 @@ +SCALE = 3 + + +def triple(x): + return SCALE * x diff --git a/tests/scenario_root/test_helper_modules_blacklisted/scenario.py b/tests/scenario_root/test_helper_modules_blacklisted/scenario.py new file mode 100644 index 0000000..699fa98 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_blacklisted/scenario.py @@ -0,0 +1,13 @@ +"""A named module that's also blacklisted makes the submission ungradable.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +autograder_config = ConfigObject(helper_modules=["helper_function"], import_blacklist=["helper_function"]) + + +@pytest.mark.grading_data(name="query", points=1) +def test_query(sandbox: StudentFixture) -> None: + assert sandbox.query("value") == 1 diff --git a/tests/scenario_root/test_helper_modules_blacklisted/student_code.py b/tests/scenario_root/test_helper_modules_blacklisted/student_code.py new file mode 100644 index 0000000..0a979d4 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_blacklisted/student_code.py @@ -0,0 +1 @@ +value = 1 diff --git a/tests/scenario_root/test_helper_modules_missing/__init__.py b/tests/scenario_root/test_helper_modules_missing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_helper_modules_missing/expected_outcome.json b/tests/scenario_root/test_helper_modules_missing/expected_outcome.json new file mode 100644 index 0000000..9695fbf --- /dev/null +++ b/tests/scenario_root/test_helper_modules_missing/expected_outcome.json @@ -0,0 +1,13 @@ +{ + "expected_data_object": { + "gradable": false, + "message": "The autograder encountered an internal error.", + "tests": [ + { + "test_id": "test_helper_modules_missing.py::test_query[student_code]", + "outcome": "error", + "message": "Autograder configuration error in helper_modules: can't use 'helper_function':" + } + ] + } +} diff --git a/tests/scenario_root/test_helper_modules_missing/scenario.py b/tests/scenario_root/test_helper_modules_missing/scenario.py new file mode 100644 index 0000000..e37171d --- /dev/null +++ b/tests/scenario_root/test_helper_modules_missing/scenario.py @@ -0,0 +1,13 @@ +"""A named module with no file makes the submission ungradable, with a message for the instructor.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +autograder_config = ConfigObject(helper_modules=["helper_function"]) + + +@pytest.mark.grading_data(name="query", points=1) +def test_query(sandbox: StudentFixture) -> None: + assert sandbox.query("value") == 1 diff --git a/tests/scenario_root/test_helper_modules_missing/student_code.py b/tests/scenario_root/test_helper_modules_missing/student_code.py new file mode 100644 index 0000000..0a979d4 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_missing/student_code.py @@ -0,0 +1 @@ +value = 1 diff --git a/tests/scenario_root/test_helper_modules_notebook/__init__.py b/tests/scenario_root/test_helper_modules_notebook/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_helper_modules_notebook/expected_outcome.json b/tests/scenario_root/test_helper_modules_notebook/expected_outcome.json new file mode 100644 index 0000000..307ade1 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_notebook/expected_outcome.json @@ -0,0 +1,20 @@ +{ + "expected_data_object": { + "score": 1.0, + "tests": [ + { + "max_points": 1, + "message": "", + "name": "notebook", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_notebook.py::test_notebook[student_code]" + } + ] + }, + "expected_outcome": { + "passed": 1, + "failed": 0 + } +} diff --git a/tests/scenario_root/test_helper_modules_notebook/helper_function.py b/tests/scenario_root/test_helper_modules_notebook/helper_function.py new file mode 100644 index 0000000..894d01f --- /dev/null +++ b/tests/scenario_root/test_helper_modules_notebook/helper_function.py @@ -0,0 +1,5 @@ +SCALE = 3 + + +def triple(x): + return SCALE * x diff --git a/tests/scenario_root/test_helper_modules_notebook/scenario.py b/tests/scenario_root/test_helper_modules_notebook/scenario.py new file mode 100644 index 0000000..1584be4 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_notebook/scenario.py @@ -0,0 +1,17 @@ +"""A tagged notebook cell can import a helper module.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +autograder_config = ConfigObject( + helper_modules=["helper_function"], + student_code_pattern="student_code.ipynb", + notebook_cell_tag="#grade", +) + + +@pytest.mark.grading_data(name="notebook", points=1) +def test_notebook(sandbox: StudentFixture) -> None: + assert sandbox.query("value") == 6 diff --git a/tests/scenario_root/test_helper_modules_notebook/student_code.ipynb b/tests/scenario_root/test_helper_modules_notebook/student_code.ipynb new file mode 100644 index 0000000..f3b7c84 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_notebook/student_code.ipynb @@ -0,0 +1,41 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Not tagged, so this doesn't run during grading\n", + "raise ValueError('This cell must not run')" + ], + "id": "cell0" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#grade\n", + "import helper_function as hf\n", + "\n", + "value = hf.triple(2)" + ], + "id": "cell1" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/scenario_root/test_helper_modules_restricted/__init__.py b/tests/scenario_root/test_helper_modules_restricted/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_helper_modules_restricted/ans.py b/tests/scenario_root/test_helper_modules_restricted/ans.py new file mode 100644 index 0000000..620af89 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_restricted/ans.py @@ -0,0 +1 @@ +ANSWER = "the reference answer" diff --git a/tests/scenario_root/test_helper_modules_restricted/blocked_import.py b/tests/scenario_root/test_helper_modules_restricted/blocked_import.py new file mode 100644 index 0000000..7197920 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_restricted/blocked_import.py @@ -0,0 +1,3 @@ +import math + +PI = math.pi diff --git a/tests/scenario_root/test_helper_modules_restricted/expected_outcome.json b/tests/scenario_root/test_helper_modules_restricted/expected_outcome.json new file mode 100644 index 0000000..b0d15cb --- /dev/null +++ b/tests/scenario_root/test_helper_modules_restricted/expected_outcome.json @@ -0,0 +1,65 @@ +{ + "expected_data_object": { + "score": 1.0, + "tests": [ + { + "max_points": 1, + "message": "", + "name": "helper module isn't whitelisted", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_restricted.py::test_helper_module[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "whitelist still applies", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_restricted.py::test_whitelist[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "blacklist still applies", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_restricted.py::test_blacklist[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "whitelisted file in the folder isn't importable", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_restricted.py::test_whitelisted_file[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "helper module's imports follow the rules", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_restricted.py::test_helper_import_rules[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "helper module's builtins follow the rules", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_restricted.py::test_helper_builtin_rules[student_code]" + } + ] + }, + "expected_outcome": { + "passed": 6, + "failed": 0 + } +} diff --git a/tests/scenario_root/test_helper_modules_restricted/helper_function.py b/tests/scenario_root/test_helper_modules_restricted/helper_function.py new file mode 100644 index 0000000..894d01f --- /dev/null +++ b/tests/scenario_root/test_helper_modules_restricted/helper_function.py @@ -0,0 +1,5 @@ +SCALE = 3 + + +def triple(x): + return SCALE * x diff --git a/tests/scenario_root/test_helper_modules_restricted/scenario.py b/tests/scenario_root/test_helper_modules_restricted/scenario.py new file mode 100644 index 0000000..6c1393d --- /dev/null +++ b/tests/scenario_root/test_helper_modules_restricted/scenario.py @@ -0,0 +1,53 @@ +"""Helper modules work with import restrictions set, and follow them as student code does.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +autograder_config = ConfigObject( + helper_modules=["helper_function", "blocked_import", "uses_open"], + # A whitelisted name isn't importable either unless it's named: the folder isn't on the path. + import_whitelist=["ans"], + import_blacklist=["math"], +) + + +@pytest.mark.grading_data(name="helper module isn't whitelisted", points=1) +def test_helper_module(sandbox: StudentFixture) -> None: + assert sandbox.query("value") == 6 + + +@pytest.mark.grading_data(name="whitelist still applies", points=1) +def test_whitelist(sandbox: StudentFixture) -> None: + response = sandbox.query_function_raw("import_json") + assert response["exception_name"] == "ImportError" + assert response["exception_message"] == "Module 'json' is not allowed to be imported." + + +@pytest.mark.grading_data(name="blacklist still applies", points=1) +def test_blacklist(sandbox: StudentFixture) -> None: + response = sandbox.query_function_raw("import_math") + assert response["exception_name"] == "ImportError" + assert response["exception_message"] == "Module 'math' is blacklisted and cannot be imported." + + +@pytest.mark.grading_data(name="whitelisted file in the folder isn't importable", points=1) +def test_whitelisted_file(sandbox: StudentFixture) -> None: + response = sandbox.query_function_raw("import_ans") + assert response["exception_name"] == "ModuleNotFoundError" + + +@pytest.mark.grading_data(name="helper module's imports follow the rules", points=1) +def test_helper_import_rules(sandbox: StudentFixture) -> None: + response = sandbox.query_function_raw("import_blocked") + assert response["exception_name"] == "ImportError" + assert response["exception_message"] == ( + "Helper module 'blocked_import' could not be loaded: ImportError: Module 'math' is blacklisted and cannot be imported." + ) + + +@pytest.mark.grading_data(name="helper module's builtins follow the rules", points=1) +def test_helper_builtin_rules(sandbox: StudentFixture) -> None: + response = sandbox.query_function_raw("call_open") + assert response["exception_name"] == "NameError" diff --git a/tests/scenario_root/test_helper_modules_restricted/student_code.py b/tests/scenario_root/test_helper_modules_restricted/student_code.py new file mode 100644 index 0000000..7c7da9b --- /dev/null +++ b/tests/scenario_root/test_helper_modules_restricted/student_code.py @@ -0,0 +1,33 @@ +import helper_function + +value = helper_function.triple(2) + + +def import_json(): + import json + + return json + + +def import_math(): + import math + + return math + + +def import_ans(): + import ans + + return ans + + +def import_blocked(): + import blocked_import + + return blocked_import.PI + + +def call_open(): + import uses_open + + return uses_open.read_file("student_code.py") diff --git a/tests/scenario_root/test_helper_modules_restricted/uses_open.py b/tests/scenario_root/test_helper_modules_restricted/uses_open.py new file mode 100644 index 0000000..2707b07 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_restricted/uses_open.py @@ -0,0 +1,2 @@ +def read_file(path): + return open(path).read() diff --git a/tests/scenario_root/test_helper_modules_syntax_error/__init__.py b/tests/scenario_root/test_helper_modules_syntax_error/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_helper_modules_syntax_error/expected_outcome.json b/tests/scenario_root/test_helper_modules_syntax_error/expected_outcome.json new file mode 100644 index 0000000..2b8023a --- /dev/null +++ b/tests/scenario_root/test_helper_modules_syntax_error/expected_outcome.json @@ -0,0 +1,13 @@ +{ + "expected_data_object": { + "gradable": false, + "message": "The autograder encountered an internal error.", + "tests": [ + { + "test_id": "test_helper_modules_syntax_error.py::test_query[student_code]", + "outcome": "error", + "message": "Autograder configuration error in helper_modules: can't use 'helper_function':" + } + ] + } +} diff --git a/tests/scenario_root/test_helper_modules_syntax_error/helper_function.py b/tests/scenario_root/test_helper_modules_syntax_error/helper_function.py new file mode 100644 index 0000000..967e47d --- /dev/null +++ b/tests/scenario_root/test_helper_modules_syntax_error/helper_function.py @@ -0,0 +1 @@ +def triple(x: diff --git a/tests/scenario_root/test_helper_modules_syntax_error/scenario.py b/tests/scenario_root/test_helper_modules_syntax_error/scenario.py new file mode 100644 index 0000000..bb89e24 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_syntax_error/scenario.py @@ -0,0 +1,13 @@ +"""A syntax error in a helper module isn't reported as the student's syntax error.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +autograder_config = ConfigObject(helper_modules=["helper_function"]) + + +@pytest.mark.grading_data(name="query", points=1) +def test_query(sandbox: StudentFixture) -> None: + assert sandbox.query("value") == 1 diff --git a/tests/scenario_root/test_helper_modules_syntax_error/student_code.py b/tests/scenario_root/test_helper_modules_syntax_error/student_code.py new file mode 100644 index 0000000..0a979d4 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_syntax_error/student_code.py @@ -0,0 +1 @@ +value = 1 diff --git a/tests/test_autograder_scenarios.py b/tests/test_autograder_scenarios.py index 89dfd1d..096b817 100644 --- a/tests/test_autograder_scenarios.py +++ b/tests/test_autograder_scenarios.py @@ -143,6 +143,8 @@ def test_autograder_scenario_with_pytester(pytester: pytest.Pytester, scenario_d assert test_id in test_results_obj, f"Test '{test_id}' not found in results." actual_test = test_results_obj[test_id] assert actual_test["outcome"] == expected_test["outcome"], f"Outcome mismatch for test '{test_id}'." + if "message" in expected_test: + assert expected_test["message"] in actual_test["message"], f"Message mismatch for test '{test_id}'." outcome_dict[CONVERSION_DICT[actual_test["outcome"]]] += 1 result.assert_outcomes(**outcome_dict) return diff --git a/tests/test_config.py b/tests/test_config.py index 380d36a..64f0cc7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -266,3 +266,44 @@ def test_summary_scores_use_test_point_values() -> None: ) assert (total_score, max_score) == (6.0, 6) + + +# --------------------------------------------------------------------------- +# helper_modules field +# --------------------------------------------------------------------------- + + +def test_helper_modules_default_is_none() -> None: + assert ConfigObject().helper_modules is None + + +def test_helper_modules_valid() -> None: + assert ConfigObject(helper_modules=["helper_function", "_private2"]).helper_modules == ["helper_function", "_private2"] + + +def test_helper_modules_not_in_params() -> None: + assert "helper_modules" not in ConfigObject(helper_modules=["helper_function"]).to_dict() + + +@pytest.mark.parametrize( + "name", + ["", " ", "..", "../ans", "a/b", "a\\b", "pkg.mod", "helper.py", "/grade/tests/ans", "1abc", "has space", "class", 5, None], +) +def test_helper_modules_rejects_invalid_names(name: object) -> None: + with pytest.raises(ValueError, match="helper_modules must contain plain module names"): + ConfigObject(helper_modules=[name]) # type: ignore[list-item] + + +def test_helper_modules_must_be_a_list() -> None: + with pytest.raises(TypeError, match="helper_modules must be a list"): + ConfigObject(helper_modules="helper_function") # type: ignore[arg-type] + + +def test_helper_modules_cannot_be_empty() -> None: + with pytest.raises(ValueError, match="helper_modules cannot be empty"): + ConfigObject(helper_modules=[]) + + +def test_helper_modules_not_supported_in_workspace_mode() -> None: + with pytest.raises(ValueError, match="not supported in workspace mode"): + ConfigObject(workspace_mode=True, helper_modules=["helper_function"]) From c13dcc654c016eb67b358e14e3f46570d82d7f39 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Fri, 6 Nov 2026 13:48:15 -0500 Subject: [PATCH 2/5] Support helper modules in workspace mode An instructor's helper usually isn't in the student's folder, and if it's a graded file then students can edit it. So workspace mode needs the setting too. Imports of a named module get the instructor's file, even if the student submits one with the same name. Co-Authored-By: Claude Opus 5.5 --- .github/copilot-instructions.md | 3 +- quick_start.md | 9 +- .../_student_code_runner.py | 18 ++-- src/pytest_prairielearn_grader/config.py | 9 +- src/pytest_prairielearn_grader/fixture.py | 3 + src/pytest_prairielearn_grader/plugin.py | 3 + src/pytest_prairielearn_grader/utils.py | 1 + .../test_helper_modules_workspace/__init__.py | 0 .../test_helper_modules_workspace/ans.py | 1 + .../expected_outcome.json | 83 +++++++++++++++++++ .../helper_function.py | 12 +++ .../test_helper_modules_workspace/scenario.py | 62 ++++++++++++++ .../second_helper.py | 2 + .../setup_code.py | 3 + .../student/calculator.py | 34 ++++++++ .../student/helper_function.py | 1 + .../student/main.py | 3 + .../__init__.py | 0 .../expected_outcome.json | 13 +++ .../scenario.py | 13 +++ .../student/calculator.py | 1 + tests/test_config.py | 6 +- 22 files changed, 263 insertions(+), 17 deletions(-) create mode 100644 tests/scenario_root/test_helper_modules_workspace/__init__.py create mode 100644 tests/scenario_root/test_helper_modules_workspace/ans.py create mode 100644 tests/scenario_root/test_helper_modules_workspace/expected_outcome.json create mode 100644 tests/scenario_root/test_helper_modules_workspace/helper_function.py create mode 100644 tests/scenario_root/test_helper_modules_workspace/scenario.py create mode 100644 tests/scenario_root/test_helper_modules_workspace/second_helper.py create mode 100644 tests/scenario_root/test_helper_modules_workspace/setup_code.py create mode 100644 tests/scenario_root/test_helper_modules_workspace/student/calculator.py create mode 100644 tests/scenario_root/test_helper_modules_workspace/student/helper_function.py create mode 100644 tests/scenario_root/test_helper_modules_workspace/student/main.py create mode 100644 tests/scenario_root/test_helper_modules_workspace_missing/__init__.py create mode 100644 tests/scenario_root/test_helper_modules_workspace_missing/expected_outcome.json create mode 100644 tests/scenario_root/test_helper_modules_workspace_missing/scenario.py create mode 100644 tests/scenario_root/test_helper_modules_workspace_missing/student/calculator.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 9388b89..64cc9ae 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -143,7 +143,8 @@ result = sandbox.query_function("student_func", arg1, arg2, timeout=5) `ConfigObject(helper_modules=["helper_function"])` lets setup, leading and student code import `helper_function.py` from the tests folder, which isn't otherwise on the import path. Students can read -these modules, so they must not contain answers. It isn't supported in workspace mode. +these modules, so they must not contain answers. In workspace mode, they take priority over student files +with the same name when imported. ### 5. Serialization diff --git a/quick_start.md b/quick_start.md index 919730c..5ec1402 100644 --- a/quick_start.md +++ b/quick_start.md @@ -378,8 +378,12 @@ import helper_function as hf - **Configuration errors:** if a listed file is missing or has a syntax error, or the module is blacklisted, the submission is ungradable. The students don't lose an attempt, and each test's message says what's wrong. -- **Fixtures:** it works with `sandbox` and `module_sandbox`, for `.py` and `.ipynb` submissions. It - isn't supported in workspace mode, where modules in the student's folder can already be imported. +- **Fixtures:** it works with `sandbox` and `module_sandbox`, for `.py` and `.ipynb` submissions, and + with `workspace_sandbox`. +- **Workspace mode:** an `import` of a listed module gets the instructor's file, even if the student + submits one with the same name, and it keeps the restricted builtins that the student's own modules + don't have. Dotted queries such as `query_function("helper_function.triple")` only search the + student's project, so they won't call a helper module. > **Warning:** Students can read everything in a helper module, so it must not contain answers. This > setting also doesn't make the rest of the tests folder private. `importlib` isn't blacklisted by @@ -1505,6 +1509,7 @@ goes through the import machinery. | `sandbox_timeout` | `float` | `1.0` | Timeout for sandbox startup (increase for larger projects). | | `import_whitelist` | `list[str] \| None` | `None` | Allowed imports. `None` = allow all. | | `import_blacklist` | `list[str] \| None` | default list | Blocked imports. | +| `helper_modules` | `list[str] \| None` | `None` | Modules from the tests folder to make importable. See [Helper Modules](#helper-modules). | ### Production vs. Local Development diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 8de85c6..7987e8d 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -212,8 +212,8 @@ def get_custom_importer( each other without instructors needing to maintain a manual whitelist of their own files. Relative imports (level > 0) are always permitted for the same reason. - Modules in *helper_modules* can be imported without being on the whitelist, unless they're on - the blacklist. + Modules in *helper_modules* can be imported without being on the whitelist, and they take + priority over workspace files with the same name. The grader rejects any that are blacklisted. """ original_import = _REAL_IMPORT @@ -238,6 +238,9 @@ def custom_import( if level > 0: return original_import(name, globals, locals, fl, level) + if helper_modules is not None and name in helper_modules: + return helper_modules.load(name) + # Workspace-local modules are always importable regardless of whitelist/blacklist. # A module is considered local if its top-level package maps to a file or directory # directly inside workspace_dir. @@ -249,8 +252,6 @@ def custom_import( # Apply blacklist / whitelist to external imports. if import_blacklist is not None and name in import_blacklist: raise ImportError(f"Module '{name}' is blacklisted and cannot be imported.") - elif helper_modules is not None and name in helper_modules: - return helper_modules.load(name) elif ( (import_whitelist is not None and name in import_whitelist) or name.startswith("_") # Allow Python internal/private modules (_io, _abc, _collections, etc.) @@ -376,6 +377,7 @@ async def workspace_runner( starting_vars: dict[str, Any] | None, builtin_whitelist: list[str] | None, names_for_user_list: list[str] | None, + helper_modules: dict[str, str] | None, ) -> tuple[dict[str, Any], dict[str, Any], ProcessStartResponse]: """ Sets up a workspace sandbox by inserting workspace_dir at the front of sys.path @@ -393,7 +395,12 @@ async def workspace_runner( student_code_vars: dict[str, Any] = {} student_code_vars["__builtins__"] = get_builtins(builtin_whitelist) student_code_vars["__builtins__"]["__name__"] = "__main__" - workspace_importer = get_custom_importer(import_whitelist, import_blacklist, workspace_dir) + workspace_importer = get_custom_importer( + import_whitelist, + import_blacklist, + workspace_dir, + helper_modules=None if helper_modules is None else _HelperModules(helper_modules, student_code_vars["__builtins__"]), + ) student_code_vars["__builtins__"]["__import__"] = workspace_importer # Patch the real builtins.__import__ so that workspace modules loaded via # importlib.import_module() (which use the real builtins, not student_code_vars) @@ -521,6 +528,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit starting_vars=ws_json_message["starting_vars"], builtin_whitelist=ws_json_message["builtin_whitelist"], names_for_user_list=ws_json_message["names_for_user_list"], + helper_modules=ws_json_message["helper_modules"], ) writer.write((json.dumps(start_response) + os.linesep).encode()) diff --git a/src/pytest_prairielearn_grader/config.py b/src/pytest_prairielearn_grader/config.py index ea00045..79a31fb 100644 --- a/src/pytest_prairielearn_grader/config.py +++ b/src/pytest_prairielearn_grader/config.py @@ -77,8 +77,10 @@ def test_student_code(sandbox): also blocks is a configuration error. A helper module runs in the sandbox with the same builtins and import rules as student code. + In workspace mode, a helper module takes priority over a student file with the same name + when imported, and it keeps these restricted builtins, unlike the student's own modules. + Students can read everything in a helper module, so it must not contain answers. - Not supported in workspace mode, where the student's folder is on the import path. Example: ["helper_function"] """ @@ -259,11 +261,6 @@ def __post_init__(self) -> None: for module_name in self.helper_modules: if not isinstance(module_name, str) or not module_name.isidentifier() or keyword.iskeyword(module_name): raise ValueError(f"helper_modules must contain plain module names without dots, paths or '.py', got: {module_name!r}") - if self.workspace_mode: - raise ValueError( - "helper_modules is not supported in workspace mode, where modules in the student's folder can already be imported. " - "Set workspace_mode=False or remove helper_modules." - ) # Validate builtin_whitelist contains non-empty strings if self.builtin_whitelist is not None: diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index e1901dd..b5880a2 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -465,11 +465,13 @@ def __init__( names_for_user_list: list[str] | None, worker_username: str | None, exec_entry: str | None = None, + helper_modules: dict[str, str] | None = None, ) -> None: super().__init__(import_whitelist, import_blacklist, starting_vars, builtin_whitelist, names_for_user_list, worker_username) self.workspace_dir = workspace_dir self.setup_code_file = setup_code_file self.exec_entry = exec_entry + self.helper_modules = helper_modules def start_workspace_server(self, *, initialization_timeout: float = DEFAULT_TIMEOUT) -> ProcessStartResponse: """Start the sandbox subprocess and initialise the workspace.""" @@ -492,6 +494,7 @@ def start_workspace_server(self, *, initialization_timeout: float = DEFAULT_TIME starting_vars=self.starting_vars, builtin_whitelist=self.builtin_whitelist, names_for_user_list=self.names_for_user_list, + helper_modules=self.helper_modules, ) self._send_json_object(json_message) return self._read_start_response() diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index a235d48..b460d3c 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -492,6 +492,7 @@ def _initialize_workspace_fixture( config_dict: dict[str, Any] = {} exec_entry: str | None = None + helper_module_names: list[str] | None = None if hasattr(request, "module") and hasattr(request.module, "autograder_config"): config_object = request.module.autograder_config if not isinstance(config_object, ConfigObject): @@ -500,6 +501,7 @@ def _initialize_workspace_fixture( initialization_timeout = config_object.sandbox_timeout config_starting_vars = config_object.starting_vars exec_entry = config_object.workspace_exec_entry + helper_module_names = config_object.helper_modules else: if hasattr(request, "module") and hasattr(request.module, "sandbox_timeout"): initialization_timeout = request.module.sandbox_timeout @@ -545,6 +547,7 @@ def _initialize_workspace_fixture( names_for_user_list=names_for_user_list, worker_username=request.config.getoption("--worker-username"), exec_entry=exec_entry, + helper_modules=_load_helper_modules(request, helper_module_names, import_blacklist), ) return fixture, initialization_timeout diff --git a/src/pytest_prairielearn_grader/utils.py b/src/pytest_prairielearn_grader/utils.py index 6cf7f1c..d89d100 100644 --- a/src/pytest_prairielearn_grader/utils.py +++ b/src/pytest_prairielearn_grader/utils.py @@ -159,6 +159,7 @@ class WorkspaceStartRequest(TypedDict): starting_vars: dict[str, Any] | None builtin_whitelist: list[str] | None names_for_user_list: list[str] | None + helper_modules: dict[str, str] | None # Module name to source code class ProcessStartResponse(TypedDict): diff --git a/tests/scenario_root/test_helper_modules_workspace/__init__.py b/tests/scenario_root/test_helper_modules_workspace/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_helper_modules_workspace/ans.py b/tests/scenario_root/test_helper_modules_workspace/ans.py new file mode 100644 index 0000000..620af89 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace/ans.py @@ -0,0 +1 @@ +ANSWER = "the reference answer" diff --git a/tests/scenario_root/test_helper_modules_workspace/expected_outcome.json b/tests/scenario_root/test_helper_modules_workspace/expected_outcome.json new file mode 100644 index 0000000..9d8a9af --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace/expected_outcome.json @@ -0,0 +1,83 @@ +{ + "expected_data_object": { + "score": 1.0, + "tests": [ + { + "max_points": 1, + "message": "", + "name": "setup code", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_workspace.py::test_setup_code" + }, + { + "max_points": 1, + "message": "", + "name": "entry point", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_workspace.py::test_entry_point" + }, + { + "max_points": 1, + "message": "", + "name": "student module", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_workspace.py::test_student_module" + }, + { + "max_points": 1, + "message": "", + "name": "helper module beats student file", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_workspace.py::test_student_file_with_same_name" + }, + { + "max_points": 1, + "message": "", + "name": "builtins", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_workspace.py::test_builtins" + }, + { + "max_points": 1, + "message": "", + "name": "unnamed file", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_workspace.py::test_unnamed_file" + }, + { + "max_points": 1, + "message": "", + "name": "whitelist", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_workspace.py::test_whitelist" + }, + { + "max_points": 1, + "message": "", + "name": "dotted queries search the student's project", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_helper_modules_workspace.py::test_dotted_queries" + } + ] + }, + "expected_outcome": { + "passed": 8, + "failed": 0 + } +} diff --git a/tests/scenario_root/test_helper_modules_workspace/helper_function.py b/tests/scenario_root/test_helper_modules_workspace/helper_function.py new file mode 100644 index 0000000..a39533e --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace/helper_function.py @@ -0,0 +1,12 @@ +SCALE = 3 + + +def triple(x): + return SCALE * x + + +SOURCE = "instructor" + + +def total(values): + return sum(values) diff --git a/tests/scenario_root/test_helper_modules_workspace/scenario.py b/tests/scenario_root/test_helper_modules_workspace/scenario.py new file mode 100644 index 0000000..af9c0d1 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace/scenario.py @@ -0,0 +1,62 @@ +"""Helper modules in workspace mode, including a student file with the same name as one.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import WorkspaceFixture + +autograder_config = ConfigObject( + workspace_mode=True, + workspace_exec_entry="main.py", + helper_modules=["helper_function", "second_helper"], + # A whitelisted name isn't importable either unless it's named: the folder isn't on the path. + import_whitelist=["ans"], + names_for_user=["from_setup"], + sandbox_timeout=5, +) + + +@pytest.mark.grading_data(name="setup code", points=1) +def test_setup_code(workspace_sandbox: WorkspaceFixture) -> None: + assert workspace_sandbox.query("from_setup") == 6 + + +@pytest.mark.grading_data(name="entry point", points=1) +def test_entry_point(workspace_sandbox: WorkspaceFixture) -> None: + assert workspace_sandbox.query("from_main") == 15 + + +@pytest.mark.grading_data(name="student module", points=1) +def test_student_module(workspace_sandbox: WorkspaceFixture) -> None: + assert workspace_sandbox.query_function("calculator.triple_it", 2) == 6 + assert workspace_sandbox.query_function("calculator.add_one", 2) == 3 + + +@pytest.mark.grading_data(name="helper module beats student file", points=1) +def test_student_file_with_same_name(workspace_sandbox: WorkspaceFixture) -> None: + assert workspace_sandbox.query_function("calculator.source") == "instructor" + + +@pytest.mark.grading_data(name="builtins", points=1) +def test_builtins(workspace_sandbox: WorkspaceFixture) -> None: + assert workspace_sandbox.query_function("calculator.own_sum", [1, 2]) == 3 + response = workspace_sandbox.query_function_raw("calculator.total", [1, 2]) + assert response["exception_name"] == "NameError" + + +@pytest.mark.grading_data(name="unnamed file", points=1) +def test_unnamed_file(workspace_sandbox: WorkspaceFixture) -> None: + response = workspace_sandbox.query_function_raw("calculator.import_ans") + assert response["exception_name"] == "ModuleNotFoundError" + + +@pytest.mark.grading_data(name="whitelist", points=1) +def test_whitelist(workspace_sandbox: WorkspaceFixture) -> None: + response = workspace_sandbox.query_function_raw("calculator.import_json") + assert response["exception_name"] == "ImportError" + + +@pytest.mark.grading_data(name="dotted queries search the student's project", points=1) +def test_dotted_queries(workspace_sandbox: WorkspaceFixture) -> None: + assert workspace_sandbox.query("helper_function.SOURCE") == "student" + assert workspace_sandbox.query_function_raw("second_helper.add_one", 1)["status"] != "success" diff --git a/tests/scenario_root/test_helper_modules_workspace/second_helper.py b/tests/scenario_root/test_helper_modules_workspace/second_helper.py new file mode 100644 index 0000000..fe81f17 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace/second_helper.py @@ -0,0 +1,2 @@ +def add_one(x): + return x + 1 diff --git a/tests/scenario_root/test_helper_modules_workspace/setup_code.py b/tests/scenario_root/test_helper_modules_workspace/setup_code.py new file mode 100644 index 0000000..3104a64 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace/setup_code.py @@ -0,0 +1,3 @@ +import helper_function + +from_setup = helper_function.triple(2) diff --git a/tests/scenario_root/test_helper_modules_workspace/student/calculator.py b/tests/scenario_root/test_helper_modules_workspace/student/calculator.py new file mode 100644 index 0000000..e0575e9 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace/student/calculator.py @@ -0,0 +1,34 @@ +import helper_function +import second_helper + + +def triple_it(x): + return helper_function.triple(x) + + +def source(): + return helper_function.SOURCE + + +def add_one(x): + return second_helper.add_one(x) + + +def total(values): + return helper_function.total(values) + + +def own_sum(values): + return sum(values) + + +def import_ans(): + import ans + + return ans.ANSWER + + +def import_json(): + import json + + return json diff --git a/tests/scenario_root/test_helper_modules_workspace/student/helper_function.py b/tests/scenario_root/test_helper_modules_workspace/student/helper_function.py new file mode 100644 index 0000000..5ddbe32 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace/student/helper_function.py @@ -0,0 +1 @@ +SOURCE = "student" diff --git a/tests/scenario_root/test_helper_modules_workspace/student/main.py b/tests/scenario_root/test_helper_modules_workspace/student/main.py new file mode 100644 index 0000000..768efc2 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace/student/main.py @@ -0,0 +1,3 @@ +import helper_function + +from_main = helper_function.triple(5) diff --git a/tests/scenario_root/test_helper_modules_workspace_missing/__init__.py b/tests/scenario_root/test_helper_modules_workspace_missing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_helper_modules_workspace_missing/expected_outcome.json b/tests/scenario_root/test_helper_modules_workspace_missing/expected_outcome.json new file mode 100644 index 0000000..acf260f --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace_missing/expected_outcome.json @@ -0,0 +1,13 @@ +{ + "expected_data_object": { + "gradable": false, + "message": "The autograder encountered an internal error.", + "tests": [ + { + "test_id": "test_helper_modules_workspace_missing.py::test_query", + "outcome": "error", + "message": "Autograder configuration error in helper_modules: can't use 'helper_function':" + } + ] + } +} diff --git a/tests/scenario_root/test_helper_modules_workspace_missing/scenario.py b/tests/scenario_root/test_helper_modules_workspace_missing/scenario.py new file mode 100644 index 0000000..3999ef4 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace_missing/scenario.py @@ -0,0 +1,13 @@ +"""A named module with no file makes a workspace submission ungradable too.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import WorkspaceFixture + +autograder_config = ConfigObject(workspace_mode=True, helper_modules=["helper_function"]) + + +@pytest.mark.grading_data(name="query", points=1) +def test_query(workspace_sandbox: WorkspaceFixture) -> None: + assert workspace_sandbox.query("calculator.VALUE") == 1 diff --git a/tests/scenario_root/test_helper_modules_workspace_missing/student/calculator.py b/tests/scenario_root/test_helper_modules_workspace_missing/student/calculator.py new file mode 100644 index 0000000..b15b1b0 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace_missing/student/calculator.py @@ -0,0 +1 @@ +VALUE = 1 diff --git a/tests/test_config.py b/tests/test_config.py index 64f0cc7..c82357e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -304,6 +304,6 @@ def test_helper_modules_cannot_be_empty() -> None: ConfigObject(helper_modules=[]) -def test_helper_modules_not_supported_in_workspace_mode() -> None: - with pytest.raises(ValueError, match="not supported in workspace mode"): - ConfigObject(workspace_mode=True, helper_modules=["helper_function"]) +def test_helper_modules_in_workspace_mode() -> None: + config = ConfigObject(workspace_mode=True, helper_modules=["helper_function"]) + assert config.helper_modules == ["helper_function"] From 83d8cf059f7278e6815d481b11a68ad52323f646 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Fri, 6 Nov 2026 16:47:31 -0500 Subject: [PATCH 3/5] Tell students when a file clashes with a helper module In workspace mode an import of a helper module silently replaced a student file with the same name, so the student's code ran against a module they didn't write. Fail with a message asking them to rename it. An identical copy is allowed, since instructors may give students one to work with. Co-Authored-By: Claude Opus 5.5 --- quick_start.md | 11 +++--- src/pytest_prairielearn_grader/plugin.py | 33 ++++++++++++++++- .../expected_outcome.json | 4 +-- .../test_helper_modules_workspace/scenario.py | 7 ++-- .../student/helper_function.py | 13 ++++++- .../__init__.py | 0 .../expected_outcome.json | 16 +++++++++ .../helper_function.py | 1 + .../scenario.py | 13 +++++++ .../student/helper_function.py | 1 + .../student/main.py | 1 + tests/test_helper_name_clashes.py | 35 +++++++++++++++++++ 12 files changed, 123 insertions(+), 12 deletions(-) create mode 100644 tests/scenario_root/test_helper_modules_workspace_name_clash/__init__.py create mode 100644 tests/scenario_root/test_helper_modules_workspace_name_clash/expected_outcome.json create mode 100644 tests/scenario_root/test_helper_modules_workspace_name_clash/helper_function.py create mode 100644 tests/scenario_root/test_helper_modules_workspace_name_clash/scenario.py create mode 100644 tests/scenario_root/test_helper_modules_workspace_name_clash/student/helper_function.py create mode 100644 tests/scenario_root/test_helper_modules_workspace_name_clash/student/main.py create mode 100644 tests/test_helper_name_clashes.py diff --git a/quick_start.md b/quick_start.md index 5ec1402..ccd1a91 100644 --- a/quick_start.md +++ b/quick_start.md @@ -380,10 +380,13 @@ import helper_function as hf message says what's wrong. - **Fixtures:** it works with `sandbox` and `module_sandbox`, for `.py` and `.ipynb` submissions, and with `workspace_sandbox`. -- **Workspace mode:** an `import` of a listed module gets the instructor's file, even if the student - submits one with the same name, and it keeps the restricted builtins that the student's own modules - don't have. Dotted queries such as `query_function("helper_function.triple")` only search the - student's project, so they won't call a helper module. +- **Workspace mode:** students can have several files that import each other, so a listed name is + reserved. If a student submits a different `helper_function.py` or a `helper_function/` folder, the + submission fails with a message asking them to rename it and counts as an attempt. An identical copy + is fine, so you can give students one to work with. A listed module keeps the restricted builtins + that the student's own modules don't have. Dotted queries such as + `query_function("helper_function.triple")` only search the student's project, so they won't call a + helper module. > **Warning:** Students can read everything in a helper module, so it must not contain answers. This > setting also doesn't make the rest of the tests folder private. `importlib` isn't blacklisted by diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index b460d3c..1b23816 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -131,6 +131,27 @@ def _load_helper_modules( return sources +def _find_helper_name_clashes(workspace_dir: Path, helper_modules: dict[str, str] | None) -> list[str]: + """ + Return the student's files and folders that an import of a helper module would silently replace. + + An identical copy isn't a clash, since instructors may give students one to work with. + """ + clashes: list[str] = [] + for name, source in (helper_modules or {}).items(): + file_path = workspace_dir / f"{name}.py" + if (workspace_dir / name).is_dir(): + clashes.append(f"{name}/") + elif file_path.is_file(): + try: + is_copy = file_path.read_text(encoding="utf-8") == source + except (OSError, UnicodeDecodeError): + is_copy = False + if not is_copy: + clashes.append(file_path.name) + return clashes + + def _initialize_sandbox_fixture( request: pytest.FixtureRequest, data_json: dict[str, Any] | None, @@ -537,6 +558,16 @@ def _initialize_workspace_fixture( data_dir = get_datadir(request.module) setup_code_file = (data_dir / "setup_code.py") if data_dir is not None else Path("setup_code.py") + helper_modules = _load_helper_modules(request, helper_module_names, import_blacklist) + clashes = _find_helper_name_clashes(workspace_dir, helper_modules) + if clashes: + # On one line, because the default output level only shows the first line of a failure. + pytest.fail( + f"These names are reserved for modules that this question provides: {', '.join(clashes)}. " + "Rename yours and update any imports that use them.", + pytrace=False, + ) + fixture = WorkspaceFixture( workspace_dir=workspace_dir, setup_code_file=setup_code_file, @@ -547,7 +578,7 @@ def _initialize_workspace_fixture( names_for_user_list=names_for_user_list, worker_username=request.config.getoption("--worker-username"), exec_entry=exec_entry, - helper_modules=_load_helper_modules(request, helper_module_names, import_blacklist), + helper_modules=helper_modules, ) return fixture, initialization_timeout diff --git a/tests/scenario_root/test_helper_modules_workspace/expected_outcome.json b/tests/scenario_root/test_helper_modules_workspace/expected_outcome.json index 9d8a9af..590f124 100644 --- a/tests/scenario_root/test_helper_modules_workspace/expected_outcome.json +++ b/tests/scenario_root/test_helper_modules_workspace/expected_outcome.json @@ -32,11 +32,11 @@ { "max_points": 1, "message": "", - "name": "helper module beats student file", + "name": "identical copy of a helper module", "outcome": "passed", "points": 1.0, "points_frac": 1.0, - "test_id": "test_helper_modules_workspace.py::test_student_file_with_same_name" + "test_id": "test_helper_modules_workspace.py::test_identical_copy" }, { "max_points": 1, diff --git a/tests/scenario_root/test_helper_modules_workspace/scenario.py b/tests/scenario_root/test_helper_modules_workspace/scenario.py index af9c0d1..d8ab9cd 100644 --- a/tests/scenario_root/test_helper_modules_workspace/scenario.py +++ b/tests/scenario_root/test_helper_modules_workspace/scenario.py @@ -1,4 +1,4 @@ -"""Helper modules in workspace mode, including a student file with the same name as one.""" +"""Helper modules in workspace mode, including a student's identical copy of one.""" import pytest @@ -32,8 +32,8 @@ def test_student_module(workspace_sandbox: WorkspaceFixture) -> None: assert workspace_sandbox.query_function("calculator.add_one", 2) == 3 -@pytest.mark.grading_data(name="helper module beats student file", points=1) -def test_student_file_with_same_name(workspace_sandbox: WorkspaceFixture) -> None: +@pytest.mark.grading_data(name="identical copy of a helper module", points=1) +def test_identical_copy(workspace_sandbox: WorkspaceFixture) -> None: assert workspace_sandbox.query_function("calculator.source") == "instructor" @@ -58,5 +58,4 @@ def test_whitelist(workspace_sandbox: WorkspaceFixture) -> None: @pytest.mark.grading_data(name="dotted queries search the student's project", points=1) def test_dotted_queries(workspace_sandbox: WorkspaceFixture) -> None: - assert workspace_sandbox.query("helper_function.SOURCE") == "student" assert workspace_sandbox.query_function_raw("second_helper.add_one", 1)["status"] != "success" diff --git a/tests/scenario_root/test_helper_modules_workspace/student/helper_function.py b/tests/scenario_root/test_helper_modules_workspace/student/helper_function.py index 5ddbe32..a39533e 100644 --- a/tests/scenario_root/test_helper_modules_workspace/student/helper_function.py +++ b/tests/scenario_root/test_helper_modules_workspace/student/helper_function.py @@ -1 +1,12 @@ -SOURCE = "student" +SCALE = 3 + + +def triple(x): + return SCALE * x + + +SOURCE = "instructor" + + +def total(values): + return sum(values) diff --git a/tests/scenario_root/test_helper_modules_workspace_name_clash/__init__.py b/tests/scenario_root/test_helper_modules_workspace_name_clash/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_helper_modules_workspace_name_clash/expected_outcome.json b/tests/scenario_root/test_helper_modules_workspace_name_clash/expected_outcome.json new file mode 100644 index 0000000..f4a2c60 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace_name_clash/expected_outcome.json @@ -0,0 +1,16 @@ +{ + "expected_data_object": { + "score": 0.0, + "tests": [ + { + "test_id": "test_helper_modules_workspace_name_clash.py::test_query", + "outcome": "error", + "message": "These names are reserved for modules that this question provides: helper_function.py.", + "max_points": 1, + "points": 0.0, + "points_frac": 0.0, + "name": "query" + } + ] + } +} diff --git a/tests/scenario_root/test_helper_modules_workspace_name_clash/helper_function.py b/tests/scenario_root/test_helper_modules_workspace_name_clash/helper_function.py new file mode 100644 index 0000000..1fecb4d --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace_name_clash/helper_function.py @@ -0,0 +1 @@ +SOURCE = "instructor" diff --git a/tests/scenario_root/test_helper_modules_workspace_name_clash/scenario.py b/tests/scenario_root/test_helper_modules_workspace_name_clash/scenario.py new file mode 100644 index 0000000..9c2b0e1 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace_name_clash/scenario.py @@ -0,0 +1,13 @@ +"""A student file with the same name as a helper module gets a message asking them to rename it.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import WorkspaceFixture + +autograder_config = ConfigObject(workspace_mode=True, workspace_exec_entry="main.py", helper_modules=["helper_function"]) + + +@pytest.mark.grading_data(name="query", points=1) +def test_query(workspace_sandbox: WorkspaceFixture) -> None: + assert workspace_sandbox.query("VALUE") == 1 diff --git a/tests/scenario_root/test_helper_modules_workspace_name_clash/student/helper_function.py b/tests/scenario_root/test_helper_modules_workspace_name_clash/student/helper_function.py new file mode 100644 index 0000000..5ddbe32 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace_name_clash/student/helper_function.py @@ -0,0 +1 @@ +SOURCE = "student" diff --git a/tests/scenario_root/test_helper_modules_workspace_name_clash/student/main.py b/tests/scenario_root/test_helper_modules_workspace_name_clash/student/main.py new file mode 100644 index 0000000..b15b1b0 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace_name_clash/student/main.py @@ -0,0 +1 @@ +VALUE = 1 diff --git a/tests/test_helper_name_clashes.py b/tests/test_helper_name_clashes.py new file mode 100644 index 0000000..987a08b --- /dev/null +++ b/tests/test_helper_name_clashes.py @@ -0,0 +1,35 @@ +"""Which student files count as clashing with a helper module.""" + +from pathlib import Path + +from pytest_prairielearn_grader.plugin import _find_helper_name_clashes + + +def test_no_helper_modules(tmp_path: Path) -> None: + (tmp_path / "helper.py").write_text("x = 1\n") + assert _find_helper_name_clashes(tmp_path, None) == [] + + +def test_unrelated_file(tmp_path: Path) -> None: + (tmp_path / "other.py").write_text("x = 1\n") + assert _find_helper_name_clashes(tmp_path, {"helper": "x = 1\n"}) == [] + + +def test_identical_copy(tmp_path: Path) -> None: + (tmp_path / "helper.py").write_text("x = 1\n") + assert _find_helper_name_clashes(tmp_path, {"helper": "x = 1\n"}) == [] + + +def test_different_file(tmp_path: Path) -> None: + (tmp_path / "helper.py").write_text("x = 2\n") + assert _find_helper_name_clashes(tmp_path, {"helper": "x = 1\n"}) == ["helper.py"] + + +def test_file_that_isnt_utf8(tmp_path: Path) -> None: + (tmp_path / "helper.py").write_bytes(b"\xff") + assert _find_helper_name_clashes(tmp_path, {"helper": "x = 1\n"}) == ["helper.py"] + + +def test_folder(tmp_path: Path) -> None: + (tmp_path / "helper").mkdir() + assert _find_helper_name_clashes(tmp_path, {"helper": "x = 1\n"}) == ["helper/"] From 6111ade2fe3cdfd90a944a3c6d3f8aacf4e058d1 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Mon, 9 Nov 2026 11:20:28 -0500 Subject: [PATCH 4/5] Drop the runtime-written test_student.py from the helper scenario import_scenario already checks that student code can't import the grader's test file, which sits next to the data folder like test_student.py does in a real question. The written file was just another unnamed file in the data folder, which import_ans covers. Co-Authored-By: Claude Opus 5.5 --- .../test_helper_modules/expected_outcome.json | 9 --------- tests/scenario_root/test_helper_modules/scenario.py | 7 +------ tests/scenario_root/test_helper_modules/student_code.py | 6 ------ 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/tests/scenario_root/test_helper_modules/expected_outcome.json b/tests/scenario_root/test_helper_modules/expected_outcome.json index 2d42b00..01f6ae1 100644 --- a/tests/scenario_root/test_helper_modules/expected_outcome.json +++ b/tests/scenario_root/test_helper_modules/expected_outcome.json @@ -65,15 +65,6 @@ "points_frac": 1.0, "test_id": "test_helper_modules.py::test_not_named[student_code-import_ans]" }, - { - "max_points": 1, - "message": "", - "name": "modules that aren't named can't be imported", - "outcome": "passed", - "points": 1.0, - "points_frac": 1.0, - "test_id": "test_helper_modules.py::test_not_named[student_code-import_test_student]" - }, { "max_points": 1, "message": "", diff --git a/tests/scenario_root/test_helper_modules/scenario.py b/tests/scenario_root/test_helper_modules/scenario.py index 480a6cb..cf4343b 100644 --- a/tests/scenario_root/test_helper_modules/scenario.py +++ b/tests/scenario_root/test_helper_modules/scenario.py @@ -1,15 +1,10 @@ """Modules named by helper_modules can be imported, and nothing else in the tests folder can.""" -from pathlib import Path - import pytest from pytest_prairielearn_grader import ConfigObject from pytest_prairielearn_grader.fixture import StudentFixture -# Written here because the outer test run would collect a file with this name in the scenario folder. -Path(__file__).with_suffix("").joinpath("test_student.py").write_text('ANSWER = "the reference answer"\n') - autograder_config = ConfigObject( helper_modules=["helper_function", "second_helper"], sandbox_timeout=5, @@ -51,7 +46,7 @@ def test_second_helper(sandbox: StudentFixture) -> None: @pytest.mark.grading_data(name="modules that aren't named can't be imported", points=1) @pytest.mark.parametrize( "function_name", - ["import_ans", "import_test_student", "import_setup_code", "import_scenario", "helper_imports_ans"], + ["import_ans", "import_setup_code", "import_scenario", "helper_imports_ans"], ) def test_not_named(sandbox: StudentFixture, function_name: str) -> None: response = sandbox.query_function_raw(function_name) diff --git a/tests/scenario_root/test_helper_modules/student_code.py b/tests/scenario_root/test_helper_modules/student_code.py index 3bd33dc..04d2d88 100644 --- a/tests/scenario_root/test_helper_modules/student_code.py +++ b/tests/scenario_root/test_helper_modules/student_code.py @@ -34,12 +34,6 @@ def import_ans(): return ans.ANSWER -def import_test_student(): - import test_student - - return test_student.ANSWER - - def import_setup_code(): import setup_code From f48b370bad9b6f83498b92bf18d2edcdebbdd689 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Mon, 9 Nov 2026 11:58:44 -0500 Subject: [PATCH 5/5] Reserve helper module names outright and trim the helper tests A student file or folder named like a helper module now always fails with the rename message, without the exception for an identical copy. Fix the config docstring and the Copilot notes, which still said a helper took priority over a student file. Drop the workspace missing-helper scenario, which went through the same code as the regular one. Co-Authored-By: Claude Opus 5.5 --- .github/copilot-instructions.md | 4 ++-- quick_start.md | 6 +++--- src/pytest_prairielearn_grader/config.py | 5 +++-- src/pytest_prairielearn_grader/plugin.py | 18 ++++-------------- .../expected_outcome.json | 9 --------- .../helper_function.py | 3 --- .../test_helper_modules_workspace/scenario.py | 7 +------ .../student/calculator.py | 4 ---- .../student/helper_function.py | 12 ------------ .../__init__.py | 0 .../expected_outcome.json | 13 ------------- .../scenario.py | 13 ------------- .../student/calculator.py | 1 - tests/test_helper_name_clashes.py | 12 +----------- 14 files changed, 14 insertions(+), 93 deletions(-) delete mode 100644 tests/scenario_root/test_helper_modules_workspace/student/helper_function.py delete mode 100644 tests/scenario_root/test_helper_modules_workspace_missing/__init__.py delete mode 100644 tests/scenario_root/test_helper_modules_workspace_missing/expected_outcome.json delete mode 100644 tests/scenario_root/test_helper_modules_workspace_missing/scenario.py delete mode 100644 tests/scenario_root/test_helper_modules_workspace_missing/student/calculator.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 64cc9ae..c939487 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -143,8 +143,8 @@ result = sandbox.query_function("student_func", arg1, arg2, timeout=5) `ConfigObject(helper_modules=["helper_function"])` lets setup, leading and student code import `helper_function.py` from the tests folder, which isn't otherwise on the import path. Students can read -these modules, so they must not contain answers. In workspace mode, they take priority over student files -with the same name when imported. +these modules, so they must not contain answers. In workspace mode, their names are reserved: a student +file or folder with one fails the submission with a message asking the student to rename it. ### 5. Serialization diff --git a/quick_start.md b/quick_start.md index ccd1a91..951b8e1 100644 --- a/quick_start.md +++ b/quick_start.md @@ -381,9 +381,9 @@ import helper_function as hf - **Fixtures:** it works with `sandbox` and `module_sandbox`, for `.py` and `.ipynb` submissions, and with `workspace_sandbox`. - **Workspace mode:** students can have several files that import each other, so a listed name is - reserved. If a student submits a different `helper_function.py` or a `helper_function/` folder, the - submission fails with a message asking them to rename it and counts as an attempt. An identical copy - is fine, so you can give students one to work with. A listed module keeps the restricted builtins + reserved. If a student submits a `helper_function.py` or a `helper_function/` folder, the + submission fails with a message asking them to rename it and counts as an attempt. Don't give + students a copy of a helper module in their workspace. A listed module keeps the restricted builtins that the student's own modules don't have. Dotted queries such as `query_function("helper_function.triple")` only search the student's project, so they won't call a helper module. diff --git a/src/pytest_prairielearn_grader/config.py b/src/pytest_prairielearn_grader/config.py index 79a31fb..1610344 100644 --- a/src/pytest_prairielearn_grader/config.py +++ b/src/pytest_prairielearn_grader/config.py @@ -77,8 +77,9 @@ def test_student_code(sandbox): also blocks is a configuration error. A helper module runs in the sandbox with the same builtins and import rules as student code. - In workspace mode, a helper module takes priority over a student file with the same name - when imported, and it keeps these restricted builtins, unlike the student's own modules. + In workspace mode, a listed name is reserved: a student file or folder with that name fails + the submission with a message asking the student to rename it. A helper module keeps these + restricted builtins, unlike the student's own modules. Students can read everything in a helper module, so it must not contain answers. diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 1b23816..6b51b67 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -132,23 +132,13 @@ def _load_helper_modules( def _find_helper_name_clashes(workspace_dir: Path, helper_modules: dict[str, str] | None) -> list[str]: - """ - Return the student's files and folders that an import of a helper module would silently replace. - - An identical copy isn't a clash, since instructors may give students one to work with. - """ + """Return the student's files and folders that an import of a helper module would silently replace.""" clashes: list[str] = [] - for name, source in (helper_modules or {}).items(): - file_path = workspace_dir / f"{name}.py" + for name in helper_modules or {}: if (workspace_dir / name).is_dir(): clashes.append(f"{name}/") - elif file_path.is_file(): - try: - is_copy = file_path.read_text(encoding="utf-8") == source - except (OSError, UnicodeDecodeError): - is_copy = False - if not is_copy: - clashes.append(file_path.name) + elif (workspace_dir / f"{name}.py").is_file(): + clashes.append(f"{name}.py") return clashes diff --git a/tests/scenario_root/test_helper_modules_workspace/expected_outcome.json b/tests/scenario_root/test_helper_modules_workspace/expected_outcome.json index 590f124..ff693e0 100644 --- a/tests/scenario_root/test_helper_modules_workspace/expected_outcome.json +++ b/tests/scenario_root/test_helper_modules_workspace/expected_outcome.json @@ -29,15 +29,6 @@ "points_frac": 1.0, "test_id": "test_helper_modules_workspace.py::test_student_module" }, - { - "max_points": 1, - "message": "", - "name": "identical copy of a helper module", - "outcome": "passed", - "points": 1.0, - "points_frac": 1.0, - "test_id": "test_helper_modules_workspace.py::test_identical_copy" - }, { "max_points": 1, "message": "", diff --git a/tests/scenario_root/test_helper_modules_workspace/helper_function.py b/tests/scenario_root/test_helper_modules_workspace/helper_function.py index a39533e..db7b74a 100644 --- a/tests/scenario_root/test_helper_modules_workspace/helper_function.py +++ b/tests/scenario_root/test_helper_modules_workspace/helper_function.py @@ -5,8 +5,5 @@ def triple(x): return SCALE * x -SOURCE = "instructor" - - def total(values): return sum(values) diff --git a/tests/scenario_root/test_helper_modules_workspace/scenario.py b/tests/scenario_root/test_helper_modules_workspace/scenario.py index d8ab9cd..9cc7ac6 100644 --- a/tests/scenario_root/test_helper_modules_workspace/scenario.py +++ b/tests/scenario_root/test_helper_modules_workspace/scenario.py @@ -1,4 +1,4 @@ -"""Helper modules in workspace mode, including a student's identical copy of one.""" +"""Helper modules in workspace mode.""" import pytest @@ -32,11 +32,6 @@ def test_student_module(workspace_sandbox: WorkspaceFixture) -> None: assert workspace_sandbox.query_function("calculator.add_one", 2) == 3 -@pytest.mark.grading_data(name="identical copy of a helper module", points=1) -def test_identical_copy(workspace_sandbox: WorkspaceFixture) -> None: - assert workspace_sandbox.query_function("calculator.source") == "instructor" - - @pytest.mark.grading_data(name="builtins", points=1) def test_builtins(workspace_sandbox: WorkspaceFixture) -> None: assert workspace_sandbox.query_function("calculator.own_sum", [1, 2]) == 3 diff --git a/tests/scenario_root/test_helper_modules_workspace/student/calculator.py b/tests/scenario_root/test_helper_modules_workspace/student/calculator.py index e0575e9..52a8c0e 100644 --- a/tests/scenario_root/test_helper_modules_workspace/student/calculator.py +++ b/tests/scenario_root/test_helper_modules_workspace/student/calculator.py @@ -6,10 +6,6 @@ def triple_it(x): return helper_function.triple(x) -def source(): - return helper_function.SOURCE - - def add_one(x): return second_helper.add_one(x) diff --git a/tests/scenario_root/test_helper_modules_workspace/student/helper_function.py b/tests/scenario_root/test_helper_modules_workspace/student/helper_function.py deleted file mode 100644 index a39533e..0000000 --- a/tests/scenario_root/test_helper_modules_workspace/student/helper_function.py +++ /dev/null @@ -1,12 +0,0 @@ -SCALE = 3 - - -def triple(x): - return SCALE * x - - -SOURCE = "instructor" - - -def total(values): - return sum(values) diff --git a/tests/scenario_root/test_helper_modules_workspace_missing/__init__.py b/tests/scenario_root/test_helper_modules_workspace_missing/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/scenario_root/test_helper_modules_workspace_missing/expected_outcome.json b/tests/scenario_root/test_helper_modules_workspace_missing/expected_outcome.json deleted file mode 100644 index acf260f..0000000 --- a/tests/scenario_root/test_helper_modules_workspace_missing/expected_outcome.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "expected_data_object": { - "gradable": false, - "message": "The autograder encountered an internal error.", - "tests": [ - { - "test_id": "test_helper_modules_workspace_missing.py::test_query", - "outcome": "error", - "message": "Autograder configuration error in helper_modules: can't use 'helper_function':" - } - ] - } -} diff --git a/tests/scenario_root/test_helper_modules_workspace_missing/scenario.py b/tests/scenario_root/test_helper_modules_workspace_missing/scenario.py deleted file mode 100644 index 3999ef4..0000000 --- a/tests/scenario_root/test_helper_modules_workspace_missing/scenario.py +++ /dev/null @@ -1,13 +0,0 @@ -"""A named module with no file makes a workspace submission ungradable too.""" - -import pytest - -from pytest_prairielearn_grader import ConfigObject -from pytest_prairielearn_grader.fixture import WorkspaceFixture - -autograder_config = ConfigObject(workspace_mode=True, helper_modules=["helper_function"]) - - -@pytest.mark.grading_data(name="query", points=1) -def test_query(workspace_sandbox: WorkspaceFixture) -> None: - assert workspace_sandbox.query("calculator.VALUE") == 1 diff --git a/tests/scenario_root/test_helper_modules_workspace_missing/student/calculator.py b/tests/scenario_root/test_helper_modules_workspace_missing/student/calculator.py deleted file mode 100644 index b15b1b0..0000000 --- a/tests/scenario_root/test_helper_modules_workspace_missing/student/calculator.py +++ /dev/null @@ -1 +0,0 @@ -VALUE = 1 diff --git a/tests/test_helper_name_clashes.py b/tests/test_helper_name_clashes.py index 987a08b..d3f9732 100644 --- a/tests/test_helper_name_clashes.py +++ b/tests/test_helper_name_clashes.py @@ -15,18 +15,8 @@ def test_unrelated_file(tmp_path: Path) -> None: assert _find_helper_name_clashes(tmp_path, {"helper": "x = 1\n"}) == [] -def test_identical_copy(tmp_path: Path) -> None: +def test_file(tmp_path: Path) -> None: (tmp_path / "helper.py").write_text("x = 1\n") - assert _find_helper_name_clashes(tmp_path, {"helper": "x = 1\n"}) == [] - - -def test_different_file(tmp_path: Path) -> None: - (tmp_path / "helper.py").write_text("x = 2\n") - assert _find_helper_name_clashes(tmp_path, {"helper": "x = 1\n"}) == ["helper.py"] - - -def test_file_that_isnt_utf8(tmp_path: Path) -> None: - (tmp_path / "helper.py").write_bytes(b"\xff") assert _find_helper_name_clashes(tmp_path, {"helper": "x = 1\n"}) == ["helper.py"]