diff --git a/quick_start.md b/quick_start.md index 951b8e1..fcdfa53 100644 --- a/quick_start.md +++ b/quick_start.md @@ -1502,6 +1502,25 @@ def test_global_state(workspace_sandbox: WorkspaceFixture) -> None: Without `workspace_exec_entry` (the default), no file is executed at startup and all access goes through the import machinery. +### Student Files Named Like Loaded Modules + +The sandbox loads some modules for itself before any student or setup code runs, such as +`random`, `json`, `types`, `typing`, `logging` and `math`. Python reuses a module that's already +loaded, so a student file named `random.py` (or a folder named `json/`) at the top of the project +would be silently ignored in favor of the standard library module. + +Instead, startup fails and every test is an error worth 0 points, with a message listing the files +and asking the student to rename them: + +```text +Student code execution failed with an exception: ImportError +Exception Message: These names are already used by modules that the grader has loaded: json/, random.py. Rename yours and update any imports that use them. +``` + +Only top-level `.py` files and folders are checked, and only against modules the sandbox has +actually loaded. A file such as `calendar.py` or `utils.py` that doesn't match a loaded module is +imported normally. The list of loaded modules can change between Python versions. + ### ConfigObject Settings for Workspaces | Parameter | Type | Default | Description | diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 7987e8d..6e43c1b 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -265,6 +265,60 @@ def custom_import( return custom_import +def _module_is_from_dir(module: types.ModuleType, directory: str) -> bool: + """Whether *module* was loaded from a file or package inside *directory*.""" + locations: list[str] = [] + module_file = getattr(module, "__file__", None) + if isinstance(module_file, str): + locations.append(module_file) + module_path = getattr(module, "__path__", None) + if module_path is not None: + try: + locations.extend(str(p) for p in module_path) + except TypeError: + pass + for location in locations: + real_location = os.path.realpath(location) + if real_location == directory or real_location.startswith(directory + os.sep): + return True + return False + + +def _find_shadowed_workspace_files(workspace_dir: str) -> list[str]: + """ + Return the top-level ``.py`` files and folders in *workspace_dir* whose names match a module + that this process has already loaded from somewhere else. + + Python reuses an already loaded module on import, so the student's file would be silently + ignored. Folders are listed with a trailing slash. + """ + real_dir = os.path.realpath(workspace_dir) + clashes: list[str] = [] + for entry in sorted(os.listdir(real_dir)): + path = os.path.join(real_dir, entry) + if entry.endswith(".py") and os.path.isfile(path): + name, label = entry[:-3], entry + elif os.path.isdir(path): + name, label = entry, entry + "/" + else: + continue + module = sys.modules.get(name) + if module is not None and not _module_is_from_dir(module, real_dir): + clashes.append(label) + return clashes + + +def _find_helper_name_clashes(workspace_dir: str, helper_modules: dict[str, str] | None) -> list[str]: + """Return the student's files and folders that an import of a helper module would silently replace.""" + clashes: list[str] = [] + for name in helper_modules or {}: + if os.path.isdir(os.path.join(workspace_dir, name)): + clashes.append(f"{name}/") + elif os.path.isfile(os.path.join(workspace_dir, f"{name}.py")): + clashes.append(f"{name}.py") + return clashes + + async def student_code_runner( setup_code: str | None, student_code: str, @@ -385,6 +439,11 @@ async def workspace_runner( executes an entry-point file (exec_entry) and runs setup_code exactly as the regular student_code_runner does. """ + # Check before anything else runs, so the result depends only on what the sandbox loads + # for itself and not on what the setup code or the student's code imports. + helper_clashes = _find_helper_name_clashes(workspace_dir, helper_modules) + shadowed_files = [f for f in _find_shadowed_workspace_files(workspace_dir) if f not in helper_clashes] + stdout_capture = io.StringIO() stderr_capture = io.StringIO() execution_error: Exception | None = None @@ -392,6 +451,23 @@ async def workspace_runner( local_vars = deepcopy(starting_vars) if starting_vars else {} local_vars["__from_server_json"] = from_server_json + if helper_clashes or shadowed_files: + reasons = [] + if helper_clashes: + reasons.append(f"These names are reserved for modules that this question provides: {', '.join(helper_clashes)}.") + if shadowed_files: + reasons.append(f"These names are already used by modules that the grader has loaded: {', '.join(shadowed_files)}.") + clash_error = ImportError(" ".join(reasons) + " Rename yours and update any imports that use them.") + start_response: ProcessStartResponse = { + "status": ProcessStatusCode.EXCEPTION, + "stdout": "", + "stderr": "", + "execution_error": type(clash_error).__name__, + "execution_message": str(clash_error), + "execution_traceback": "", + } + return local_vars, {}, start_response + student_code_vars: dict[str, Any] = {} student_code_vars["__builtins__"] = get_builtins(builtin_whitelist) student_code_vars["__builtins__"]["__name__"] = "__main__" diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 6b51b67..93e5ce4 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -15,6 +15,7 @@ import _pytest.terminal import pytest from _pytest.config import Config +from _pytest.outcomes import Failed from prettytable import PrettyTable from .config import ConfigObject @@ -34,6 +35,19 @@ DEFAULT_IMPORT_BLACKLIST = ["os", "sys", "subprocess", "pathlib", "shutil"] +class SandboxStartupFailed(Failed): + """ + Raised when the student code sandbox fails to start (an exception, timeout or no response + while running student code at startup). + + The message is already formatted for the test's output level, so the result collector + shows it to the student as it is instead of wrapping it in another exception header. + """ + + def __init__(self, msg: str | None = None, pytrace: bool = False) -> None: + super().__init__(msg, pytrace) + + class TestResult(NamedTuple): """Container for test execution results.""" @@ -127,21 +141,10 @@ def _load_helper_modules( except (OSError, ValueError, SyntaxError) as e: message = f"Autograder configuration error in helper_modules: can't use '{name}': {e}" request.config.result_collector_plugin.collection_errors.append(message) # type: ignore[attr-defined] - pytest.fail(message, pytrace=False) + raise SandboxStartupFailed(message) from e return sources -def _find_helper_name_clashes(workspace_dir: Path, helper_modules: dict[str, str] | None) -> list[str]: - """Return the student's files and folders that an import of a helper module would silently replace.""" - clashes: list[str] = [] - for name in helper_modules or {}: - if (workspace_dir / name).is_dir(): - clashes.append(f"{name}/") - elif (workspace_dir / f"{name}.py").is_file(): - clashes.append(f"{name}.py") - return clashes - - def _initialize_sandbox_fixture( request: pytest.FixtureRequest, data_json: dict[str, Any] | None, @@ -288,29 +291,33 @@ def _handle_sandbox_startup_errors( if exception_name == "SyntaxError": _record_syntax_error_as_ungradable(request, response) + exception_message = response.get("execution_message", "") or "" + if output_level == GradingOutputLevel.Friendly: - exception_message = response.get("execution_message", "") or "" - pytest.fail(exception_message or str(exception_name), pytrace=False) + raise SandboxStartupFailed(exception_message or str(exception_name)) fail_message = f"Student code execution failed with an exception: {exception_name}" - exception_message = response.get("execution_message", "") or "" + + if output_level == GradingOutputLevel.ExceptionName: + raise SandboxStartupFailed(fail_message) + fail_message += f"{os.linesep}Exception Message: {exception_message}" if output_level == GradingOutputLevel.ExceptionMessage: - pytest.fail(fail_message, pytrace=False) + raise SandboxStartupFailed(fail_message) assert output_level == GradingOutputLevel.FullTraceback if exception_traceback := response.get("execution_traceback", ""): fail_message += f"{os.linesep * 2}{exception_traceback}" - pytest.fail(fail_message, pytrace=False) + raise SandboxStartupFailed(fail_message) elif response_status == ProcessStatusCode.TIMEOUT: - pytest.fail("Student code initialization timed out", pytrace=False) + raise SandboxStartupFailed("Student code initialization timed out") elif response_status == ProcessStatusCode.NO_RESPONSE: - pytest.fail(f"No response from initialization with timeout {initialization_timeout}", pytrace=False) + raise SandboxStartupFailed(f"No response from initialization with timeout {initialization_timeout}") elif response_status != ProcessStatusCode.SUCCESS: logger.warning(f"Unexpected status in response from student code server: {response}") @@ -548,16 +555,6 @@ def _initialize_workspace_fixture( data_dir = get_datadir(request.module) setup_code_file = (data_dir / "setup_code.py") if data_dir is not None else Path("setup_code.py") - helper_modules = _load_helper_modules(request, helper_module_names, import_blacklist) - clashes = _find_helper_name_clashes(workspace_dir, helper_modules) - if clashes: - # On one line, because the default output level only shows the first line of a failure. - pytest.fail( - f"These names are reserved for modules that this question provides: {', '.join(clashes)}. " - "Rename yours and update any imports that use them.", - pytrace=False, - ) - fixture = WorkspaceFixture( workspace_dir=workspace_dir, setup_code_file=setup_code_file, @@ -568,7 +565,7 @@ def _initialize_workspace_fixture( names_for_user_list=names_for_user_list, worker_username=request.config.getoption("--worker-username"), exec_entry=exec_entry, - helper_modules=helper_modules, + helper_modules=_load_helper_modules(request, helper_module_names, import_blacklist), ) return fixture, initialization_timeout @@ -985,7 +982,12 @@ def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> Iter logger.debug(f"Grading output level set to: {output_level}") - if output_level == GradingOutputLevel.Friendly: + if isinstance(call.excinfo.value, SandboxStartupFailed): + # The sandbox startup handler already formatted this message for the + # output level, so show it in full without another exception header. + feedback_obj.add_message(str(call.excinfo.value)) + + elif output_level == GradingOutputLevel.Friendly: # Friendly mode: show only the exception message text with # no exception class name, traceback, or "failed with" prefix. exception_message = str(call.excinfo.value) diff --git a/tests/scenario_root/test_helper_modules_workspace_name_clash/expected_outcome.json b/tests/scenario_root/test_helper_modules_workspace_name_clash/expected_outcome.json index f4a2c60..554fa06 100644 --- a/tests/scenario_root/test_helper_modules_workspace_name_clash/expected_outcome.json +++ b/tests/scenario_root/test_helper_modules_workspace_name_clash/expected_outcome.json @@ -5,7 +5,7 @@ { "test_id": "test_helper_modules_workspace_name_clash.py::test_query", "outcome": "error", - "message": "These names are reserved for modules that this question provides: helper_function.py.", + "message": "Student code execution failed with an exception: ImportError\nException Message: These names are reserved for modules that this question provides: helper_function.py. These names are already used by modules that the grader has loaded: random.py. Rename yours and update any imports that use them.", "max_points": 1, "points": 0.0, "points_frac": 0.0, diff --git a/tests/scenario_root/test_helper_modules_workspace_name_clash/scenario.py b/tests/scenario_root/test_helper_modules_workspace_name_clash/scenario.py index 9c2b0e1..2ca1aab 100644 --- a/tests/scenario_root/test_helper_modules_workspace_name_clash/scenario.py +++ b/tests/scenario_root/test_helper_modules_workspace_name_clash/scenario.py @@ -1,4 +1,4 @@ -"""A student file with the same name as a helper module gets a message asking them to rename it.""" +"""A student file named like a helper module, or like a module the grader has loaded, gets a rename message.""" import pytest diff --git a/tests/scenario_root/test_helper_modules_workspace_name_clash/student/random.py b/tests/scenario_root/test_helper_modules_workspace_name_clash/student/random.py new file mode 100644 index 0000000..5ddbe32 --- /dev/null +++ b/tests/scenario_root/test_helper_modules_workspace_name_clash/student/random.py @@ -0,0 +1 @@ +SOURCE = "student" diff --git a/tests/scenario_root/test_private_namespace/expected_outcome.json b/tests/scenario_root/test_private_namespace/expected_outcome.json index c3d055c..2f2879c 100644 --- a/tests/scenario_root/test_private_namespace/expected_outcome.json +++ b/tests/scenario_root/test_private_namespace/expected_outcome.json @@ -4,7 +4,7 @@ "tests": [ { "max_points": 1, - "message": "Student code execution failed with an exception: Failed\nException Message: Student code execution failed with an exception: NameError", + "message": "Student code execution failed with an exception: NameError\nException Message: name 'other_secret_value' is not defined", "name": "test_private_namespace.py::test_random[student_code_fail_other]", "outcome": "error", "points": 0.0, @@ -13,7 +13,7 @@ }, { "max_points": 1, - "message": "Student code execution failed with an exception: Failed\nException Message: Student code execution failed with an exception: NameError", + "message": "Student code execution failed with an exception: NameError\nException Message: name 'super_secret_value' is not defined", "name": "test_private_namespace.py::test_random[student_code_fail]", "outcome": "error", "points": 0.0, @@ -22,7 +22,7 @@ }, { "max_points": 1, - "message": "Student code execution failed with an exception: Failed\nException Message: Student code execution failed with an exception: NameError", + "message": "Student code execution failed with an exception: NameError\nException Message: name '__data_params' is not defined", "name": "test_private_namespace.py::test_random[student_code_fail_read_builtin]", "outcome": "error", "points": 0.0, diff --git a/tests/scenario_root/test_startup_exception_message/__init__.py b/tests/scenario_root/test_startup_exception_message/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_startup_exception_message/expected_outcome.json b/tests/scenario_root/test_startup_exception_message/expected_outcome.json new file mode 100644 index 0000000..354a744 --- /dev/null +++ b/tests/scenario_root/test_startup_exception_message/expected_outcome.json @@ -0,0 +1,31 @@ +{ + "expected_data_object": { + "score": 0.0, + "tests": [ + { + "test_id": "test_startup_exception_message.py::test_default_level[student_code]", + "message": "Student code execution failed with an exception: ValueError\nException Message: first line of the reason\nsecond line of the reason", + "outcome": "error", + "max_points": 1, + "points": 0.0, + "points_frac": 0.0 + }, + { + "test_id": "test_startup_exception_message.py::test_name_level[student_code]", + "message": "Student code execution failed with an exception: ValueError", + "outcome": "error", + "max_points": 1, + "points": 0.0, + "points_frac": 0.0 + }, + { + "test_id": "test_startup_exception_message.py::test_traceback_level[student_code]", + "message": "Student code execution failed with an exception: ValueError\nException Message: first line of the reason\nsecond line of the reason\n\nTraceback (most recent call last):", + "outcome": "error", + "max_points": 1, + "points": 0.0, + "points_frac": 0.0 + } + ] + } +} diff --git a/tests/scenario_root/test_startup_exception_message/scenario.py b/tests/scenario_root/test_startup_exception_message/scenario.py new file mode 100644 index 0000000..992fe76 --- /dev/null +++ b/tests/scenario_root/test_startup_exception_message/scenario.py @@ -0,0 +1,22 @@ +"""An exception raised while student code starts up reaches the results with its full message.""" + +import pytest + +from pytest_prairielearn_grader.fixture import StudentFixture + + +@pytest.mark.grading_data(name="Default output level", points=1) +def test_default_level(sandbox: StudentFixture) -> None: + sandbox.query("x") + + +@pytest.mark.grading_data(name="Exception name output level", points=1) +@pytest.mark.output(level="none") +def test_name_level(sandbox: StudentFixture) -> None: + sandbox.query("x") + + +@pytest.mark.grading_data(name="Traceback output level", points=1) +@pytest.mark.output(level="traceback") +def test_traceback_level(sandbox: StudentFixture) -> None: + sandbox.query("x") diff --git a/tests/scenario_root/test_startup_exception_message/student_code.py b/tests/scenario_root/test_startup_exception_message/student_code.py new file mode 100644 index 0000000..2671579 --- /dev/null +++ b/tests/scenario_root/test_startup_exception_message/student_code.py @@ -0,0 +1 @@ +raise ValueError("first line of the reason\nsecond line of the reason") diff --git a/tests/scenario_root/test_workspace_shadowed_module/__init__.py b/tests/scenario_root/test_workspace_shadowed_module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_workspace_shadowed_module/expected_outcome.json b/tests/scenario_root/test_workspace_shadowed_module/expected_outcome.json new file mode 100644 index 0000000..5db5346 --- /dev/null +++ b/tests/scenario_root/test_workspace_shadowed_module/expected_outcome.json @@ -0,0 +1,16 @@ +{ + "expected_data_object": { + "gradable": true, + "score": 0.0, + "tests": [ + { + "test_id": "test_workspace_shadowed_module.py::test_uses_random", + "message": "Student code execution failed with an exception: ImportError\nException Message: These names are already used by modules that the grader has loaded: json/, random.py. Rename yours and update any imports that use them.", + "outcome": "error", + "max_points": 1, + "points": 0.0, + "points_frac": 0.0 + } + ] + } +} diff --git a/tests/scenario_root/test_workspace_shadowed_module/scenario.py b/tests/scenario_root/test_workspace_shadowed_module/scenario.py new file mode 100644 index 0000000..bf631dc --- /dev/null +++ b/tests/scenario_root/test_workspace_shadowed_module/scenario.py @@ -0,0 +1,22 @@ +""" +A student file or folder named like a module the sandbox has already loaded fails startup with a +message asking the student to rename it, since Python would ignore it and reuse the loaded module. + +The student's project has random.py and json/ (both loaded by the sandbox before student code runs) +and calendar.py (not loaded, so it isn't a clash and isn't listed). +""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import WorkspaceFixture + +autograder_config = ConfigObject( + sandbox_timeout=5.0, + workspace_mode=True, +) + + +@pytest.mark.grading_data(name="Uses the student's random module", points=1) +def test_uses_random(workspace_sandbox: WorkspaceFixture) -> None: + assert workspace_sandbox.query("solution.VALUE") == "mine" diff --git a/tests/scenario_root/test_workspace_shadowed_module/student/calendar.py b/tests/scenario_root/test_workspace_shadowed_module/student/calendar.py new file mode 100644 index 0000000..3e3e4d2 --- /dev/null +++ b/tests/scenario_root/test_workspace_shadowed_module/student/calendar.py @@ -0,0 +1 @@ +VALUE = "mine" diff --git a/tests/scenario_root/test_workspace_shadowed_module/student/json/__init__.py b/tests/scenario_root/test_workspace_shadowed_module/student/json/__init__.py new file mode 100644 index 0000000..3e3e4d2 --- /dev/null +++ b/tests/scenario_root/test_workspace_shadowed_module/student/json/__init__.py @@ -0,0 +1 @@ +VALUE = "mine" diff --git a/tests/scenario_root/test_workspace_shadowed_module/student/random.py b/tests/scenario_root/test_workspace_shadowed_module/student/random.py new file mode 100644 index 0000000..3e3e4d2 --- /dev/null +++ b/tests/scenario_root/test_workspace_shadowed_module/student/random.py @@ -0,0 +1 @@ +VALUE = "mine" diff --git a/tests/scenario_root/test_workspace_shadowed_module/student/solution.py b/tests/scenario_root/test_workspace_shadowed_module/student/solution.py new file mode 100644 index 0000000..af01bb4 --- /dev/null +++ b/tests/scenario_root/test_workspace_shadowed_module/student/solution.py @@ -0,0 +1,3 @@ +import random + +VALUE = random.VALUE diff --git a/tests/scenario_root/test_workspace_startup_exception/__init__.py b/tests/scenario_root/test_workspace_startup_exception/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_workspace_startup_exception/expected_outcome.json b/tests/scenario_root/test_workspace_startup_exception/expected_outcome.json new file mode 100644 index 0000000..e317c5b --- /dev/null +++ b/tests/scenario_root/test_workspace_startup_exception/expected_outcome.json @@ -0,0 +1,15 @@ +{ + "expected_data_object": { + "score": 0.0, + "tests": [ + { + "test_id": "test_workspace_startup_exception.py::test_entry_raises", + "message": "Student code execution failed with an exception: ImportError\nException Message: main.py could not find the data it needs", + "outcome": "error", + "max_points": 1, + "points": 0.0, + "points_frac": 0.0 + } + ] + } +} diff --git a/tests/scenario_root/test_workspace_startup_exception/scenario.py b/tests/scenario_root/test_workspace_startup_exception/scenario.py new file mode 100644 index 0000000..3a56c77 --- /dev/null +++ b/tests/scenario_root/test_workspace_startup_exception/scenario.py @@ -0,0 +1,17 @@ +"""An exception raised by the workspace entry file reaches the results with its message.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import WorkspaceFixture + +autograder_config = ConfigObject( + sandbox_timeout=5.0, + workspace_mode=True, + workspace_exec_entry="main.py", +) + + +@pytest.mark.grading_data(name="Entry file raises", points=1) +def test_entry_raises(workspace_sandbox: WorkspaceFixture) -> None: + workspace_sandbox.query("main.x") diff --git a/tests/scenario_root/test_workspace_startup_exception/student/main.py b/tests/scenario_root/test_workspace_startup_exception/student/main.py new file mode 100644 index 0000000..32c84f4 --- /dev/null +++ b/tests/scenario_root/test_workspace_startup_exception/student/main.py @@ -0,0 +1 @@ +raise ImportError("main.py could not find the data it needs") diff --git a/tests/test_helper_name_clashes.py b/tests/test_helper_name_clashes.py index d3f9732..cbda98f 100644 --- a/tests/test_helper_name_clashes.py +++ b/tests/test_helper_name_clashes.py @@ -2,24 +2,24 @@ from pathlib import Path -from pytest_prairielearn_grader.plugin import _find_helper_name_clashes +from pytest_prairielearn_grader._student_code_runner import _find_helper_name_clashes def test_no_helper_modules(tmp_path: Path) -> None: (tmp_path / "helper.py").write_text("x = 1\n") - assert _find_helper_name_clashes(tmp_path, None) == [] + assert _find_helper_name_clashes(str(tmp_path), None) == [] def test_unrelated_file(tmp_path: Path) -> None: (tmp_path / "other.py").write_text("x = 1\n") - assert _find_helper_name_clashes(tmp_path, {"helper": "x = 1\n"}) == [] + assert _find_helper_name_clashes(str(tmp_path), {"helper": "x = 1\n"}) == [] def test_file(tmp_path: Path) -> None: (tmp_path / "helper.py").write_text("x = 1\n") - assert _find_helper_name_clashes(tmp_path, {"helper": "x = 1\n"}) == ["helper.py"] + assert _find_helper_name_clashes(str(tmp_path), {"helper": "x = 1\n"}) == ["helper.py"] def test_folder(tmp_path: Path) -> None: (tmp_path / "helper").mkdir() - assert _find_helper_name_clashes(tmp_path, {"helper": "x = 1\n"}) == ["helper/"] + assert _find_helper_name_clashes(str(tmp_path), {"helper": "x = 1\n"}) == ["helper/"]