From 3ceddb787bd5e905c5637779067df990e66fefe6 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Wed, 18 Nov 2026 17:26:30 -0500 Subject: [PATCH 1/6] Collapse duplicated student and workspace code paths Refactor only; no behavior change. Unifies the near-identical student and workspace implementations behind one shared, parameterized path in each layer: - _student_code_runner.py: student_code_runner and workspace_runner now wrap a single _run_sandbox helper; workspace mode is selected by workspace_dir and the main code object to exec is supplied by a load_main callback. - fixture.py: lift helper_modules into _SandboxBase and extract the shared subprocess/connect/send/read sequence into _SandboxBase._start_server. - plugin.py: _initialize_sandbox_fixture and _initialize_workspace_fixture share a new _resolve_sandbox_config helper that resolves the common config. Public API (sandbox, module_sandbox, workspace_sandbox fixtures, ConfigObject, StudentFixture, WorkspaceFixture) is unchanged. Co-Authored-By: Claude Opus 4.8 --- .../_student_code_runner.py | 183 +++++++++--------- src/pytest_prairielearn_grader/fixture.py | 145 ++++++++------ src/pytest_prairielearn_grader/plugin.py | 153 +++++++-------- 3 files changed, 235 insertions(+), 246 deletions(-) diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 53cb6e5..5607775 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -447,10 +447,9 @@ def _find_helper_name_clashes(workspace_dir: str, helper_modules: dict[str, str] return clashes -async def student_code_runner( +async def _run_sandbox( + *, setup_code: str | None, - student_code: str, - student_file_name: str, timeout: float, import_whitelist: list[str] | None, import_blacklist: list[str] | None, @@ -458,7 +457,21 @@ async def student_code_runner( builtin_whitelist: list[str] | None, names_for_user_list: list[str] | None, helper_modules: dict[str, str] | None, + workspace_dir: str | None, + load_main: Callable[[], tuple[str, str]] | None, + main_timeout_message: str, ) -> tuple[dict[str, Any], dict[str, Any], ProcessStartResponse]: + """ + Shared implementation behind student_code_runner and workspace_runner. + + Runs the setup code, injects the names_for_user variables, and then runs a main code object: + the student submission for student mode, or the workspace entry-point file for workspace mode. + *load_main*, called inside the capture/timeout region, returns that code's (source, file_name); + pass None to skip the main step. When *workspace_dir* is given, the importer also replaces the + real builtins.__import__ and the directory is put at the front of sys.path, so that student + modules loaded via importlib.import_module() go through the same restrictions and resolve + against the workspace. + """ stdout_capture = io.StringIO() stderr_capture = io.StringIO() execution_error: Exception | None = None @@ -468,16 +481,27 @@ async def student_code_runner( student_code_vars: dict[str, Any] = {} student_code_vars["__builtins__"] = get_builtins(builtin_whitelist) - student_code_vars["__builtins__"]["__name__"] = "__main__" # Set __name__ to "__main__" to mimic the main module server_data = ServerDataImport(local_vars.get("__data_params", {})) - student_code_vars["__builtins__"]["__import__"] = get_custom_importer( + importer = get_custom_importer( import_whitelist, import_blacklist, + workspace_dir, helper_modules=None if helper_modules is None else _HelperModules(helper_modules, student_code_vars["__builtins__"]), server_data=server_data, ) _block_blacklisted_functions(import_blacklist) + student_code_vars["__builtins__"]["__import__"] = importer + if workspace_dir is not None: + # Patch the real builtins.__import__ so that workspace modules loaded via + # importlib.import_module() (which use the real builtins, not student_code_vars) + # also go through the same restrictions. The subprocess is dedicated to one + # grading session so this global patch is safe. + builtins.__import__ = importer # type: ignore[assignment] + # Insert workspace_dir at the front of sys.path so imports resolve against it. + # Use insert(0, ...) to take precedence over any previously added paths. + if workspace_dir not in sys.path: + sys.path.insert(0, workspace_dir) try: # First, execute the setup code if provided @@ -513,19 +537,19 @@ async def student_code_runner( # If not in local_vars (setup_code), try getting from starting_vars student_code_vars[var_name] = deepcopy(starting_vars[var_name]) - if execution_error is None: + if execution_error is None and load_main is not None: try: - # Next, compile student code. Make sure to handle errors in this later - # TODO have a better filename - code_setup = compile(student_code, student_file_name, "exec") + # Compile and run the main code (student submission or workspace entry point). + main_source, main_file_name = load_main() + main_code = compile(main_source, main_file_name, "exec") with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture): await asyncio.wait_for( - asyncio.get_event_loop().run_in_executor(executor, exec, code_setup, student_code_vars, student_code_vars), + asyncio.get_event_loop().run_in_executor(executor, exec, main_code, student_code_vars, student_code_vars), timeout=timeout, ) except asyncio.TimeoutError: - execution_error = asyncio.TimeoutError("Student code execution timed out") + execution_error = asyncio.TimeoutError(main_timeout_message) except Exception as e: execution_error = e # TODO this traceback only shows the last line with the exception. @@ -554,6 +578,33 @@ async def student_code_runner( return local_vars, student_code_vars, result_dict +async def student_code_runner( + setup_code: str | None, + student_code: str, + student_file_name: str, + timeout: float, + import_whitelist: list[str] | None, + import_blacklist: list[str] | None, + starting_vars: dict[str, Any] | None, + builtin_whitelist: list[str] | None, + names_for_user_list: list[str] | None, + helper_modules: dict[str, str] | None, +) -> tuple[dict[str, Any], dict[str, Any], ProcessStartResponse]: + return await _run_sandbox( + setup_code=setup_code, + timeout=timeout, + import_whitelist=import_whitelist, + import_blacklist=import_blacklist, + starting_vars=starting_vars, + builtin_whitelist=builtin_whitelist, + names_for_user_list=names_for_user_list, + helper_modules=helper_modules, + workspace_dir=None, + load_main=lambda: (student_code, student_file_name), + main_timeout_message="Student code execution timed out", + ) + + async def workspace_runner( workspace_dir: str, exec_entry: str | None, @@ -577,14 +628,9 @@ async def workspace_runner( helper_clashes = _find_helper_name_clashes(workspace_dir, helper_modules) shadowed_files = [f for f in _find_shadowed_workspace_files(workspace_dir) if f not in helper_clashes] - stdout_capture = io.StringIO() - stderr_capture = io.StringIO() - execution_error: Exception | None = None - exception_traceback = None - local_vars = deepcopy(starting_vars) if starting_vars else {} - local_vars["__from_server_json"] = from_server_json - if helper_clashes or shadowed_files: + local_vars = deepcopy(starting_vars) if starting_vars else {} + local_vars["__from_server_json"] = from_server_json reasons = [] if helper_clashes: reasons.append(f"These names are reserved for modules that this question provides: {', '.join(helper_clashes)}.") @@ -601,88 +647,33 @@ async def workspace_runner( } return local_vars, {}, start_response - student_code_vars: dict[str, Any] = {} - student_code_vars["__builtins__"] = get_builtins(builtin_whitelist) - student_code_vars["__builtins__"]["__name__"] = "__main__" - server_data = ServerDataImport(local_vars.get("__data_params", {})) - workspace_importer = get_custom_importer( - import_whitelist, - import_blacklist, - workspace_dir, - helper_modules=None if helper_modules is None else _HelperModules(helper_modules, student_code_vars["__builtins__"]), - server_data=server_data, - ) - _block_blacklisted_functions(import_blacklist) - student_code_vars["__builtins__"]["__import__"] = workspace_importer - # Patch the real builtins.__import__ so that workspace modules loaded via - # importlib.import_module() (which use the real builtins, not student_code_vars) - # also go through the same restrictions. The subprocess is dedicated to one - # grading session so this global patch is safe. - builtins.__import__ = workspace_importer # type: ignore[assignment] - - # Insert workspace_dir at the front of sys.path so imports resolve against it. - # Use insert(0, ...) to take precedence over any previously added paths. - if workspace_dir not in sys.path: - sys.path.insert(0, workspace_dir) - - try: - if setup_code: - code_setup = compile(setup_code, "", "exec") - with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture): - await asyncio.wait_for( - asyncio.get_event_loop().run_in_executor(executor, exec, code_setup, student_code_vars, local_vars), - timeout=timeout, - ) - except asyncio.TimeoutError: - execution_error = asyncio.TimeoutError("Setup code execution timed out") - except Exception as e: - execution_error = e - - server_data.available = False - - if names_for_user_list is not None: - for var_name in names_for_user_list: - if var_name in local_vars: - student_code_vars[var_name] = deepcopy(local_vars[var_name]) - elif starting_vars is not None and var_name in starting_vars: - student_code_vars[var_name] = deepcopy(starting_vars[var_name]) - # Optionally exec an entry-point file at startup (e.g. "main.py") so that # module-level side effects in the student's root script are applied. - if execution_error is None and exec_entry is not None: - entry_path = pathlib.Path(workspace_dir) / exec_entry - try: + load_main: Callable[[], tuple[str, str]] | None = None + if exec_entry is not None: + entry = exec_entry + + def load_entry() -> tuple[str, str]: + entry_path = pathlib.Path(workspace_dir) / entry entry_source = entry_path.read_text(encoding="utf-8") - entry_code = compile(entry_source, str(entry_path), "exec") populate_linecache(entry_source, str(entry_path)) - with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture): - await asyncio.wait_for( - asyncio.get_event_loop().run_in_executor(executor, exec, entry_code, student_code_vars, student_code_vars), - timeout=timeout, - ) - except asyncio.TimeoutError: - execution_error = asyncio.TimeoutError("Entry-point file execution timed out") - except Exception as e: - execution_error = e - exception_traceback = traceback.format_exc(limit=-1) - - if execution_error is None: - status = ProcessStatusCode.SUCCESS - elif isinstance(execution_error, asyncio.TimeoutError): - status = ProcessStatusCode.TIMEOUT - else: - status = ProcessStatusCode.EXCEPTION - - result_dict: ProcessStartResponse = { - "status": status, - "stdout": stdout_capture.getvalue(), - "stderr": stderr_capture.getvalue(), - "execution_error": type(execution_error).__name__ if execution_error else None, - "execution_message": str(execution_error) if execution_error else None, - "execution_traceback": str(exception_traceback), - } - - return local_vars, student_code_vars, result_dict + return entry_source, str(entry_path) + + load_main = load_entry + + return await _run_sandbox( + setup_code=setup_code, + timeout=timeout, + import_whitelist=import_whitelist, + import_blacklist=import_blacklist, + starting_vars=starting_vars, + builtin_whitelist=builtin_whitelist, + names_for_user_list=names_for_user_list, + helper_modules=helper_modules, + workspace_dir=workspace_dir, + load_main=load_main, + main_timeout_message="Entry-point file execution timed out", + ) def encode_server_failure(message: str) -> bytes: diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 596062d..6b87ef8 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -9,6 +9,7 @@ import subprocess import sys import tempfile +from collections.abc import Callable from importlib.resources import files from pathlib import Path from typing import Any @@ -224,6 +225,7 @@ class _SandboxBase: builtin_whitelist: list[str] | None names_for_user_list: list[str] | None worker_username: str | None + helper_modules: dict[str, str] | None data_files: dict[str, Path] | None _data_files_dir: Path | None _accumulated_stdout: list[str] @@ -238,6 +240,7 @@ def __init__( builtin_whitelist: list[str] | None, names_for_user_list: list[str] | None, worker_username: str | None, + helper_modules: dict[str, str] | None = None, data_files: dict[str, Path] | None = None, ) -> None: self.import_whitelist = import_whitelist @@ -246,6 +249,7 @@ def __init__( self.builtin_whitelist = builtin_whitelist self.names_for_user_list = names_for_user_list self.worker_username = worker_username + self.helper_modules = helper_modules self.data_files = data_files self._data_files_dir = None self.process = None @@ -254,6 +258,14 @@ def __init__( self._server_failure = None self._timed_out_call = None + def _start_server(self, initialization_timeout: float, build_start_message: Callable[[], Any]) -> ProcessStartResponse: + """Spawn the subprocess, send the start message it builds, and read the start response.""" + self._start_subprocess() + json_message = build_start_message() + self._connect_to_server(initialization_timeout) + self._send_json_object(json_message) + return self._read_start_response(initialization_timeout) + def _assert_process_running(self) -> None: assert self.process is not None, "Sandbox process is not running. Please start it first." process_return_code = self.process.poll() @@ -488,14 +500,20 @@ def __init__( data_files: dict[str, Path] | None = None, ) -> None: super().__init__( - import_whitelist, import_blacklist, starting_vars, builtin_whitelist, names_for_user_list, worker_username, data_files + import_whitelist, + import_blacklist, + starting_vars, + builtin_whitelist, + names_for_user_list, + worker_username, + helper_modules=helper_modules, + data_files=data_files, ) self.leading_file = file_names.leading_file self.trailing_file = file_names.trailing_file self.student_code_file = file_names.student_code_file self.setup_code_file = file_names.setup_code_file self.notebook_cell_tag = notebook_cell_tag - self.helper_modules = helper_modules def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_TIMEOUT) -> ProcessStartResponse: if self.worker_username is not None: @@ -503,43 +521,40 @@ def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_T else: logger.debug("Starting student code server without dropping privileges.") - self._start_subprocess() - - student_code = "" - if self.leading_file.is_file(): - student_code += self.leading_file.read_text(encoding="utf-8") - student_code += os.linesep - if self.student_code_file.is_file(): - if self.student_code_file.suffix.lower() == ".ipynb": - student_code += _extract_notebook_code(self.student_code_file, self.notebook_cell_tag) - else: - student_code += self.student_code_file.read_text(encoding="utf-8") - if self.trailing_file.is_file(): - student_code += os.linesep - student_code += self.trailing_file.read_text(encoding="utf-8") - - setup_code = None - if self.setup_code_file.is_file(): - setup_code = self.setup_code_file.read_text(encoding="utf-8") - - self._connect_to_server(initialization_timeout) - - # TODO make this a shared type - json_message = ProcessStartRequest( - message_type=MessageType.START, - student_code=student_code, - student_file_name=str(self.student_code_file), - setup_code=setup_code, - initialization_timeout=initialization_timeout, - import_whitelist=self.import_whitelist, - import_blacklist=self.import_blacklist, - starting_vars=self.starting_vars, - builtin_whitelist=self.builtin_whitelist, - names_for_user_list=self.names_for_user_list, - helper_modules=self.helper_modules, - ) - self._send_json_object(json_message) - return self._read_start_response(initialization_timeout) + def build_start_message() -> ProcessStartRequest: + student_code = "" + if self.leading_file.is_file(): + student_code += self.leading_file.read_text(encoding="utf-8") + student_code += os.linesep + if self.student_code_file.is_file(): + if self.student_code_file.suffix.lower() == ".ipynb": + student_code += _extract_notebook_code(self.student_code_file, self.notebook_cell_tag) + else: + student_code += self.student_code_file.read_text(encoding="utf-8") + if self.trailing_file.is_file(): + student_code += os.linesep + student_code += self.trailing_file.read_text(encoding="utf-8") + + setup_code = None + if self.setup_code_file.is_file(): + setup_code = self.setup_code_file.read_text(encoding="utf-8") + + # TODO make this a shared type + return ProcessStartRequest( + message_type=MessageType.START, + student_code=student_code, + student_file_name=str(self.student_code_file), + setup_code=setup_code, + initialization_timeout=initialization_timeout, + import_whitelist=self.import_whitelist, + import_blacklist=self.import_blacklist, + starting_vars=self.starting_vars, + builtin_whitelist=self.builtin_whitelist, + names_for_user_list=self.names_for_user_list, + helper_modules=self.helper_modules, + ) + + return self._start_server(initialization_timeout, build_start_message) def query_setup_raw(self, var_to_query: str) -> SetupQueryResponse: self._assert_no_server_failure() @@ -602,38 +617,42 @@ def __init__( data_files: dict[str, Path] | None = None, ) -> None: super().__init__( - import_whitelist, import_blacklist, starting_vars, builtin_whitelist, names_for_user_list, worker_username, data_files + import_whitelist, + import_blacklist, + starting_vars, + builtin_whitelist, + names_for_user_list, + worker_username, + helper_modules=helper_modules, + data_files=data_files, ) self.workspace_dir = workspace_dir self.setup_code_file = setup_code_file self.exec_entry = exec_entry - self.helper_modules = helper_modules def start_workspace_server(self, *, initialization_timeout: float = DEFAULT_TIMEOUT) -> ProcessStartResponse: """Start the sandbox subprocess and initialise the workspace.""" - self._start_subprocess() - setup_code: str | None = None - if self.setup_code_file.is_file(): - setup_code = self.setup_code_file.read_text(encoding="utf-8") - - self._connect_to_server(initialization_timeout) - - json_message = WorkspaceStartRequest( - message_type=MessageType.START_WORKSPACE, - workspace_dir=str(self.workspace_dir), - exec_entry=self.exec_entry, - setup_code=setup_code, - initialization_timeout=initialization_timeout, - import_whitelist=self.import_whitelist, - import_blacklist=self.import_blacklist, - starting_vars=self.starting_vars, - builtin_whitelist=self.builtin_whitelist, - names_for_user_list=self.names_for_user_list, - helper_modules=self.helper_modules, - ) - self._send_json_object(json_message) - return self._read_start_response(initialization_timeout) + def build_start_message() -> WorkspaceStartRequest: + setup_code: str | None = None + if self.setup_code_file.is_file(): + setup_code = self.setup_code_file.read_text(encoding="utf-8") + + return WorkspaceStartRequest( + message_type=MessageType.START_WORKSPACE, + workspace_dir=str(self.workspace_dir), + exec_entry=self.exec_entry, + setup_code=setup_code, + initialization_timeout=initialization_timeout, + import_whitelist=self.import_whitelist, + import_blacklist=self.import_blacklist, + starting_vars=self.starting_vars, + builtin_whitelist=self.builtin_whitelist, + names_for_user_list=self.names_for_user_list, + helper_modules=self.helper_modules, + ) + + return self._start_server(initialization_timeout, build_start_message) def __repr__(self) -> str: return f"WorkspaceFixture(workspace_dir={self.workspace_dir})" diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 6352b15..603043d 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -293,46 +293,50 @@ def tests_dir(request: pytest.FixtureRequest) -> Path: return path -def _initialize_sandbox_fixture( +@dataclass(frozen=True, slots=True) +class _ResolvedSandboxConfig: + """The configuration shared by the student and workspace fixtures, resolved once per fixture.""" + + initialization_timeout: float + import_whitelist: list[str] | None + import_blacklist: list[str] | None + builtin_whitelist: list[str] | None + starting_vars: dict[str, Any] + names_for_user_list: list[str] | None + helper_module_names: list[str] | None + data_file_names: list[str] | None + # The module-level ConfigObject, kept so each fixture can read its own extra fields + # (notebook_cell_tag for the student fixture, workspace_exec_entry for the workspace one). + config_object: ConfigObject | None + + +def _resolve_sandbox_config( request: pytest.FixtureRequest, data_json: dict[str, Any] | None, - file_names: StudentFiles, -) -> tuple[StudentFixture, float]: +) -> _ResolvedSandboxConfig: """ - Common initialization logic for both sandbox and module_sandbox fixtures. - Handles parameter parsing, timeout configuration, and StudentFixture creation. - Returns the fixture and the initialization timeout. + Resolve the configuration common to the sandbox, module_sandbox and workspace_sandbox fixtures. - Args: - request: The pytest fixture request - data_json: Data loaded from data.json file - file_names: StudentFiles namedtuple with file paths + Reads params from data.json and the module-level ConfigObject (or module-level timeout/mark + when there's no ConfigObject), merges them with the ConfigObject taking precedence, and injects + the names_for_user values into starting_vars. """ - # Default timeout TODO make this a command line option? initialization_timeout: float = DEFAULT_SANDBOX_TIMEOUT - - if data_json is None: - params_dict = {} - logger.debug("No data.json provided, using empty params") - else: - params_dict = data_json.get("params", {}) - logger.debug(f"Loaded params from data.json: {list(params_dict.keys())}") + params_dict = data_json.get("params", {}) if data_json is not None else {} # Check for module-level ConfigObject that overrides all other configuration config_dict: dict[str, Any] = {} - notebook_cell_tag: str | None = None + config_object: ConfigObject | None = None helper_module_names: list[str] | None = None data_file_names: list[str] | None = None if hasattr(request, "module") and hasattr(request.module, "autograder_config"): config_object = request.module.autograder_config if not isinstance(config_object, ConfigObject): pytest.fail(f"autograder_config must be a ConfigObject instance, got {type(config_object).__name__}") - logger.debug("Using module-level autograder_config - overriding all other configuration sources") config_dict = config_object.to_dict() # Handle timeout and starting_vars separately as they're not in to_dict() initialization_timeout = config_object.sandbox_timeout config_starting_vars = config_object.starting_vars - notebook_cell_tag = config_object.notebook_cell_tag helper_module_names = config_object.helper_modules data_file_names = config_object.data_files else: @@ -340,13 +344,11 @@ def _initialize_sandbox_fixture( # This allows setting: sandbox_timeout = 0.5 at module level if hasattr(request, "module") and hasattr(request.module, "sandbox_timeout"): initialization_timeout = request.module.sandbox_timeout - logger.debug(f"Using module-level timeout: {initialization_timeout}s") # Check for the custom mark (overrides module-level setting) marker = request.node.get_closest_marker("sandbox_timeout") if marker and marker.args: initialization_timeout = marker.args[0] - logger.debug(f"Using test-specific timeout override: {initialization_timeout}s") config_starting_vars = {} @@ -374,8 +376,6 @@ def _initialize_sandbox_fixture( **config_starting_vars, } - logger.debug(f"Import restrictions - whitelist: {import_whitelist}, blacklist: {import_blacklist}") - # Process names_for_user_list to inject variables from params_dict or starting_vars if names_for_user_list is not None: for name in names_for_user_list: @@ -387,20 +387,45 @@ def _initialize_sandbox_fixture( starting_vars[name] = value - fixture = StudentFixture( - file_names=file_names, + return _ResolvedSandboxConfig( + initialization_timeout=initialization_timeout, import_whitelist=import_whitelist, import_blacklist=import_blacklist, - starting_vars=starting_vars, builtin_whitelist=builtin_whitelist, + starting_vars=starting_vars, names_for_user_list=names_for_user_list, + helper_module_names=helper_module_names, + data_file_names=data_file_names, + config_object=config_object, + ) + + +def _initialize_sandbox_fixture( + request: pytest.FixtureRequest, + data_json: dict[str, Any] | None, + file_names: StudentFiles, +) -> tuple[StudentFixture, float]: + """ + Common initialization logic for both sandbox and module_sandbox fixtures. + Returns the StudentFixture and the initialization timeout. + """ + config = _resolve_sandbox_config(request, data_json) + notebook_cell_tag = config.config_object.notebook_cell_tag if config.config_object is not None else None + + fixture = StudentFixture( + file_names=file_names, + import_whitelist=config.import_whitelist, + import_blacklist=config.import_blacklist, + starting_vars=config.starting_vars, + builtin_whitelist=config.builtin_whitelist, + names_for_user_list=config.names_for_user_list, worker_username=request.config.getoption("--worker-username"), notebook_cell_tag=notebook_cell_tag, - helper_modules=_load_helper_modules(request, helper_module_names, import_blacklist), - data_files=_resolve_data_files(request, data_file_names), + helper_modules=_load_helper_modules(request, config.helper_module_names, config.import_blacklist), + data_files=_resolve_data_files(request, config.data_file_names), ) - return fixture, initialization_timeout + return fixture, config.initialization_timeout def _get_config_output_level(module: ModuleType | None) -> str | None: @@ -655,54 +680,8 @@ def _initialize_workspace_fixture( workspace_dir: Path, ) -> tuple[WorkspaceFixture, float]: """Initialise a WorkspaceFixture analogously to _initialize_sandbox_fixture.""" - initialization_timeout: float = DEFAULT_SANDBOX_TIMEOUT - - params_dict = data_json.get("params", {}) if data_json is not None else {} - - config_dict: dict[str, Any] = {} - exec_entry: str | None = None - helper_module_names: list[str] | None = None - data_file_names: list[str] | None = None - if hasattr(request, "module") and hasattr(request.module, "autograder_config"): - config_object = request.module.autograder_config - if not isinstance(config_object, ConfigObject): - pytest.fail(f"autograder_config must be a ConfigObject instance, got {type(config_object).__name__}") - config_dict = config_object.to_dict() - initialization_timeout = config_object.sandbox_timeout - config_starting_vars = config_object.starting_vars - exec_entry = config_object.workspace_exec_entry - helper_module_names = config_object.helper_modules - data_file_names = config_object.data_files - else: - if hasattr(request, "module") and hasattr(request.module, "sandbox_timeout"): - initialization_timeout = request.module.sandbox_timeout - marker = request.node.get_closest_marker("sandbox_timeout") - if marker and marker.args: - initialization_timeout = marker.args[0] - config_starting_vars = {} - - names_for_user_list: list[str] | None = None - if "names_for_user" in config_dict: - names_for_user_list = config_dict["names_for_user"] - elif "names_for_user" in params_dict: - names_for_user_list = [item["name"] for item in params_dict["names_for_user"]] - - merged_params = {**params_dict, **config_dict} - import_whitelist = merged_params.get("import_whitelist") - import_blacklist = merged_params.get("import_blacklist", DEFAULT_IMPORT_BLACKLIST) - builtin_whitelist = merged_params.get("builtin_whitelist") - - starting_vars: dict[str, Any] = { - "__data_params": deepcopy(params_dict) if data_json is not None else {}, - **config_starting_vars, - } - if names_for_user_list is not None: - for name in names_for_user_list: - if name in starting_vars and name != "__data_params": - value = starting_vars[name] - else: - value = params_dict.get(name, None) - starting_vars[name] = value + config = _resolve_sandbox_config(request, data_json) + exec_entry = config.config_object.workspace_exec_entry if config.config_object is not None else None # Resolve setup_code_file from the test module's tests folder tests_dir = get_tests_dir(request.module) @@ -711,17 +690,17 @@ def _initialize_workspace_fixture( fixture = WorkspaceFixture( workspace_dir=workspace_dir, setup_code_file=setup_code_file, - import_whitelist=import_whitelist, - import_blacklist=import_blacklist, - starting_vars=starting_vars, - builtin_whitelist=builtin_whitelist, - names_for_user_list=names_for_user_list, + import_whitelist=config.import_whitelist, + import_blacklist=config.import_blacklist, + starting_vars=config.starting_vars, + builtin_whitelist=config.builtin_whitelist, + names_for_user_list=config.names_for_user_list, worker_username=request.config.getoption("--worker-username"), exec_entry=exec_entry, - helper_modules=_load_helper_modules(request, helper_module_names, import_blacklist), - data_files=_resolve_data_files(request, data_file_names), + helper_modules=_load_helper_modules(request, config.helper_module_names, config.import_blacklist), + data_files=_resolve_data_files(request, config.data_file_names), ) - return fixture, initialization_timeout + return fixture, config.initialization_timeout @pytest.fixture From 4dc15a871c22b7cb0dbfe4a08eac2f51b8f6c8d4 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Wed, 18 Nov 2026 17:48:10 -0500 Subject: [PATCH 2/6] Replace the sandbox socket RPC with a multiprocessing pipe Issue #57: swap the TCP-socket, JSON-lines transport to the sandbox for a multiprocessing child process and a Connection pipe. - The sandbox runs as a spawn-context Process that drops privileges first, then serves requests over conn.recv()/conn.send() in a blocking loop. spawn (not fork) keeps the child from inheriting the grader's memory. - The sandbox runner and function runner become synchronous: the per-call timeout is now executor.submit(fn).result(timeout=...) instead of the asyncio.wait_for wrapper, keeping the single-worker concurrency model. - Query/function return values still cross the pipe as JSON strings, so only JSON data (never a student object) is unpickled in the grader process. Co-Authored-By: Claude Opus 4.8 --- .../_student_code_runner.py | 214 +++++++----------- src/pytest_prairielearn_grader/fixture.py | 143 +++++------- tests/test_subtests.py | 16 +- 3 files changed, 148 insertions(+), 225 deletions(-) diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 5607775..687cdc2 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -1,4 +1,3 @@ -import asyncio import builtins import concurrent.futures import importlib @@ -16,6 +15,7 @@ from contextlib import redirect_stderr from contextlib import redirect_stdout from copy import deepcopy +from multiprocessing.connection import Connection from typing import Any from pytest_prairielearn_grader.json_utils import from_server_json @@ -39,6 +39,7 @@ from pytest_prairielearn_grader.utils import StudentQueryResponse from pytest_prairielearn_grader.utils import WorkspaceStartRequest from pytest_prairielearn_grader.utils import deserialize_object_unsafe +from pytest_prairielearn_grader.utils import drop_privileges from pytest_prairielearn_grader.utils import get_builtins from pytest_prairielearn_grader.utils import serialize_object_unsafe @@ -49,8 +50,6 @@ # is later replaced by a workspace_importer in workspace_runner. _REAL_IMPORT: ImportFunction = builtins.__import__ # type: ignore[assignment] -HOST = "127.0.0.1" # Loopback address, means "this computer only" - # Global ThreadPoolExecutor for CPU-bound tasks # It's good practice to create this once and reuse it. @@ -120,15 +119,15 @@ def _look_up_function(name: str, student_code_vars: dict[str, Any], workspace_mo return student_code_vars[name] -async def student_function_runner( +def student_function_runner( student_code_vars: dict[str, Any], func_name: str, timeout: float, args_tup: Any, kwargs_dict: Any, workspace_mode: bool = False, -) -> str: - """Run a student function and return its StudentFunctionResponse encoded as a JSON string.""" +) -> StudentFunctionResponse: + """Run a student function and return its StudentFunctionResponse.""" stdout_capture = io.StringIO() stderr_capture = io.StringIO() execution_error: Exception | None = None @@ -144,12 +143,11 @@ def student_function_temp() -> Any: return student_function(*args_tup, **kwargs_dict) with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture): - future = asyncio.get_event_loop().run_in_executor(executor, student_function_temp) - done, _ = await asyncio.wait({future}, timeout=timeout) - if done: - result = future.result() - else: - execution_error = asyncio.TimeoutError(f"Function timed out after {timeout} seconds") + future = executor.submit(student_function_temp) + try: + result = future.result(timeout=timeout) + except concurrent.futures.TimeoutError: + execution_error = TimeoutError(f"Function timed out after {timeout} seconds") timed_out = True except FunctionNotFoundError as e: execution_error = NameError(str(e)) @@ -176,9 +174,16 @@ def student_function_temp() -> Any: "traceback": exception_traceback, } + # On any execution error the call produced no value, so leave it as None. + if execution_error is not None: + return function_response + try: - function_response["value"] = to_json(result) - return json.dumps(function_response) + # The value is sent back to the grader as a JSON string so that only JSON data, never a + # student-controlled object, crosses the pipe into the grader process. json.dumps also + # raises here for a value that can't be represented, which drives the NOT_SERIALIZABLE path. + function_response["value"] = json.dumps(to_json(result)) + return function_response except Exception as e: not_serializable_response: StudentFunctionResponse = { "status": FunctionStatusCode.NOT_SERIALIZABLE, @@ -191,7 +196,7 @@ def student_function_temp() -> Any: ), "traceback": None, } - return json.dumps(not_serializable_response) + return not_serializable_response def _find_blacklisted(name: str, fromlist: Sequence[str], import_blacklist: list[str]) -> str | None: @@ -447,7 +452,7 @@ def _find_helper_name_clashes(workspace_dir: str, helper_modules: dict[str, str] return clashes -async def _run_sandbox( +def _run_sandbox( *, setup_code: str | None, timeout: float, @@ -509,13 +514,10 @@ async def _run_sandbox( # Compile the setup code code_setup = compile(setup_code, "", "exec") with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture): - await asyncio.wait_for( - asyncio.get_event_loop().run_in_executor(executor, exec, code_setup, student_code_vars, local_vars), - timeout=timeout, - ) + executor.submit(exec, code_setup, student_code_vars, local_vars).result(timeout=timeout) - except asyncio.TimeoutError: - execution_error = asyncio.TimeoutError("Setup code execution timed out") + except concurrent.futures.TimeoutError: + execution_error = TimeoutError("Setup code execution timed out") # TODO need to create a different message for setup code errors. This should result # in a different error message reported from the test case. except Exception as e: @@ -543,13 +545,10 @@ async def _run_sandbox( main_source, main_file_name = load_main() main_code = compile(main_source, main_file_name, "exec") with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture): - await asyncio.wait_for( - asyncio.get_event_loop().run_in_executor(executor, exec, main_code, student_code_vars, student_code_vars), - timeout=timeout, - ) + executor.submit(exec, main_code, student_code_vars, student_code_vars).result(timeout=timeout) - except asyncio.TimeoutError: - execution_error = asyncio.TimeoutError(main_timeout_message) + except concurrent.futures.TimeoutError: + execution_error = TimeoutError(main_timeout_message) except Exception as e: execution_error = e # TODO this traceback only shows the last line with the exception. @@ -561,7 +560,7 @@ async def _run_sandbox( # Determine the status based on the type of error if execution_error is None: status = ProcessStatusCode.SUCCESS - elif isinstance(execution_error, asyncio.TimeoutError): + elif isinstance(execution_error, TimeoutError): status = ProcessStatusCode.TIMEOUT else: status = ProcessStatusCode.EXCEPTION @@ -578,7 +577,7 @@ async def _run_sandbox( return local_vars, student_code_vars, result_dict -async def student_code_runner( +def student_code_runner( setup_code: str | None, student_code: str, student_file_name: str, @@ -590,7 +589,7 @@ async def student_code_runner( names_for_user_list: list[str] | None, helper_modules: dict[str, str] | None, ) -> tuple[dict[str, Any], dict[str, Any], ProcessStartResponse]: - return await _run_sandbox( + return _run_sandbox( setup_code=setup_code, timeout=timeout, import_whitelist=import_whitelist, @@ -605,7 +604,7 @@ async def student_code_runner( ) -async def workspace_runner( +def workspace_runner( workspace_dir: str, exec_entry: str | None, setup_code: str | None, @@ -661,7 +660,7 @@ def load_entry() -> tuple[str, str]: load_main = load_entry - return await _run_sandbox( + return _run_sandbox( setup_code=setup_code, timeout=timeout, import_whitelist=import_whitelist, @@ -676,54 +675,47 @@ def load_entry() -> tuple[str, str]: ) -def encode_server_failure(message: str) -> bytes: - response: ServerFailureResponse = {"status": ServerStatusCode.FAILURE, "message": message} - return (json.dumps(response) + os.linesep).encode() +def server_failure_response(message: str) -> ServerFailureResponse: + return {"status": ServerStatusCode.FAILURE, "message": message} -async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: +def _serve(conn: Connection, worker_username: str | None, data_files_dir: str | None) -> None: """ - Reads lines from stdin asynchronously and responds on stdout. - Mimics a simple server handling requests. - """ - # try: - # json_message = json.loads(message) - # result = await asyncio.wait_for( - # asyncio.get_event_loop().run_in_executor(executor, _run_blocking_task, task_payload), timeout=timeout_seconds - # ) - # # Example: send an acknowledgement back - # response_message = {"status": "received", "data": json_message} - # writer.write(json.dumps(response_message).encode("utf-8") + b"\n") # Add newline for stream parsing - # await writer.drain() - # # ------------------------------------ - - # except json.JSONDecodeError as e: - # error_response = {"status": "error", "message": f"Invalid JSON: {e}"} - # writer.write(json.dumps(error_response).encode("utf-8") + b"\n") - # await writer.drain() - # except UnicodeDecodeError as e: - # error_response = {"status": "error", "message": f"Invalid UTF-8 encoding: {e}"} - # writer.write(json.dumps(error_response).encode("utf-8") + b"\n") - # await writer.drain() + Entry point for the sandbox child process. Drops privileges, then serves requests over the pipe. - try: - student_code_vars: None | dict = None - local_vars: None | dict = None - workspace_mode: bool = False + The grader spawns this with a multiprocessing.Process and holds the other end of the pipe. Each + request is a message dict received with conn.recv(); the response dict is sent with conn.send(). + The loop ends when the grader closes its end (conn.recv() raises EOFError) or the process is + terminated. + """ + # multiprocessing.Process has no preexec_fn, so the privilege drop is the first thing the child + # does, before it reads or runs anything on the grader's behalf. + if worker_username is not None and sys.platform != "win32": + drop_privileges(worker_username) + # With data files, the sandbox runs in a folder holding copies of them so student code can read + # them by relative path. subprocess set this with cwd=...; multiprocessing has no such option. + if data_files_dir is not None: + os.chdir(data_files_dir) + # Student code runs on a worker thread with no display, so matplotlib must not pick a GUI backend. + os.environ.setdefault("MPLBACKEND", "Agg") - async for line_bytes in reader: - line = line_bytes.decode().strip() - if not line: # Handle empty lines - continue + student_code_vars: None | dict = None + local_vars: None | dict = None + workspace_mode: bool = False - json_message = json.loads(line) + try: + while True: + try: + json_message = conn.recv() + except EOFError: + break msg_type = json_message.get("message_type") if msg_type == MessageType.START_WORKSPACE: ws_json_message: WorkspaceStartRequest = json_message workspace_mode = True - local_vars, student_code_vars, start_response = await workspace_runner( + local_vars, student_code_vars, start_response = workspace_runner( workspace_dir=ws_json_message["workspace_dir"], exec_entry=ws_json_message["exec_entry"], setup_code=ws_json_message["setup_code"], @@ -736,7 +728,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit helper_modules=ws_json_message["helper_modules"], ) - writer.write((json.dumps(start_response) + os.linesep).encode()) + conn.send(start_response) elif msg_type == MessageType.START: start_json_message: ProcessStartRequest = json_message @@ -754,7 +746,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit populate_linecache(student_code, student_file_name) - local_vars, student_code_vars, start_response = await student_code_runner( + local_vars, student_code_vars, start_response = student_code_runner( setup_code=setup_code, student_code=student_code, student_file_name=student_file_name, @@ -767,7 +759,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit helper_modules=start_json_message["helper_modules"], ) - writer.write((json.dumps(start_response) + os.linesep).encode()) + conn.send(start_response) elif msg_type == MessageType.QUERY_SETUP: assert local_vars is not None @@ -782,7 +774,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit else: setup_query_response = {"status": QueryStatusCode.NOT_FOUND, "value_encoded": ""} - writer.write((json.dumps(setup_query_response) + os.linesep).encode()) + conn.send(setup_query_response) elif msg_type == MessageType.QUERY: assert student_code_vars is not None @@ -799,22 +791,20 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit raise KeyError(var_to_query) except (KeyError, AttributeError, ImportError): - not_found_response: StudentQueryResponse = {"status": QueryStatusCode.NOT_FOUND, "value": "", "error": None} - encoded_query_response = json.dumps(not_found_response) + query_response: StudentQueryResponse = {"status": QueryStatusCode.NOT_FOUND, "value": "", "error": None} else: try: - query_response: StudentQueryResponse = {"status": QueryStatusCode.SUCCESS, "value": to_json(value), "error": None} - encoded_query_response = json.dumps(query_response) + # Send the value as a JSON string: see student_function_runner for why. + query_response = {"status": QueryStatusCode.SUCCESS, "value": json.dumps(to_json(value)), "error": None} except Exception as e: - not_serializable_response: StudentQueryResponse = { + query_response = { "status": QueryStatusCode.NOT_SERIALIZABLE, "value": None, "error": f"Variable '{var_to_query}' has a value of type {type(value).__name__}, " f"which can't be sent back to the grader: {e}", } - encoded_query_response = json.dumps(not_serializable_response) - writer.write((encoded_query_response + os.linesep).encode()) + conn.send(query_response) elif msg_type == MessageType.QUERY_FUNCTION: assert student_code_vars is not None @@ -825,64 +815,18 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit kwargs = deserialize_object_unsafe(query_function_json_message["kwargs_encoded"]) query_timeout = query_function_json_message["query_timeout"] - encoded_function_response = await student_function_runner( + function_response = student_function_runner( student_code_vars, func_name, query_timeout, args, kwargs, workspace_mode=workspace_mode ) - writer.write((encoded_function_response + os.linesep).encode()) - - # TODO handle cases of different payloads - # The first payload should be student code - if line.lower() == "exit": - writer.write(("Goodbye!" + os.linesep).encode()) - await writer.drain() - break # Exit the loop and terminate the server + conn.send(function_response) - # Simulate processing a request - # response = f"Server processed: '{line.upper()}'\n" - - await writer.drain() # Ensure the response is written to stdout - - except asyncio.CancelledError: - writer.write(encode_server_failure("Server was cancelled.")) - except asyncio.TimeoutError: - writer.write(encode_server_failure("Student code timed out.")) except Exception as e: - writer.write(encode_server_failure(f"An error occurred: {e}")) - finally: - # It's good practice to close transports and writers - # print("Closing server connections...") - await writer.drain() # Ensure all data is sent before closing - writer.close() - await writer.wait_closed() # Wait for the writer to finish closing - - -async def main(): - """ - Starts the asynchronous socket server. - """ - # Ensure ProactorEventLoop is used on Windows for robust socket operations - if sys.platform == "win32": + # Report an unexpected server-level failure (e.g. a message that couldn't be unpickled) so + # the grader can explain it, then stop; the grader closes its end on the next query. try: - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - print("Using ProactorEventLoop on Windows.", file=sys.stderr) - except NotImplementedError: - print("ProactorEventLoop not available, continuing with default loop.", file=sys.stderr) - - line_limit = 10 * 1024 * 1024 # 10 MB line limit to handle large messages - - # Start the server, binding to the specified host and port - server = await asyncio.start_server(handle_client, HOST, 0, limit=line_limit) - addr = server.sockets[0].getsockname() - print(f"{addr[0]}, {addr[1]}", flush=True) - - async with server: - # Run forever, or until the server is explicitly stopped - await server.serve_forever() - - -if __name__ == "__main__": - # Student code runs on a worker thread with no display, so matplotlib must not pick a GUI backend. - os.environ.setdefault("MPLBACKEND", "Agg") - asyncio.run(main()) + conn.send(server_failure_response(f"An error occurred: {e}")) + except Exception: + pass + finally: + conn.close() diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 6b87ef8..4f68ff0 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -1,20 +1,21 @@ import base64 import json import logging +import multiprocessing import os import shutil import signal -import socket import stat -import subprocess import sys import tempfile from collections.abc import Callable -from importlib.resources import files +from multiprocessing.connection import Connection +from multiprocessing.context import SpawnProcess from pathlib import Path from typing import Any from typing import NamedTuple +from ._student_code_runner import _serve from .json_utils import from_json from .utils import FunctionStatusCode from .utils import MessageType @@ -31,16 +32,17 @@ from .utils import StudentQueryResponse from .utils import WorkspaceStartRequest from .utils import deserialize_object_unsafe -from .utils import drop_privileges from .utils import format_call from .utils import serialize_object_unsafe DataFixture = dict[str, Any] -SCRIPT_PATH = str(files("pytest_prairielearn_grader").joinpath("_student_code_runner.py")) +# The sandbox runs in a child process started with the "spawn" method. spawn gives the child a fresh +# interpreter instead of forking the grader's memory, which would break the sandbox's isolation. +_MP_CONTEXT = multiprocessing.get_context("spawn") DEFAULT_TIMEOUT = 1.0 # Extra time to wait for the start response, so that the sandbox's own -# initialization timeout is reported instead of racing against the socket timeout. +# initialization timeout is reported instead of racing against the read timeout. START_RESPONSE_GRACE = 0.5 # Seconds to wait for the sandbox's process to exit after it closes the connection, to read why it stopped. STOPPED_PROCESS_WAIT = 1.0 @@ -213,12 +215,13 @@ def to_dict(self) -> dict: class _SandboxBase: """ - Shared subprocess/socket plumbing used by StudentFixture and WorkspaceFixture. + Shared child-process/pipe plumbing used by StudentFixture and WorkspaceFixture. Not intended to be instantiated directly. """ - process: subprocess.Popen | None - student_socket: socket.socket | None + process: SpawnProcess | None + _conn: Connection | None + _read_timeout: float import_whitelist: list[str] | None import_blacklist: list[str] | None starting_vars: dict[str, Any] | None @@ -253,60 +256,47 @@ def __init__( self.data_files = data_files self._data_files_dir = None self.process = None - self.student_socket = None + self._conn = None + self._read_timeout = DEFAULT_TIMEOUT self._accumulated_stdout = [] self._server_failure = None self._timed_out_call = None def _start_server(self, initialization_timeout: float, build_start_message: Callable[[], Any]) -> ProcessStartResponse: - """Spawn the subprocess, send the start message it builds, and read the start response.""" + """Spawn the child process, send the start message it builds, and read the start response.""" self._start_subprocess() json_message = build_start_message() - self._connect_to_server(initialization_timeout) self._send_json_object(json_message) return self._read_start_response(initialization_timeout) def _assert_process_running(self) -> None: assert self.process is not None, "Sandbox process is not running. Please start it first." - process_return_code = self.process.poll() - if process_return_code is not None: - raise RuntimeError(f"Sandbox process terminated with code {process_return_code}.") + if self.process.exitcode is not None: + raise RuntimeError(f"Sandbox process terminated with code {self.process.exitcode}.") def _send_json_object(self, json_object: Any) -> None: - assert self.student_socket is not None, "Socket is not connected. Please start the sandbox server first." - self.student_socket.sendall((json.dumps(json_object) + os.linesep).encode("utf-8")) - - def _read_from_socket(self) -> bytes: - """Read from the socket until a line terminator is found.""" - buffer = bytearray() - terminator = os.linesep.encode("utf-8") - max_len: int | None = None # TODO add max length parameter? - assert self.student_socket is not None, "Socket is not connected. Please start the sandbox server first." - chunk_size = 4096 - chunk: bytes = b"" - # TODO maybe set a hard iteration limit to avoid infinite loops? - while (idx := chunk.rfind(terminator)) == -1: - try: - chunk = self.student_socket.recv(chunk_size) - except TimeoutError as e: - raise TimeoutError("Socket read timed out.") from e - if not chunk: - self._server_failure = self._describe_stopped_sandbox() - self._assert_no_server_failure() - buffer.extend(chunk) - if max_len is not None and len(buffer) >= max_len: - raise Exception(f"Maximum read length of {max_len} exceeded.") - loc = len(buffer) - len(chunk) + idx + len(terminator) - return buffer[:loc] + assert self._conn is not None, "Pipe is not connected. Please start the sandbox server first." + self._conn.send(json_object) + + def _recv_message(self) -> Any: + """Read one message from the pipe, waiting up to the current read timeout for it to arrive.""" + assert self._conn is not None, "Pipe is not connected. Please start the sandbox server first." + if not self._conn.poll(self._read_timeout): + raise TimeoutError("Pipe read timed out.") + try: + return self._conn.recv() + except EOFError as e: + # The child closed its end without sending a reply, i.e. its process stopped. + self._server_failure = self._describe_stopped_sandbox() + self._assert_no_server_failure() + raise RuntimeError(self._server_failure) from e # unreachable: the assert above raises def _describe_stopped_sandbox(self) -> str: """Say why the sandbox closed the connection, from how its process exited.""" return_code = None if self.process is not None: - try: - return_code = self.process.wait(timeout=STOPPED_PROCESS_WAIT) - except subprocess.TimeoutExpired: - pass + self.process.join(timeout=STOPPED_PROCESS_WAIT) + return_code = self.process.exitcode # The kernel's out-of-memory killer, and container memory limits, stop a process with SIGKILL. if sys.platform != "win32" and return_code == -signal.SIGKILL: return "The sandbox running your code was killed, probably because your code used too much memory." @@ -322,50 +312,42 @@ def _assert_no_server_failure(self) -> None: def _read_query_response(self) -> Any: """Read a response to a query, raising a clear error if the server failed and closed the connection.""" - data = json.loads(self._read_from_socket().decode()) + data = self._recv_message() if data.get("status") == ServerStatusCode.FAILURE: self._server_failure = f"The sandbox stopped unexpectedly, so later queries won't work: {data['message']}" self._assert_no_server_failure() + # A successful query or function call sends its value as a JSON string; decode it back to a + # structure so the response matches what callers (and from_json) expect. + if data.get("status") == QueryStatusCode.SUCCESS: + data["value"] = json.loads(data["value"]) return data def _start_subprocess(self) -> None: - """Spawn the runner subprocess and store it in ``self.process``.""" - - def try_drop_privileges() -> None: - if self.worker_username is not None: - drop_privileges(self.worker_username) - + """Spawn the runner child process and store it in ``self.process``.""" # With data files, the sandbox starts in a folder holding copies of them, so student code can # read them by relative path. Without them, the starting folder is left as it was. if self.data_files is not None: self._data_files_dir = _make_data_files_dir(self.data_files) - self.process = subprocess.Popen( - args=(sys.executable, SCRIPT_PATH), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=self._data_files_dir, - preexec_fn=None if sys.platform == "win32" else try_drop_privileges, + parent_conn, child_conn = _MP_CONTEXT.Pipe() + data_files_dir = None if self._data_files_dir is None else str(self._data_files_dir) + self.process = _MP_CONTEXT.Process( + target=_serve, + args=(child_conn, self.worker_username, data_files_dir), + daemon=True, ) + self.process.start() + # Close the child's end in the grader so that if the child process dies, the grader's recv + # sees EOF instead of blocking on a handle it still holds open itself. + child_conn.close() + self._conn = parent_conn self._assert_process_running() - def _connect_to_server(self, timeout: float) -> None: - """Read the host/port printed by the subprocess and connect the socket.""" - assert self.process is not None - assert self.process.stdout is not None - host, port = self.process.stdout.readline().decode().strip().split(",") - self.student_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.student_socket.settimeout(timeout) - self.student_socket.connect((host, int(port))) - def _read_start_response(self, initialization_timeout: float) -> ProcessStartResponse: """Read and return the ProcessStartResponse sent after the start message.""" - assert self.student_socket is not None - self.student_socket.settimeout(initialization_timeout + START_RESPONSE_GRACE) + self._read_timeout = initialization_timeout + START_RESPONSE_GRACE try: - data = self._read_from_socket().decode() - res: ProcessStartResponse = json.loads(data) + res: ProcessStartResponse = self._recv_message() if res.get("stdout"): self._accumulated_stdout.append(res["stdout"]) except Exception as e: @@ -383,8 +365,7 @@ def query_raw(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT self._assert_no_server_failure() self._assert_process_running() json_message = StudentQueryRequest(message_type=MessageType.QUERY, var=var_to_query, query_timeout=query_timeout) - assert self.student_socket is not None - self.student_socket.settimeout(query_timeout) + self._read_timeout = query_timeout self._send_json_object(json_message) try: data: StudentQueryResponse = self._read_query_response() @@ -424,9 +405,8 @@ def query_function_raw( kwargs_encoded=serialize_object_unsafe(kwargs), query_timeout=query_timeout, ) - assert self.student_socket is not None - self.student_socket.settimeout(query_timeout) - self.student_socket.sendall((json.dumps(json_message) + os.linesep).encode("utf-8")) + self._read_timeout = query_timeout + self._send_json_object(json_message) try: data: StudentFunctionResponse = self._read_query_response() except TimeoutError as error: @@ -468,12 +448,12 @@ def get_accumulated_stdout(self) -> str: return "".join(self._accumulated_stdout) def _cleanup(self) -> None: - if self.student_socket is not None: - self.student_socket.close() - self.student_socket = None + if self._conn is not None: + self._conn.close() + self._conn = None if self.process is not None: self.process.terminate() - self.process.wait() + self.process.join() self.process = None if self._data_files_dir is not None: _remove_data_files_dir(self._data_files_dir) @@ -560,9 +540,8 @@ def query_setup_raw(self, var_to_query: str) -> SetupQueryResponse: self._assert_no_server_failure() self._assert_process_running() json_message: SetupQueryRequest = {"message_type": MessageType.QUERY_SETUP, "var": var_to_query} - assert self.student_socket is not None, "Student socket is not connected. Please start the student code server first." self._send_json_object(json_message) - data: SetupQueryResponse = json.loads(self._read_from_socket().decode()) + data: SetupQueryResponse = self._recv_message() return data def query_setup(self, var_to_query: str) -> Any: diff --git a/tests/test_subtests.py b/tests/test_subtests.py index a6d4462..4f73fc0 100644 --- a/tests/test_subtests.py +++ b/tests/test_subtests.py @@ -229,25 +229,25 @@ def test_second(module_sandbox): def test_timeout_reported_by_student_process_stops_sandbox(pytester: pytest.Pytester) -> None: - # The socket waits a second longer than the call's timeout, so the student process's own + # The grader waits a second longer than the call's timeout, so the student process's own # timeout reply always arrives first. result = run_grader( pytester, """ - class SlowSocket: - def __init__(self, sock): - self.sock = sock + class SlowConnection: + def __init__(self, conn): + self.conn = conn - def settimeout(self, timeout): - self.sock.settimeout(timeout + 1) + def poll(self, timeout=None): + return self.conn.poll(timeout + 1 if timeout is not None else None) def __getattr__(self, name): - return getattr(self.sock, name) + return getattr(self.conn, name) @pytest.mark.grading_data(name="case", points=1) def test_case(sandbox, subtests, monkeypatch): - monkeypatch.setattr(sandbox, "student_socket", SlowSocket(sandbox.student_socket)) + monkeypatch.setattr(sandbox, "_conn", SlowConnection(sandbox._conn)) for x in [1, 4, 2, 5]: with subtests.test(): assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x, query_timeout=0.2) From cdc510c623f227211d1943f968e7b8eb4501ba63 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Wed, 18 Nov 2026 18:08:07 -0500 Subject: [PATCH 3/6] Keep plugin.py in sync with main after the squash merge --- src/pytest_prairielearn_grader/plugin.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 66b410b..0f25504 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -322,7 +322,13 @@ def _resolve_sandbox_config( the names_for_user values into starting_vars. """ initialization_timeout: float = DEFAULT_SANDBOX_TIMEOUT - params_dict = data_json.get("params", {}) if data_json is not None else {} + + if data_json is None: + params_dict = {} + logger.debug("No data.json provided, using empty params") + else: + params_dict = data_json.get("params", {}) + logger.debug(f"Loaded params from data.json: {list(params_dict.keys())}") # Check for module-level ConfigObject that overrides all other configuration config_dict: dict[str, Any] = {} @@ -333,6 +339,7 @@ def _resolve_sandbox_config( config_object = request.module.autograder_config if not isinstance(config_object, ConfigObject): pytest.fail(f"autograder_config must be a ConfigObject instance, got {type(config_object).__name__}") + logger.debug("Using module-level autograder_config - overriding all other configuration sources") config_dict = config_object.to_dict() # Handle timeout and starting_vars separately as they're not in to_dict() initialization_timeout = config_object.sandbox_timeout @@ -344,11 +351,13 @@ def _resolve_sandbox_config( # This allows setting: sandbox_timeout = 0.5 at module level if hasattr(request, "module") and hasattr(request.module, "sandbox_timeout"): initialization_timeout = request.module.sandbox_timeout + logger.debug(f"Using module-level timeout: {initialization_timeout}s") # Check for the custom mark (overrides module-level setting) marker = request.node.get_closest_marker("sandbox_timeout") if marker and marker.args: initialization_timeout = marker.args[0] + logger.debug(f"Using test-specific timeout override: {initialization_timeout}s") config_starting_vars = {} From 7e88ac20677b4903760aeae86ff2706a840bc261 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Wed, 18 Nov 2026 18:12:00 -0500 Subject: [PATCH 4/6] Use contextlib.suppress for the best-effort failure send (ruff S110) --- src/pytest_prairielearn_grader/_student_code_runner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 687cdc2..19c87c1 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -14,6 +14,7 @@ from collections.abc import Sequence from contextlib import redirect_stderr from contextlib import redirect_stdout +from contextlib import suppress from copy import deepcopy from multiprocessing.connection import Connection from typing import Any @@ -823,10 +824,9 @@ def _serve(conn: Connection, worker_username: str | None, data_files_dir: str | except Exception as e: # Report an unexpected server-level failure (e.g. a message that couldn't be unpickled) so - # the grader can explain it, then stop; the grader closes its end on the next query. - try: + # the grader can explain it, then stop; the grader closes its end on the next query. The send + # itself can fail if the grader already closed the pipe, and there's nothing left to do then. + with suppress(Exception): conn.send(server_failure_response(f"An error occurred: {e}")) - except Exception: - pass finally: conn.close() From 89f9c14767ffcf45349372192f03305e71e68875 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Wed, 18 Nov 2026 18:15:51 -0500 Subject: [PATCH 5/6] Inline server_failure_response into its single caller --- src/pytest_prairielearn_grader/_student_code_runner.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 19c87c1..a8fa0b8 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -676,10 +676,6 @@ def load_entry() -> tuple[str, str]: ) -def server_failure_response(message: str) -> ServerFailureResponse: - return {"status": ServerStatusCode.FAILURE, "message": message} - - def _serve(conn: Connection, worker_username: str | None, data_files_dir: str | None) -> None: """ Entry point for the sandbox child process. Drops privileges, then serves requests over the pipe. @@ -826,7 +822,8 @@ def _serve(conn: Connection, worker_username: str | None, data_files_dir: str | # Report an unexpected server-level failure (e.g. a message that couldn't be unpickled) so # the grader can explain it, then stop; the grader closes its end on the next query. The send # itself can fail if the grader already closed the pipe, and there's nothing left to do then. + failure: ServerFailureResponse = {"status": ServerStatusCode.FAILURE, "message": f"An error occurred: {e}"} with suppress(Exception): - conn.send(server_failure_response(f"An error occurred: {e}")) + conn.send(failure) finally: conn.close() From 32e21b64bea3a4da204257d42fdb930a7666b693 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Wed, 18 Nov 2026 18:20:21 -0500 Subject: [PATCH 6/6] Add a large-payload round-trip test for the pipe transport --- tests/test_large_payload.py | 46 +++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/test_large_payload.py diff --git a/tests/test_large_payload.py b/tests/test_large_payload.py new file mode 100644 index 0000000..6cfd412 --- /dev/null +++ b/tests/test_large_payload.py @@ -0,0 +1,46 @@ +import json +import textwrap +from typing import Any + +import pytest + +pytest_plugins = ("pytester",) + +# A list far larger than the OS pipe buffer (~64 KiB) once serialized, so a response and an argument +# each span many underlying reads/writes. The socket transport this replaced capped a request at +# 10 MB; the pipe has no such cap, so this also guards the multi-chunk framing in both directions. +STUDENT_CODE = """ +big_list = list(range(200_000)) + + +def sum_values(values): + return sum(values) +""" + +TEST_HEADER = """ +import pytest +""" + + +def run_grader(pytester: pytest.Pytester, tests: str) -> dict[str, dict[str, Any]]: + pytester.path.joinpath("test_grader.py").write_text(TEST_HEADER + textwrap.dedent(tests)) + pytester.mkdir("test_grader").joinpath("student_code.py").write_text(STUDENT_CODE) + pytester.runpytest("test_grader.py") + results = json.loads(pytester.path.joinpath("autograder_results.json").read_text()) + return {result["name"]: result for result in results["tests"]} + + +def test_large_value_and_argument_round_trip(pytester: pytest.Pytester) -> None: + result = run_grader( + pytester, + """ + @pytest.mark.grading_data(name="case", points=1) + def test_case(sandbox): + big = sandbox.query("big_list") + assert big == list(range(200_000)) + assert sandbox.query_function("sum_values", big) == sum(range(200_000)) + """, + )["case"] + + assert result["points_frac"] == 1 + assert result["outcome"] == "passed"