diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 0ba7abd..a8fa0b8 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -1,4 +1,3 @@ -import asyncio import builtins import concurrent.futures import importlib @@ -15,7 +14,9 @@ from collections.abc import Sequence from contextlib import redirect_stderr from contextlib import redirect_stdout +from contextlib import suppress from copy import deepcopy +from multiprocessing.connection import Connection from typing import Any from pytest_prairielearn_grader.json_utils import from_server_json @@ -39,6 +40,7 @@ from pytest_prairielearn_grader.utils import StudentQueryResponse from pytest_prairielearn_grader.utils import WorkspaceStartRequest from pytest_prairielearn_grader.utils import deserialize_object_unsafe +from pytest_prairielearn_grader.utils import drop_privileges from pytest_prairielearn_grader.utils import get_builtins from pytest_prairielearn_grader.utils import serialize_object_unsafe @@ -49,8 +51,6 @@ # is later replaced by a workspace_importer in workspace_runner. _REAL_IMPORT: ImportFunction = builtins.__import__ # type: ignore[assignment] -HOST = "127.0.0.1" # Loopback address, means "this computer only" - # Global ThreadPoolExecutor for CPU-bound tasks # It's good practice to create this once and reuse it. @@ -120,15 +120,15 @@ def _look_up_function(name: str, student_code_vars: dict[str, Any], workspace_mo return student_code_vars[name] -async def student_function_runner( +def student_function_runner( student_code_vars: dict[str, Any], func_name: str, timeout: float, args_tup: Any, kwargs_dict: Any, workspace_mode: bool = False, -) -> str: - """Run a student function and return its StudentFunctionResponse encoded as a JSON string.""" +) -> StudentFunctionResponse: + """Run a student function and return its StudentFunctionResponse.""" stdout_capture = io.StringIO() stderr_capture = io.StringIO() execution_error: Exception | None = None @@ -144,12 +144,11 @@ def student_function_temp() -> Any: return student_function(*args_tup, **kwargs_dict) with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture): - future = asyncio.get_event_loop().run_in_executor(executor, student_function_temp) - done, _ = await asyncio.wait({future}, timeout=timeout) - if done: - result = future.result() - else: - execution_error = asyncio.TimeoutError(f"Function timed out after {timeout} seconds") + future = executor.submit(student_function_temp) + try: + result = future.result(timeout=timeout) + except concurrent.futures.TimeoutError: + execution_error = TimeoutError(f"Function timed out after {timeout} seconds") timed_out = True except FunctionNotFoundError as e: execution_error = NameError(str(e)) @@ -176,9 +175,16 @@ def student_function_temp() -> Any: "traceback": exception_traceback, } + # On any execution error the call produced no value, so leave it as None. + if execution_error is not None: + return function_response + try: - function_response["value"] = to_json(result) - return json.dumps(function_response) + # The value is sent back to the grader as a JSON string so that only JSON data, never a + # student-controlled object, crosses the pipe into the grader process. json.dumps also + # raises here for a value that can't be represented, which drives the NOT_SERIALIZABLE path. + function_response["value"] = json.dumps(to_json(result)) + return function_response except Exception as e: not_serializable_response: StudentFunctionResponse = { "status": FunctionStatusCode.NOT_SERIALIZABLE, @@ -191,7 +197,7 @@ def student_function_temp() -> Any: ), "traceback": None, } - return json.dumps(not_serializable_response) + return not_serializable_response def _find_blacklisted(name: str, fromlist: Sequence[str], import_blacklist: list[str]) -> str | None: @@ -447,7 +453,7 @@ def _find_helper_name_clashes(workspace_dir: str, helper_modules: dict[str, str] return clashes -async def _run_sandbox( +def _run_sandbox( *, setup_code: str | None, timeout: float, @@ -509,13 +515,10 @@ async def _run_sandbox( # Compile the setup code code_setup = compile(setup_code, "", "exec") with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture): - await asyncio.wait_for( - asyncio.get_event_loop().run_in_executor(executor, exec, code_setup, student_code_vars, local_vars), - timeout=timeout, - ) + executor.submit(exec, code_setup, student_code_vars, local_vars).result(timeout=timeout) - except asyncio.TimeoutError: - execution_error = asyncio.TimeoutError("Setup code execution timed out") + except concurrent.futures.TimeoutError: + execution_error = TimeoutError("Setup code execution timed out") # TODO need to create a different message for setup code errors. This should result # in a different error message reported from the test case. except Exception as e: @@ -543,13 +546,10 @@ async def _run_sandbox( main_source, main_file_name = load_main() main_code = compile(main_source, main_file_name, "exec") with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture): - await asyncio.wait_for( - asyncio.get_event_loop().run_in_executor(executor, exec, main_code, student_code_vars, student_code_vars), - timeout=timeout, - ) + executor.submit(exec, main_code, student_code_vars, student_code_vars).result(timeout=timeout) - except asyncio.TimeoutError: - execution_error = asyncio.TimeoutError(main_timeout_message) + except concurrent.futures.TimeoutError: + execution_error = TimeoutError(main_timeout_message) except Exception as e: execution_error = e # TODO this traceback only shows the last line with the exception. @@ -561,7 +561,7 @@ async def _run_sandbox( # Determine the status based on the type of error if execution_error is None: status = ProcessStatusCode.SUCCESS - elif isinstance(execution_error, asyncio.TimeoutError): + elif isinstance(execution_error, TimeoutError): status = ProcessStatusCode.TIMEOUT else: status = ProcessStatusCode.EXCEPTION @@ -578,7 +578,7 @@ async def _run_sandbox( return local_vars, student_code_vars, result_dict -async def student_code_runner( +def student_code_runner( setup_code: str | None, student_code: str, student_file_name: str, @@ -590,7 +590,7 @@ async def student_code_runner( names_for_user_list: list[str] | None, helper_modules: dict[str, str] | None, ) -> tuple[dict[str, Any], dict[str, Any], ProcessStartResponse]: - return await _run_sandbox( + return _run_sandbox( setup_code=setup_code, timeout=timeout, import_whitelist=import_whitelist, @@ -605,7 +605,7 @@ async def student_code_runner( ) -async def workspace_runner( +def workspace_runner( workspace_dir: str, exec_entry: str | None, setup_code: str | None, @@ -661,7 +661,7 @@ def load_entry() -> tuple[str, str]: load_main = load_entry - return await _run_sandbox( + return _run_sandbox( setup_code=setup_code, timeout=timeout, import_whitelist=import_whitelist, @@ -676,31 +676,43 @@ def load_entry() -> tuple[str, str]: ) -def encode_server_failure(message: str) -> bytes: - response: ServerFailureResponse = {"status": ServerStatusCode.FAILURE, "message": message} - return (json.dumps(response) + os.linesep).encode() - +def _serve(conn: Connection, worker_username: str | None, data_files_dir: str | None) -> None: + """ + Entry point for the sandbox child process. Drops privileges, then serves requests over the pipe. -async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - """Read JSON request lines from the socket and write a JSON response line for each.""" - try: - student_code_vars: None | dict = None - local_vars: None | dict = None - workspace_mode: bool = False + The grader spawns this with a multiprocessing.Process and holds the other end of the pipe. Each + request is a message dict received with conn.recv(); the response dict is sent with conn.send(). + The loop ends when the grader closes its end (conn.recv() raises EOFError) or the process is + terminated. + """ + # multiprocessing.Process has no preexec_fn, so the privilege drop is the first thing the child + # does, before it reads or runs anything on the grader's behalf. + if worker_username is not None and sys.platform != "win32": + drop_privileges(worker_username) + # With data files, the sandbox runs in a folder holding copies of them so student code can read + # them by relative path. subprocess set this with cwd=...; multiprocessing has no such option. + if data_files_dir is not None: + os.chdir(data_files_dir) + # Student code runs on a worker thread with no display, so matplotlib must not pick a GUI backend. + os.environ.setdefault("MPLBACKEND", "Agg") - async for line_bytes in reader: - line = line_bytes.decode().strip() - if not line: # Handle empty lines - continue + student_code_vars: None | dict = None + local_vars: None | dict = None + workspace_mode: bool = False - json_message = json.loads(line) + try: + while True: + try: + json_message = conn.recv() + except EOFError: + break msg_type = json_message.get("message_type") if msg_type == MessageType.START_WORKSPACE: ws_json_message: WorkspaceStartRequest = json_message workspace_mode = True - local_vars, student_code_vars, start_response = await workspace_runner( + local_vars, student_code_vars, start_response = workspace_runner( workspace_dir=ws_json_message["workspace_dir"], exec_entry=ws_json_message["exec_entry"], setup_code=ws_json_message["setup_code"], @@ -713,7 +725,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit helper_modules=ws_json_message["helper_modules"], ) - writer.write((json.dumps(start_response) + os.linesep).encode()) + conn.send(start_response) elif msg_type == MessageType.START: start_json_message: ProcessStartRequest = json_message @@ -731,7 +743,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit populate_linecache(student_code, student_file_name) - local_vars, student_code_vars, start_response = await student_code_runner( + local_vars, student_code_vars, start_response = student_code_runner( setup_code=setup_code, student_code=student_code, student_file_name=student_file_name, @@ -744,7 +756,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit helper_modules=start_json_message["helper_modules"], ) - writer.write((json.dumps(start_response) + os.linesep).encode()) + conn.send(start_response) elif msg_type == MessageType.QUERY_SETUP: assert local_vars is not None @@ -759,7 +771,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit else: setup_query_response = {"status": QueryStatusCode.NOT_FOUND, "value_encoded": ""} - writer.write((json.dumps(setup_query_response) + os.linesep).encode()) + conn.send(setup_query_response) elif msg_type == MessageType.QUERY: assert student_code_vars is not None @@ -776,22 +788,20 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit raise KeyError(var_to_query) except (KeyError, AttributeError, ImportError): - not_found_response: StudentQueryResponse = {"status": QueryStatusCode.NOT_FOUND, "value": "", "error": None} - encoded_query_response = json.dumps(not_found_response) + query_response: StudentQueryResponse = {"status": QueryStatusCode.NOT_FOUND, "value": "", "error": None} else: try: - query_response: StudentQueryResponse = {"status": QueryStatusCode.SUCCESS, "value": to_json(value), "error": None} - encoded_query_response = json.dumps(query_response) + # Send the value as a JSON string: see student_function_runner for why. + query_response = {"status": QueryStatusCode.SUCCESS, "value": json.dumps(to_json(value)), "error": None} except Exception as e: - not_serializable_response: StudentQueryResponse = { + query_response = { "status": QueryStatusCode.NOT_SERIALIZABLE, "value": None, "error": f"Variable '{var_to_query}' has a value of type {type(value).__name__}, " f"which can't be sent back to the grader: {e}", } - encoded_query_response = json.dumps(not_serializable_response) - writer.write((encoded_query_response + os.linesep).encode()) + conn.send(query_response) elif msg_type == MessageType.QUERY_FUNCTION: assert student_code_vars is not None @@ -802,54 +812,18 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit kwargs = deserialize_object_unsafe(query_function_json_message["kwargs_encoded"]) query_timeout = query_function_json_message["query_timeout"] - encoded_function_response = await student_function_runner( + function_response = student_function_runner( student_code_vars, func_name, query_timeout, args, kwargs, workspace_mode=workspace_mode ) - writer.write((encoded_function_response + os.linesep).encode()) - - await writer.drain() # Ensure the response is written to stdout + conn.send(function_response) - except asyncio.CancelledError: - writer.write(encode_server_failure("Server was cancelled.")) - except asyncio.TimeoutError: - writer.write(encode_server_failure("Student code timed out.")) except Exception as e: - writer.write(encode_server_failure(f"An error occurred: {e}")) + # Report an unexpected server-level failure (e.g. a message that couldn't be unpickled) so + # the grader can explain it, then stop; the grader closes its end on the next query. The send + # itself can fail if the grader already closed the pipe, and there's nothing left to do then. + failure: ServerFailureResponse = {"status": ServerStatusCode.FAILURE, "message": f"An error occurred: {e}"} + with suppress(Exception): + conn.send(failure) finally: - # It's good practice to close transports and writers - # print("Closing server connections...") - await writer.drain() # Ensure all data is sent before closing - writer.close() - await writer.wait_closed() # Wait for the writer to finish closing - - -async def main(): - """ - Starts the asynchronous socket server. - """ - # Ensure ProactorEventLoop is used on Windows for robust socket operations - if sys.platform == "win32": - try: - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - print("Using ProactorEventLoop on Windows.", file=sys.stderr) - except NotImplementedError: - print("ProactorEventLoop not available, continuing with default loop.", file=sys.stderr) - - line_limit = 10 * 1024 * 1024 # 10 MB line limit to handle large messages - - # Start the server, binding to the specified host and port - server = await asyncio.start_server(handle_client, HOST, 0, limit=line_limit) - addr = server.sockets[0].getsockname() - print(f"{addr[0]}, {addr[1]}", flush=True) - - async with server: - # Run forever, or until the server is explicitly stopped - await server.serve_forever() - - -if __name__ == "__main__": - # Student code runs on a worker thread with no display, so matplotlib must not pick a GUI backend. - os.environ.setdefault("MPLBACKEND", "Agg") - asyncio.run(main()) + conn.close() diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 6b87ef8..4f68ff0 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -1,20 +1,21 @@ import base64 import json import logging +import multiprocessing import os import shutil import signal -import socket import stat -import subprocess import sys import tempfile from collections.abc import Callable -from importlib.resources import files +from multiprocessing.connection import Connection +from multiprocessing.context import SpawnProcess from pathlib import Path from typing import Any from typing import NamedTuple +from ._student_code_runner import _serve from .json_utils import from_json from .utils import FunctionStatusCode from .utils import MessageType @@ -31,16 +32,17 @@ from .utils import StudentQueryResponse from .utils import WorkspaceStartRequest from .utils import deserialize_object_unsafe -from .utils import drop_privileges from .utils import format_call from .utils import serialize_object_unsafe DataFixture = dict[str, Any] -SCRIPT_PATH = str(files("pytest_prairielearn_grader").joinpath("_student_code_runner.py")) +# The sandbox runs in a child process started with the "spawn" method. spawn gives the child a fresh +# interpreter instead of forking the grader's memory, which would break the sandbox's isolation. +_MP_CONTEXT = multiprocessing.get_context("spawn") DEFAULT_TIMEOUT = 1.0 # Extra time to wait for the start response, so that the sandbox's own -# initialization timeout is reported instead of racing against the socket timeout. +# initialization timeout is reported instead of racing against the read timeout. START_RESPONSE_GRACE = 0.5 # Seconds to wait for the sandbox's process to exit after it closes the connection, to read why it stopped. STOPPED_PROCESS_WAIT = 1.0 @@ -213,12 +215,13 @@ def to_dict(self) -> dict: class _SandboxBase: """ - Shared subprocess/socket plumbing used by StudentFixture and WorkspaceFixture. + Shared child-process/pipe plumbing used by StudentFixture and WorkspaceFixture. Not intended to be instantiated directly. """ - process: subprocess.Popen | None - student_socket: socket.socket | None + process: SpawnProcess | None + _conn: Connection | None + _read_timeout: float import_whitelist: list[str] | None import_blacklist: list[str] | None starting_vars: dict[str, Any] | None @@ -253,60 +256,47 @@ def __init__( self.data_files = data_files self._data_files_dir = None self.process = None - self.student_socket = None + self._conn = None + self._read_timeout = DEFAULT_TIMEOUT self._accumulated_stdout = [] self._server_failure = None self._timed_out_call = None def _start_server(self, initialization_timeout: float, build_start_message: Callable[[], Any]) -> ProcessStartResponse: - """Spawn the subprocess, send the start message it builds, and read the start response.""" + """Spawn the child process, send the start message it builds, and read the start response.""" self._start_subprocess() json_message = build_start_message() - self._connect_to_server(initialization_timeout) self._send_json_object(json_message) return self._read_start_response(initialization_timeout) def _assert_process_running(self) -> None: assert self.process is not None, "Sandbox process is not running. Please start it first." - process_return_code = self.process.poll() - if process_return_code is not None: - raise RuntimeError(f"Sandbox process terminated with code {process_return_code}.") + if self.process.exitcode is not None: + raise RuntimeError(f"Sandbox process terminated with code {self.process.exitcode}.") def _send_json_object(self, json_object: Any) -> None: - assert self.student_socket is not None, "Socket is not connected. Please start the sandbox server first." - self.student_socket.sendall((json.dumps(json_object) + os.linesep).encode("utf-8")) - - def _read_from_socket(self) -> bytes: - """Read from the socket until a line terminator is found.""" - buffer = bytearray() - terminator = os.linesep.encode("utf-8") - max_len: int | None = None # TODO add max length parameter? - assert self.student_socket is not None, "Socket is not connected. Please start the sandbox server first." - chunk_size = 4096 - chunk: bytes = b"" - # TODO maybe set a hard iteration limit to avoid infinite loops? - while (idx := chunk.rfind(terminator)) == -1: - try: - chunk = self.student_socket.recv(chunk_size) - except TimeoutError as e: - raise TimeoutError("Socket read timed out.") from e - if not chunk: - self._server_failure = self._describe_stopped_sandbox() - self._assert_no_server_failure() - buffer.extend(chunk) - if max_len is not None and len(buffer) >= max_len: - raise Exception(f"Maximum read length of {max_len} exceeded.") - loc = len(buffer) - len(chunk) + idx + len(terminator) - return buffer[:loc] + assert self._conn is not None, "Pipe is not connected. Please start the sandbox server first." + self._conn.send(json_object) + + def _recv_message(self) -> Any: + """Read one message from the pipe, waiting up to the current read timeout for it to arrive.""" + assert self._conn is not None, "Pipe is not connected. Please start the sandbox server first." + if not self._conn.poll(self._read_timeout): + raise TimeoutError("Pipe read timed out.") + try: + return self._conn.recv() + except EOFError as e: + # The child closed its end without sending a reply, i.e. its process stopped. + self._server_failure = self._describe_stopped_sandbox() + self._assert_no_server_failure() + raise RuntimeError(self._server_failure) from e # unreachable: the assert above raises def _describe_stopped_sandbox(self) -> str: """Say why the sandbox closed the connection, from how its process exited.""" return_code = None if self.process is not None: - try: - return_code = self.process.wait(timeout=STOPPED_PROCESS_WAIT) - except subprocess.TimeoutExpired: - pass + self.process.join(timeout=STOPPED_PROCESS_WAIT) + return_code = self.process.exitcode # The kernel's out-of-memory killer, and container memory limits, stop a process with SIGKILL. if sys.platform != "win32" and return_code == -signal.SIGKILL: return "The sandbox running your code was killed, probably because your code used too much memory." @@ -322,50 +312,42 @@ def _assert_no_server_failure(self) -> None: def _read_query_response(self) -> Any: """Read a response to a query, raising a clear error if the server failed and closed the connection.""" - data = json.loads(self._read_from_socket().decode()) + data = self._recv_message() if data.get("status") == ServerStatusCode.FAILURE: self._server_failure = f"The sandbox stopped unexpectedly, so later queries won't work: {data['message']}" self._assert_no_server_failure() + # A successful query or function call sends its value as a JSON string; decode it back to a + # structure so the response matches what callers (and from_json) expect. + if data.get("status") == QueryStatusCode.SUCCESS: + data["value"] = json.loads(data["value"]) return data def _start_subprocess(self) -> None: - """Spawn the runner subprocess and store it in ``self.process``.""" - - def try_drop_privileges() -> None: - if self.worker_username is not None: - drop_privileges(self.worker_username) - + """Spawn the runner child process and store it in ``self.process``.""" # With data files, the sandbox starts in a folder holding copies of them, so student code can # read them by relative path. Without them, the starting folder is left as it was. if self.data_files is not None: self._data_files_dir = _make_data_files_dir(self.data_files) - self.process = subprocess.Popen( - args=(sys.executable, SCRIPT_PATH), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=self._data_files_dir, - preexec_fn=None if sys.platform == "win32" else try_drop_privileges, + parent_conn, child_conn = _MP_CONTEXT.Pipe() + data_files_dir = None if self._data_files_dir is None else str(self._data_files_dir) + self.process = _MP_CONTEXT.Process( + target=_serve, + args=(child_conn, self.worker_username, data_files_dir), + daemon=True, ) + self.process.start() + # Close the child's end in the grader so that if the child process dies, the grader's recv + # sees EOF instead of blocking on a handle it still holds open itself. + child_conn.close() + self._conn = parent_conn self._assert_process_running() - def _connect_to_server(self, timeout: float) -> None: - """Read the host/port printed by the subprocess and connect the socket.""" - assert self.process is not None - assert self.process.stdout is not None - host, port = self.process.stdout.readline().decode().strip().split(",") - self.student_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.student_socket.settimeout(timeout) - self.student_socket.connect((host, int(port))) - def _read_start_response(self, initialization_timeout: float) -> ProcessStartResponse: """Read and return the ProcessStartResponse sent after the start message.""" - assert self.student_socket is not None - self.student_socket.settimeout(initialization_timeout + START_RESPONSE_GRACE) + self._read_timeout = initialization_timeout + START_RESPONSE_GRACE try: - data = self._read_from_socket().decode() - res: ProcessStartResponse = json.loads(data) + res: ProcessStartResponse = self._recv_message() if res.get("stdout"): self._accumulated_stdout.append(res["stdout"]) except Exception as e: @@ -383,8 +365,7 @@ def query_raw(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT self._assert_no_server_failure() self._assert_process_running() json_message = StudentQueryRequest(message_type=MessageType.QUERY, var=var_to_query, query_timeout=query_timeout) - assert self.student_socket is not None - self.student_socket.settimeout(query_timeout) + self._read_timeout = query_timeout self._send_json_object(json_message) try: data: StudentQueryResponse = self._read_query_response() @@ -424,9 +405,8 @@ def query_function_raw( kwargs_encoded=serialize_object_unsafe(kwargs), query_timeout=query_timeout, ) - assert self.student_socket is not None - self.student_socket.settimeout(query_timeout) - self.student_socket.sendall((json.dumps(json_message) + os.linesep).encode("utf-8")) + self._read_timeout = query_timeout + self._send_json_object(json_message) try: data: StudentFunctionResponse = self._read_query_response() except TimeoutError as error: @@ -468,12 +448,12 @@ def get_accumulated_stdout(self) -> str: return "".join(self._accumulated_stdout) def _cleanup(self) -> None: - if self.student_socket is not None: - self.student_socket.close() - self.student_socket = None + if self._conn is not None: + self._conn.close() + self._conn = None if self.process is not None: self.process.terminate() - self.process.wait() + self.process.join() self.process = None if self._data_files_dir is not None: _remove_data_files_dir(self._data_files_dir) @@ -560,9 +540,8 @@ def query_setup_raw(self, var_to_query: str) -> SetupQueryResponse: self._assert_no_server_failure() self._assert_process_running() json_message: SetupQueryRequest = {"message_type": MessageType.QUERY_SETUP, "var": var_to_query} - assert self.student_socket is not None, "Student socket is not connected. Please start the student code server first." self._send_json_object(json_message) - data: SetupQueryResponse = json.loads(self._read_from_socket().decode()) + data: SetupQueryResponse = self._recv_message() return data def query_setup(self, var_to_query: str) -> Any: diff --git a/tests/test_large_payload.py b/tests/test_large_payload.py new file mode 100644 index 0000000..6cfd412 --- /dev/null +++ b/tests/test_large_payload.py @@ -0,0 +1,46 @@ +import json +import textwrap +from typing import Any + +import pytest + +pytest_plugins = ("pytester",) + +# A list far larger than the OS pipe buffer (~64 KiB) once serialized, so a response and an argument +# each span many underlying reads/writes. The socket transport this replaced capped a request at +# 10 MB; the pipe has no such cap, so this also guards the multi-chunk framing in both directions. +STUDENT_CODE = """ +big_list = list(range(200_000)) + + +def sum_values(values): + return sum(values) +""" + +TEST_HEADER = """ +import pytest +""" + + +def run_grader(pytester: pytest.Pytester, tests: str) -> dict[str, dict[str, Any]]: + pytester.path.joinpath("test_grader.py").write_text(TEST_HEADER + textwrap.dedent(tests)) + pytester.mkdir("test_grader").joinpath("student_code.py").write_text(STUDENT_CODE) + pytester.runpytest("test_grader.py") + results = json.loads(pytester.path.joinpath("autograder_results.json").read_text()) + return {result["name"]: result for result in results["tests"]} + + +def test_large_value_and_argument_round_trip(pytester: pytest.Pytester) -> None: + result = run_grader( + pytester, + """ + @pytest.mark.grading_data(name="case", points=1) + def test_case(sandbox): + big = sandbox.query("big_list") + assert big == list(range(200_000)) + assert sandbox.query_function("sum_values", big) == sum(range(200_000)) + """, + )["case"] + + assert result["points_frac"] == 1 + assert result["outcome"] == "passed" diff --git a/tests/test_subtests.py b/tests/test_subtests.py index a6d4462..4f73fc0 100644 --- a/tests/test_subtests.py +++ b/tests/test_subtests.py @@ -229,25 +229,25 @@ def test_second(module_sandbox): def test_timeout_reported_by_student_process_stops_sandbox(pytester: pytest.Pytester) -> None: - # The socket waits a second longer than the call's timeout, so the student process's own + # The grader waits a second longer than the call's timeout, so the student process's own # timeout reply always arrives first. result = run_grader( pytester, """ - class SlowSocket: - def __init__(self, sock): - self.sock = sock + class SlowConnection: + def __init__(self, conn): + self.conn = conn - def settimeout(self, timeout): - self.sock.settimeout(timeout + 1) + def poll(self, timeout=None): + return self.conn.poll(timeout + 1 if timeout is not None else None) def __getattr__(self, name): - return getattr(self.sock, name) + return getattr(self.conn, name) @pytest.mark.grading_data(name="case", points=1) def test_case(sandbox, subtests, monkeypatch): - monkeypatch.setattr(sandbox, "student_socket", SlowSocket(sandbox.student_socket)) + monkeypatch.setattr(sandbox, "_conn", SlowConnection(sandbox._conn)) for x in [1, 4, 2, 5]: with subtests.test(): assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x, query_timeout=0.2)