diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 2672270..77b34ea 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -24,10 +24,13 @@ # ask Gemini how to do it from pytest_prairielearn_grader.json_utils import to_json from pytest_prairielearn_grader.utils import FunctionStatusCode +from pytest_prairielearn_grader.utils import MessageType from pytest_prairielearn_grader.utils import ProcessStartRequest from pytest_prairielearn_grader.utils import ProcessStartResponse from pytest_prairielearn_grader.utils import ProcessStatusCode from pytest_prairielearn_grader.utils import QueryStatusCode +from pytest_prairielearn_grader.utils import ServerFailureResponse +from pytest_prairielearn_grader.utils import ServerStatusCode from pytest_prairielearn_grader.utils import SetupQueryRequest from pytest_prairielearn_grader.utils import SetupQueryResponse from pytest_prairielearn_grader.utils import StudentFunctionRequest @@ -414,6 +417,11 @@ async def workspace_runner( return local_vars, student_code_vars, result_dict +def encode_server_failure(message: str) -> bytes: + response: ServerFailureResponse = {"status": ServerStatusCode.FAILURE, "message": message} + return (json.dumps(response) + os.linesep).encode() + + async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: """ Reads lines from stdin asynchronously and responds on stdout. @@ -452,7 +460,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit json_message = json.loads(line) msg_type = json_message.get("message_type") - if msg_type == "start_workspace": + if msg_type == MessageType.START_WORKSPACE: ws_json_message: WorkspaceStartRequest = json_message workspace_mode = True @@ -470,7 +478,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit writer.write((json.dumps(start_response) + os.linesep).encode()) - elif msg_type == "start": + elif msg_type == MessageType.START: start_json_message: ProcessStartRequest = json_message # Execute the student code for the first time and load # variables into the student_code_vars dictionary @@ -500,7 +508,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit writer.write((json.dumps(start_response) + os.linesep).encode()) - elif msg_type == "query_setup": + elif msg_type == MessageType.QUERY_SETUP: assert local_vars is not None query_setup_json_message: SetupQueryRequest = json_message @@ -515,7 +523,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit writer.write((json.dumps(setup_query_response) + os.linesep).encode()) - elif msg_type == "query": + elif msg_type == MessageType.QUERY: assert student_code_vars is not None query_json_message: StudentQueryRequest = json_message @@ -547,7 +555,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit writer.write((encoded_query_response + os.linesep).encode()) - elif msg_type == "query_function": + elif msg_type == MessageType.QUERY_FUNCTION: assert student_code_vars is not None query_function_json_message: StudentFunctionRequest = json_message @@ -575,11 +583,11 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit await writer.drain() # Ensure the response is written to stdout except asyncio.CancelledError: - writer.write((json.dumps({"status": "failure", "message": "Server was cancelled."}) + os.linesep).encode()) + writer.write(encode_server_failure("Server was cancelled.")) except asyncio.TimeoutError: - writer.write((json.dumps({"status": "failure", "message": "Student code timed out."}) + os.linesep).encode()) + writer.write(encode_server_failure("Student code timed out.")) except Exception as e: - writer.write((json.dumps({"status": "failure", "message": f"An error occurred: {e}"}) + os.linesep).encode()) + writer.write(encode_server_failure(f"An error occurred: {e}")) finally: # It's good practice to close transports and writers # print("Closing server connections...") diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 6c3e1ee..3ea59a3 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -11,10 +11,12 @@ from .json_utils import from_json from .utils import FunctionStatusCode +from .utils import MessageType from .utils import ProcessStartRequest from .utils import ProcessStartResponse from .utils import ProcessStatusCode from .utils import QueryStatusCode +from .utils import ServerStatusCode from .utils import SetupQueryRequest from .utils import SetupQueryResponse from .utils import StudentFunctionRequest @@ -138,6 +140,7 @@ class _SandboxBase: names_for_user_list: list[str] | None worker_username: str | None _accumulated_stdout: list[str] + _server_failure: str | None def __init__( self, @@ -157,6 +160,7 @@ def __init__( self.process = None self.student_socket = None self._accumulated_stdout = [] + self._server_failure = None def _assert_process_running(self) -> None: assert self.process is not None, "Sandbox process is not running. Please start it first." @@ -190,6 +194,18 @@ def _read_from_socket(self) -> bytes: loc = len(buffer) - len(chunk) + idx + len(terminator) return buffer[:loc] + def _assert_no_server_failure(self) -> None: + if self._server_failure is not None: + raise RuntimeError(self._server_failure) + + def _read_query_response(self) -> Any: + """Read a response to a query, raising a clear error if the server failed and closed the connection.""" + data = json.loads(self._read_from_socket().decode()) + if data.get("status") == ServerStatusCode.FAILURE: + self._server_failure = f"The sandbox stopped unexpectedly, so later queries won't work: {data['message']}" + self._assert_no_server_failure() + return data + def _start_subprocess(self) -> None: """Spawn the runner subprocess and store it in ``self.process``.""" @@ -233,12 +249,13 @@ def _read_start_response(self) -> ProcessStartResponse: return res def query_raw(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> StudentQueryResponse: + self._assert_no_server_failure() self._assert_process_running() - json_message = StudentQueryRequest(message_type="query", var=var_to_query, query_timeout=query_timeout) + json_message = StudentQueryRequest(message_type=MessageType.QUERY, var=var_to_query, query_timeout=query_timeout) assert self.student_socket is not None self.student_socket.settimeout(query_timeout) self._send_json_object(json_message) - data: StudentQueryResponse = json.loads(self._read_from_socket().decode()) + data: StudentQueryResponse = self._read_query_response() return data def query(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> Any: @@ -260,9 +277,13 @@ def query_function_raw( than raised. Callers must stop issuing requests on this sandbox after a timeout because the student process may still write the timed-out request's response. + + Raises a RuntimeError if the sandbox server failed, for example because it + couldn't load the arguments. The sandbox can't be used after that. """ + self._assert_no_server_failure() json_message = StudentFunctionRequest( - message_type="query_function", + message_type=MessageType.QUERY_FUNCTION, function_name=function_name, args_encoded=serialize_object_unsafe(args), kwargs_encoded=serialize_object_unsafe(kwargs), @@ -272,7 +293,7 @@ def query_function_raw( self.student_socket.settimeout(query_timeout) self.student_socket.sendall((json.dumps(json_message) + os.linesep).encode("utf-8")) try: - data: StudentFunctionResponse = json.loads(self._read_from_socket().decode()) + data: StudentFunctionResponse = self._read_query_response() except TimeoutError as error: data = { "status": FunctionStatusCode.TIMEOUT, @@ -371,7 +392,7 @@ def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_T # TODO make this a shared type json_message = ProcessStartRequest( - message_type="start", + message_type=MessageType.START, student_code=student_code, student_file_name=str(self.student_code_file), setup_code=setup_code, @@ -387,7 +408,7 @@ def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_T def query_setup_raw(self, var_to_query: str) -> SetupQueryResponse: self._assert_process_running() - json_message: SetupQueryRequest = {"message_type": "query_setup", "var": var_to_query} + json_message: SetupQueryRequest = {"message_type": MessageType.QUERY_SETUP, "var": var_to_query} assert self.student_socket is not None, "Student socket is not connected. Please start the student code server first." self._send_json_object(json_message) data: SetupQueryResponse = json.loads(self._read_from_socket().decode()) @@ -458,7 +479,7 @@ def start_workspace_server(self, *, initialization_timeout: float = DEFAULT_TIME self._connect_to_server(initialization_timeout) json_message = WorkspaceStartRequest( - message_type="start_workspace", + message_type=MessageType.START_WORKSPACE, workspace_dir=str(self.workspace_dir), exec_entry=self.exec_entry, setup_code=setup_code, diff --git a/src/pytest_prairielearn_grader/utils.py b/src/pytest_prairielearn_grader/utils.py index cf0e16c..32c8acb 100644 --- a/src/pytest_prairielearn_grader/utils.py +++ b/src/pytest_prairielearn_grader/utils.py @@ -58,11 +58,37 @@ class ProcessStatusCode(StrEnum): NO_RESPONSE = "no_response" +class ServerStatusCode(StrEnum): + """ + Status codes for an error in the sandbox server itself, not in a specific request. + The server closes the connection after sending one of these. + """ + + FAILURE = "failure" + + +class MessageType(StrEnum): + """ + Types of the messages the grader sends to the sandbox server. + """ + + START = "start" + START_WORKSPACE = "start_workspace" + QUERY = "query" + QUERY_SETUP = "query_setup" + QUERY_FUNCTION = "query_function" + + +class ServerFailureResponse(TypedDict): + status: Literal[ServerStatusCode.FAILURE] + message: str + + # TODO use some inheritance on the query and response types class SetupQueryRequest(TypedDict): - message_type: Literal["query_setup"] + message_type: Literal[MessageType.QUERY_SETUP] var: str @@ -73,7 +99,7 @@ class SetupQueryResponse(TypedDict): # Variable query dict types class StudentQueryRequest(TypedDict): - message_type: Literal["query"] + message_type: Literal[MessageType.QUERY] var: str query_timeout: float @@ -87,7 +113,7 @@ class StudentQueryResponse(TypedDict): # Function query dict types class StudentFunctionRequest(TypedDict): - message_type: Literal["query_function"] + message_type: Literal[MessageType.QUERY_FUNCTION] function_name: str args_encoded: str # TODO add a stronger type for the input/output of the serialized function kwargs_encoded: str @@ -109,7 +135,7 @@ class StudentFunctionResponse(TypedDict): class ProcessStartRequest(TypedDict): - message_type: Literal["start"] + message_type: Literal[MessageType.START] student_code: str student_file_name: str setup_code: str | None @@ -122,7 +148,7 @@ class ProcessStartRequest(TypedDict): class WorkspaceStartRequest(TypedDict): - message_type: Literal["start_workspace"] + message_type: Literal[MessageType.START_WORKSPACE] workspace_dir: str exec_entry: str | None setup_code: str | None diff --git a/tests/scenario_root/test_server_failure/__init__.py b/tests/scenario_root/test_server_failure/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_server_failure/expected_outcome.json b/tests/scenario_root/test_server_failure/expected_outcome.json new file mode 100644 index 0000000..d737d69 --- /dev/null +++ b/tests/scenario_root/test_server_failure/expected_outcome.json @@ -0,0 +1,32 @@ +{ + "expected_data_object": { + "score": 0.6666666666666666, + "tests": [ + { + "max_points": 1, + "name": "clear_error", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_server_failure.py::test_failure_raises_clear_error[student_code]" + }, + { + "max_points": 1, + "name": "later_queries", + "points": 1.0, + "points_frac": 1.0, + "outcome": "passed", + "test_id": "test_server_failure.py::test_later_queries_raise_same_error[student_code]" + }, + { + "max_points": 1, + "name": "uncaught", + "points": 0.0, + "points_frac": 0.0, + "outcome": "failed", + "test_id": "test_server_failure.py::test_uncaught_failure_shows_message[student_code]", + "message": "The sandbox stopped unexpectedly, so later queries won't work: An error occurred: " + } + ] + } +} diff --git a/tests/scenario_root/test_server_failure/scenario.py b/tests/scenario_root/test_server_failure/scenario.py new file mode 100644 index 0000000..6dbd7ec --- /dev/null +++ b/tests/scenario_root/test_server_failure/scenario.py @@ -0,0 +1,33 @@ +import pytest + +from pytest_prairielearn_grader.fixture import StudentFixture + +EXPECTED_MESSAGE = "The sandbox stopped unexpectedly, so later queries won't work: An error occurred: " + + +class OnlyInTestFile: + """The sandbox can't import this file, so it can't load an instance as an argument.""" + + +@pytest.mark.grading_data(name="clear_error", points=1) +def test_failure_raises_clear_error(sandbox: StudentFixture) -> None: + assert sandbox.query("y") == 5 + + with pytest.raises(RuntimeError, match=EXPECTED_MESSAGE): + sandbox.query_function_raw("identity", OnlyInTestFile()) + + +@pytest.mark.grading_data(name="later_queries", points=1) +def test_later_queries_raise_same_error(sandbox: StudentFixture) -> None: + with pytest.raises(RuntimeError, match=EXPECTED_MESSAGE): + sandbox.query_function("identity", OnlyInTestFile()) + + with pytest.raises(RuntimeError, match=EXPECTED_MESSAGE): + sandbox.query("y") + with pytest.raises(RuntimeError, match=EXPECTED_MESSAGE): + sandbox.query_function("identity", 1) + + +@pytest.mark.grading_data(name="uncaught", points=1) +def test_uncaught_failure_shows_message(sandbox: StudentFixture) -> None: + sandbox.query_function("identity", OnlyInTestFile()) diff --git a/tests/scenario_root/test_server_failure/student_code.py b/tests/scenario_root/test_server_failure/student_code.py new file mode 100644 index 0000000..112ec2e --- /dev/null +++ b/tests/scenario_root/test_server_failure/student_code.py @@ -0,0 +1,5 @@ +y = 5 + + +def identity(x): + return x