Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 16 additions & 8 deletions src/pytest_prairielearn_grader/_student_code_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,13 @@
# ask Gemini how to do it
from pytest_prairielearn_grader.json_utils import to_json
from pytest_prairielearn_grader.utils import FunctionStatusCode
from pytest_prairielearn_grader.utils import MessageType
from pytest_prairielearn_grader.utils import ProcessStartRequest
from pytest_prairielearn_grader.utils import ProcessStartResponse
from pytest_prairielearn_grader.utils import ProcessStatusCode
from pytest_prairielearn_grader.utils import QueryStatusCode
from pytest_prairielearn_grader.utils import ServerFailureResponse
from pytest_prairielearn_grader.utils import ServerStatusCode
from pytest_prairielearn_grader.utils import SetupQueryRequest
from pytest_prairielearn_grader.utils import SetupQueryResponse
from pytest_prairielearn_grader.utils import StudentFunctionRequest
Expand Down Expand Up @@ -414,6 +417,11 @@ async def workspace_runner(
return local_vars, student_code_vars, result_dict


def encode_server_failure(message: str) -> bytes:
response: ServerFailureResponse = {"status": ServerStatusCode.FAILURE, "message": message}
return (json.dumps(response) + os.linesep).encode()


async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
"""
Reads lines from stdin asynchronously and responds on stdout.
Expand Down Expand Up @@ -452,7 +460,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
json_message = json.loads(line)

msg_type = json_message.get("message_type")
if msg_type == "start_workspace":
if msg_type == MessageType.START_WORKSPACE:
ws_json_message: WorkspaceStartRequest = json_message
workspace_mode = True

Expand All @@ -470,7 +478,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit

writer.write((json.dumps(start_response) + os.linesep).encode())

elif msg_type == "start":
elif msg_type == MessageType.START:
start_json_message: ProcessStartRequest = json_message
# Execute the student code for the first time and load
# variables into the student_code_vars dictionary
Expand Down Expand Up @@ -500,7 +508,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit

writer.write((json.dumps(start_response) + os.linesep).encode())

elif msg_type == "query_setup":
elif msg_type == MessageType.QUERY_SETUP:
assert local_vars is not None
query_setup_json_message: SetupQueryRequest = json_message

Expand All @@ -515,7 +523,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit

writer.write((json.dumps(setup_query_response) + os.linesep).encode())

elif msg_type == "query":
elif msg_type == MessageType.QUERY:
assert student_code_vars is not None
query_json_message: StudentQueryRequest = json_message

Expand Down Expand Up @@ -547,7 +555,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit

writer.write((encoded_query_response + os.linesep).encode())

elif msg_type == "query_function":
elif msg_type == MessageType.QUERY_FUNCTION:
assert student_code_vars is not None
query_function_json_message: StudentFunctionRequest = json_message

Expand Down Expand Up @@ -575,11 +583,11 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
await writer.drain() # Ensure the response is written to stdout

except asyncio.CancelledError:
writer.write((json.dumps({"status": "failure", "message": "Server was cancelled."}) + os.linesep).encode())
writer.write(encode_server_failure("Server was cancelled."))
except asyncio.TimeoutError:
writer.write((json.dumps({"status": "failure", "message": "Student code timed out."}) + os.linesep).encode())
writer.write(encode_server_failure("Student code timed out."))
except Exception as e:
writer.write((json.dumps({"status": "failure", "message": f"An error occurred: {e}"}) + os.linesep).encode())
writer.write(encode_server_failure(f"An error occurred: {e}"))
finally:
# It's good practice to close transports and writers
# print("Closing server connections...")
Expand Down
35 changes: 28 additions & 7 deletions src/pytest_prairielearn_grader/fixture.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,12 @@

from .json_utils import from_json
from .utils import FunctionStatusCode
from .utils import MessageType
from .utils import ProcessStartRequest
from .utils import ProcessStartResponse
from .utils import ProcessStatusCode
from .utils import QueryStatusCode
from .utils import ServerStatusCode
from .utils import SetupQueryRequest
from .utils import SetupQueryResponse
from .utils import StudentFunctionRequest
Expand Down Expand Up @@ -138,6 +140,7 @@ class _SandboxBase:
names_for_user_list: list[str] | None
worker_username: str | None
_accumulated_stdout: list[str]
_server_failure: str | None

def __init__(
self,
Expand All @@ -157,6 +160,7 @@ def __init__(
self.process = None
self.student_socket = None
self._accumulated_stdout = []
self._server_failure = None

def _assert_process_running(self) -> None:
assert self.process is not None, "Sandbox process is not running. Please start it first."
Expand Down Expand Up @@ -190,6 +194,18 @@ def _read_from_socket(self) -> bytes:
loc = len(buffer) - len(chunk) + idx + len(terminator)
return buffer[:loc]

def _assert_no_server_failure(self) -> None:
if self._server_failure is not None:
raise RuntimeError(self._server_failure)

def _read_query_response(self) -> Any:
"""Read a response to a query, raising a clear error if the server failed and closed the connection."""
data = json.loads(self._read_from_socket().decode())
if data.get("status") == ServerStatusCode.FAILURE:
self._server_failure = f"The sandbox stopped unexpectedly, so later queries won't work: {data['message']}"
self._assert_no_server_failure()
return data

def _start_subprocess(self) -> None:
"""Spawn the runner subprocess and store it in ``self.process``."""

Expand Down Expand Up @@ -233,12 +249,13 @@ def _read_start_response(self) -> ProcessStartResponse:
return res

def query_raw(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> StudentQueryResponse:
self._assert_no_server_failure()
self._assert_process_running()
json_message = StudentQueryRequest(message_type="query", var=var_to_query, query_timeout=query_timeout)
json_message = StudentQueryRequest(message_type=MessageType.QUERY, var=var_to_query, query_timeout=query_timeout)
assert self.student_socket is not None
self.student_socket.settimeout(query_timeout)
self._send_json_object(json_message)
data: StudentQueryResponse = json.loads(self._read_from_socket().decode())
data: StudentQueryResponse = self._read_query_response()
return data

def query(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> Any:
Expand All @@ -260,9 +277,13 @@ def query_function_raw(
than raised. Callers must stop issuing requests on this sandbox after a
timeout because the student process may still write the timed-out
request's response.

Raises a RuntimeError if the sandbox server failed, for example because it
couldn't load the arguments. The sandbox can't be used after that.
"""
self._assert_no_server_failure()
json_message = StudentFunctionRequest(
message_type="query_function",
message_type=MessageType.QUERY_FUNCTION,
function_name=function_name,
args_encoded=serialize_object_unsafe(args),
kwargs_encoded=serialize_object_unsafe(kwargs),
Expand All @@ -272,7 +293,7 @@ def query_function_raw(
self.student_socket.settimeout(query_timeout)
self.student_socket.sendall((json.dumps(json_message) + os.linesep).encode("utf-8"))
try:
data: StudentFunctionResponse = json.loads(self._read_from_socket().decode())
data: StudentFunctionResponse = self._read_query_response()
except TimeoutError as error:
data = {
"status": FunctionStatusCode.TIMEOUT,
Expand Down Expand Up @@ -371,7 +392,7 @@ def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_T

# TODO make this a shared type
json_message = ProcessStartRequest(
message_type="start",
message_type=MessageType.START,
student_code=student_code,
student_file_name=str(self.student_code_file),
setup_code=setup_code,
Expand All @@ -387,7 +408,7 @@ def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_T

def query_setup_raw(self, var_to_query: str) -> SetupQueryResponse:
self._assert_process_running()
json_message: SetupQueryRequest = {"message_type": "query_setup", "var": var_to_query}
json_message: SetupQueryRequest = {"message_type": MessageType.QUERY_SETUP, "var": var_to_query}
assert self.student_socket is not None, "Student socket is not connected. Please start the student code server first."
self._send_json_object(json_message)
data: SetupQueryResponse = json.loads(self._read_from_socket().decode())
Expand Down Expand Up @@ -458,7 +479,7 @@ def start_workspace_server(self, *, initialization_timeout: float = DEFAULT_TIME
self._connect_to_server(initialization_timeout)

json_message = WorkspaceStartRequest(
message_type="start_workspace",
message_type=MessageType.START_WORKSPACE,
workspace_dir=str(self.workspace_dir),
exec_entry=self.exec_entry,
setup_code=setup_code,
Expand Down
36 changes: 31 additions & 5 deletions src/pytest_prairielearn_grader/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,37 @@ class ProcessStatusCode(StrEnum):
NO_RESPONSE = "no_response"


class ServerStatusCode(StrEnum):
"""
Status codes for an error in the sandbox server itself, not in a specific request.
The server closes the connection after sending one of these.
"""

FAILURE = "failure"


class MessageType(StrEnum):
"""
Types of the messages the grader sends to the sandbox server.
"""

START = "start"
START_WORKSPACE = "start_workspace"
QUERY = "query"
QUERY_SETUP = "query_setup"
QUERY_FUNCTION = "query_function"


class ServerFailureResponse(TypedDict):
status: Literal[ServerStatusCode.FAILURE]
message: str


# TODO use some inheritance on the query and response types


class SetupQueryRequest(TypedDict):
message_type: Literal["query_setup"]
message_type: Literal[MessageType.QUERY_SETUP]
var: str


Expand All @@ -73,7 +99,7 @@ class SetupQueryResponse(TypedDict):

# Variable query dict types
class StudentQueryRequest(TypedDict):
message_type: Literal["query"]
message_type: Literal[MessageType.QUERY]
var: str
query_timeout: float

Expand All @@ -87,7 +113,7 @@ class StudentQueryResponse(TypedDict):

# Function query dict types
class StudentFunctionRequest(TypedDict):
message_type: Literal["query_function"]
message_type: Literal[MessageType.QUERY_FUNCTION]
function_name: str
args_encoded: str # TODO add a stronger type for the input/output of the serialized function
kwargs_encoded: str
Expand All @@ -109,7 +135,7 @@ class StudentFunctionResponse(TypedDict):


class ProcessStartRequest(TypedDict):
message_type: Literal["start"]
message_type: Literal[MessageType.START]
student_code: str
student_file_name: str
setup_code: str | None
Expand All @@ -122,7 +148,7 @@ class ProcessStartRequest(TypedDict):


class WorkspaceStartRequest(TypedDict):
message_type: Literal["start_workspace"]
message_type: Literal[MessageType.START_WORKSPACE]
workspace_dir: str
exec_entry: str | None
setup_code: str | None
Expand Down
Empty file.
32 changes: 32 additions & 0 deletions tests/scenario_root/test_server_failure/expected_outcome.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
{
"expected_data_object": {
"score": 0.6666666666666666,
"tests": [
{
"max_points": 1,
"name": "clear_error",
"points": 1.0,
"points_frac": 1.0,
"outcome": "passed",
"test_id": "test_server_failure.py::test_failure_raises_clear_error[student_code]"
},
{
"max_points": 1,
"name": "later_queries",
"points": 1.0,
"points_frac": 1.0,
"outcome": "passed",
"test_id": "test_server_failure.py::test_later_queries_raise_same_error[student_code]"
},
{
"max_points": 1,
"name": "uncaught",
"points": 0.0,
"points_frac": 0.0,
"outcome": "failed",
"test_id": "test_server_failure.py::test_uncaught_failure_shows_message[student_code]",
"message": "The sandbox stopped unexpectedly, so later queries won't work: An error occurred: "
}
]
}
}
33 changes: 33 additions & 0 deletions tests/scenario_root/test_server_failure/scenario.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import pytest

from pytest_prairielearn_grader.fixture import StudentFixture

EXPECTED_MESSAGE = "The sandbox stopped unexpectedly, so later queries won't work: An error occurred: "


class OnlyInTestFile:
"""The sandbox can't import this file, so it can't load an instance as an argument."""


@pytest.mark.grading_data(name="clear_error", points=1)
def test_failure_raises_clear_error(sandbox: StudentFixture) -> None:
assert sandbox.query("y") == 5

with pytest.raises(RuntimeError, match=EXPECTED_MESSAGE):
sandbox.query_function_raw("identity", OnlyInTestFile())


@pytest.mark.grading_data(name="later_queries", points=1)
def test_later_queries_raise_same_error(sandbox: StudentFixture) -> None:
with pytest.raises(RuntimeError, match=EXPECTED_MESSAGE):
sandbox.query_function("identity", OnlyInTestFile())

with pytest.raises(RuntimeError, match=EXPECTED_MESSAGE):
sandbox.query("y")
with pytest.raises(RuntimeError, match=EXPECTED_MESSAGE):
sandbox.query_function("identity", 1)


@pytest.mark.grading_data(name="uncaught", points=1)
def test_uncaught_failure_shows_message(sandbox: StudentFixture) -> None:
sandbox.query_function("identity", OnlyInTestFile())
5 changes: 5 additions & 0 deletions tests/scenario_root/test_server_failure/student_code.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
y = 5


def identity(x):
return x
Loading