Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion quick_start.md
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ value = sandbox.query("variable_name")
```

Retrieves the value of a variable defined in the student code or `setup_code.py`.
Raises a `RuntimeError` if the variable doesn't exist.
Raises a `NameError` if the variable doesn't exist, and a `RuntimeError` with a clear message if its value can't be sent back to the grader. Later queries on the sandbox still work after either error.

**Example:**

Expand Down
23 changes: 16 additions & 7 deletions src/pytest_prairielearn_grader/_student_code_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -529,14 +529,23 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
else:
raise KeyError(var_to_query)

query_response: StudentQueryResponse = {
"status": QueryStatusCode.SUCCESS,
"value": to_json(value),
}
except (KeyError, AttributeError, ImportError):
query_response = {"status": QueryStatusCode.NOT_FOUND, "value": ""}

writer.write((json.dumps(query_response) + os.linesep).encode())
not_found_response: StudentQueryResponse = {"status": QueryStatusCode.NOT_FOUND, "value": "", "error": None}
encoded_query_response = json.dumps(not_found_response)
else:
try:
query_response: StudentQueryResponse = {"status": QueryStatusCode.SUCCESS, "value": to_json(value), "error": None}
encoded_query_response = json.dumps(query_response)
except Exception as e:
not_serializable_response: StudentQueryResponse = {
"status": QueryStatusCode.NOT_SERIALIZABLE,
"value": None,
"error": f"Variable '{var_to_query}' has a value of type {type(value).__name__}, "
f"which can't be sent back to the grader: {e}",
}
encoded_query_response = json.dumps(not_serializable_response)

writer.write((encoded_query_response + os.linesep).encode())

elif msg_type == "query_function":
assert student_code_vars is not None
Expand Down
8 changes: 6 additions & 2 deletions src/pytest_prairielearn_grader/fixture.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from .utils import ProcessStartRequest
from .utils import ProcessStartResponse
from .utils import ProcessStatusCode
from .utils import QueryStatusCode
from .utils import SetupQueryRequest
from .utils import SetupQueryResponse
from .utils import StudentFunctionRequest
Expand Down Expand Up @@ -243,8 +244,11 @@ def query_raw(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT
def query(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> Any:
"""Query a variable from the student sandbox by name."""
response = self.query_raw(var_to_query, query_timeout=query_timeout)
if response["status"] == "not_found":
raise NameError(f"Query for '{var_to_query}' failed")
match response["status"]:
case QueryStatusCode.NOT_FOUND:
raise NameError(f"Query for '{var_to_query}' failed")
case QueryStatusCode.NOT_SERIALIZABLE:
raise RuntimeError(response["error"])
return from_json(response["value"])

def query_function_raw(
Expand Down
2 changes: 2 additions & 0 deletions src/pytest_prairielearn_grader/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ class QueryStatusCode(StrEnum):

SUCCESS = "success"
NOT_FOUND = "not_found"
NOT_SERIALIZABLE = "not_serializable"


class FunctionStatusCode(StrEnum):
Expand Down Expand Up @@ -81,6 +82,7 @@ class StudentQueryResponse(TypedDict):
# This is meant to be deserialized into a Python object
status: QueryStatusCode
value: Any
error: str | None


# Function query dict types
Expand Down
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
{
"expected_data_object": {
"score": 0.875,
"tests": [
{
"max_points": 1,
"name": "raw_response",
"points": 1.0,
"points_frac": 1.0,
"outcome": "passed",
"test_id": "test_query_not_serializable.py::test_raw_response_has_all_fields[student_code]"
},
{
"max_points": 1,
"name": "success_response",
"points": 1.0,
"points_frac": 1.0,
"outcome": "passed",
"test_id": "test_query_not_serializable.py::test_success_response_has_no_error[student_code]"
},
{
"max_points": 1,
"name": "other_values",
"points": 1.0,
"points_frac": 1.0,
"outcome": "passed",
"test_id": "test_query_not_serializable.py::test_other_unserializable_values[student_code-nested-list]"
},
{
"max_points": 1,
"name": "other_values",
"points": 1.0,
"points_frac": 1.0,
"outcome": "passed",
"test_id": "test_query_not_serializable.py::test_other_unserializable_values[student_code-self_reference-list]"
},
{
"max_points": 1,
"name": "query",
"points": 1.0,
"points_frac": 1.0,
"outcome": "passed",
"test_id": "test_query_not_serializable.py::test_query_has_clear_error[student_code]"
},
{
"max_points": 1,
"name": "sandbox_still_works",
"points": 1.0,
"points_frac": 1.0,
"outcome": "passed",
"test_id": "test_query_not_serializable.py::test_later_queries_still_work[student_code]"
},
{
"max_points": 1,
"name": "not_found",
"points": 1.0,
"points_frac": 1.0,
"outcome": "passed",
"test_id": "test_query_not_serializable.py::test_missing_variable[student_code]"
},
{
"max_points": 1,
"name": "assert_equal",
"points": 0.0,
"points_frac": 0.0,
"outcome": "failed",
"test_id": "test_query_not_serializable.py::test_assert_equal_shows_clear_error[student_code]",
"message": "Variable 'x' has a value of type Thing, which can't be sent back to the grader"
}
]
}
}
58 changes: 58 additions & 0 deletions tests/scenario_root/test_query_not_serializable/scenario.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import pytest

from pytest_prairielearn_grader.assertions import assert_equal
from pytest_prairielearn_grader.fixture import StudentFixture
from pytest_prairielearn_grader.utils import QueryStatusCode

EXPECTED_MESSAGE = "Variable 'x' has a value of type Thing, which can't be sent back to the grader"


@pytest.mark.grading_data(name="raw_response", points=1)
def test_raw_response_has_all_fields(sandbox: StudentFixture) -> None:
response = sandbox.query_raw("x")

assert response["status"] == QueryStatusCode.NOT_SERIALIZABLE
assert response["value"] is None
assert response["error"] is not None
assert EXPECTED_MESSAGE in response["error"]


@pytest.mark.grading_data(name="success_response", points=1)
def test_success_response_has_no_error(sandbox: StudentFixture) -> None:
assert sandbox.query_raw("y") == {"status": QueryStatusCode.SUCCESS, "value": 5, "error": None}


@pytest.mark.grading_data(name="other_values", points=1)
@pytest.mark.parametrize(("name", "type_name"), [("nested", "list"), ("self_reference", "list")])
def test_other_unserializable_values(sandbox: StudentFixture, name: str, type_name: str) -> None:
with pytest.raises(RuntimeError, match=f"Variable '{name}' has a value of type {type_name}, which can't be sent back to the grader"):
sandbox.query(name)


@pytest.mark.grading_data(name="query", points=1)
def test_query_has_clear_error(sandbox: StudentFixture) -> None:
with pytest.raises(RuntimeError, match=EXPECTED_MESSAGE):
sandbox.query("x")


@pytest.mark.grading_data(name="sandbox_still_works", points=1)
def test_later_queries_still_work(sandbox: StudentFixture) -> None:
with pytest.raises(RuntimeError, match=EXPECTED_MESSAGE):
sandbox.query("x")

assert sandbox.query("y") == 5
assert sandbox.query_raw("x")["status"] == QueryStatusCode.NOT_SERIALIZABLE
assert sandbox.query_raw("y")["value"] == 5


@pytest.mark.grading_data(name="not_found", points=1)
def test_missing_variable(sandbox: StudentFixture) -> None:
assert sandbox.query_raw("missing")["status"] == QueryStatusCode.NOT_FOUND
with pytest.raises(NameError, match="Query for 'missing' failed"):
sandbox.query("missing")


@pytest.mark.output(level="friendly")
@pytest.mark.grading_data(name="assert_equal", points=1)
def test_assert_equal_shows_clear_error(sandbox: StudentFixture) -> None:
assert_equal(sandbox.query("x"), 25, description="x")
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
class Thing:
pass


x = Thing()
y = 5
nested = [1, {"thing": Thing()}]
self_reference = [1]
self_reference.append(self_reference)
Loading