From 8fbca47d264b41b2e04a55d01b3645150fcf13f4 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Tue, 17 Nov 2026 11:25:50 -0500 Subject: [PATCH 1/3] Make friendly the default output level Co-Authored-By: Claude Opus 5.5 --- README.md | 11 +++--- converting.md | 7 ++-- quick_start.md | 36 +++++++++---------- src/pytest_prairielearn_grader/config.py | 10 +++--- src/pytest_prairielearn_grader/utils.py | 2 +- .../test_fixture_exception/scenario.py | 2 +- .../expected_outcome.json | 12 +++---- .../test_global_output_level/scenario.py | 13 ++++--- .../expected_outcome.json | 2 +- .../expected_outcome.json | 6 ++-- .../test_quiz_2_1/expected_outcome.json | 4 +-- .../expected_outcome.json | 2 +- .../test_stdout_feature/expected_outcome.json | 2 +- .../expected_outcome.json | 2 +- .../expected_outcome.json | 2 +- 15 files changed, 54 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index bb925cb..8da264e 100644 --- a/README.md +++ b/README.md @@ -34,28 +34,28 @@ Student code runs in a separate subprocess via Unix sockets, providing: ### Student-Friendly Feedback Mode -Control how much information students see when their code fails. Use `@pytest.mark.output(level=...)` per test, or set `output_level` globally in your `ConfigObject`: +Control how much information students see when their code fails. The default is `"friendly"`. Use `@pytest.mark.output(level=...)` per test, or set `output_level` globally in your `ConfigObject`: ```python from pytest_prairielearn_grader import ConfigObject autograder_config = ConfigObject( - output_level="friendly", # Global default for all tests + output_level="traceback", # Global default for all tests ) ``` **Output levels:** | Level | Shows | |-------|-------| +| `"friendly"` | Only the exception message, with no class name *(default)*. If a call to a student function raised the error, this includes the traceback from the student's code. | | `"none"` | Exception class name only (e.g., `AssertionError`) | -| `"message"` | Exception name + first line of message *(default)* | +| `"message"` | Exception name + first line of message | | `"traceback"` | Full exception with traceback | -| `"friendly"` | Only the exception message text — no class name, no traceback | Per-test markers override the global setting: ```python -@pytest.mark.output(level="friendly") +@pytest.mark.output(level="traceback") @pytest.mark.grading_data(name="Test add", points=5) def test_add(sandbox: StudentFixture) -> None: assert_fn_equal(sandbox, "add", args=(2, 3), expected=5) @@ -68,7 +68,6 @@ Student-friendly assertion functions that produce clean, readable failure messag ```python from pytest_prairielearn_grader.assertions import assert_equal, assert_fn_equal -@pytest.mark.output(level="friendly") @pytest.mark.grading_data(name="Test calculation", points=5) def test_calc(sandbox: StudentFixture) -> None: # Produces: "Checking: add(2, 3)\nExpected output: 5\nYour code output: 4\n..." diff --git a/converting.md b/converting.md index 090e0d3..ccdfa81 100644 --- a/converting.md +++ b/converting.md @@ -162,8 +162,6 @@ import pytest from pytest_prairielearn_grader.assertions import assert_approx_equal from pytest_prairielearn_grader.fixture import StudentFixture -pytestmark = pytest.mark.output(level="friendly") - @pytest.mark.grading_data(name="x_sq", points=10) def test_x_sq(sandbox: StudentFixture, data_json: dict) -> None: @@ -234,9 +232,8 @@ The `plots` example question is a converted plot question. ### Feedback messages -Use `@pytest.mark.output(level=GradingOutputLevel.Friendly)` on each test, as the example questions -do, or `output_level="friendly"` in the `ConfigObject`. At this level the student sees only the -assertion's message. +The default `friendly` [output level](quick_start.md#output-levels) shows the student only the +assertion's message, so tests don't need `@pytest.mark.output`. Follow [Hiding Expected Values](quick_start.md#hiding-expected-values): don't use a bare `assert` or numpy testing functions, and use `assert_true` for hidden test cases. The comparison helpers show diff --git a/quick_start.md b/quick_start.md index c3df74a..246543d 100644 --- a/quick_start.md +++ b/quick_start.md @@ -110,12 +110,9 @@ checks the result: ```python import pytest -from pytest_prairielearn_grader import ConfigObject from pytest_prairielearn_grader.assertions import assert_fn_approx_equal from pytest_prairielearn_grader.fixture import StudentFixture -autograder_config = ConfigObject(output_level="friendly") - @pytest.mark.grading_data(name="Average of one number", points=1) def test_single(sandbox: StudentFixture) -> None: @@ -134,7 +131,8 @@ def test_several(sandbox: StudentFixture, values: list[float]) -> None: tolerance. On a failure, the student sees which call was checked, the expected value and their value. See [Checking Answers](#checking-answers) for the other helpers and for checks that hide the expected value. -- The `friendly` output level shows students only the assertion's message, without a traceback. +- Students see only the assertion's message, because `friendly` is the default + [output level](#output-levels). - Expected values are computed in the test. Don't put a reference solution in a file that student code can read. @@ -474,17 +472,18 @@ shows both frames. The output level sets how much of a failure students see: -| Level | Shows to student | -| ------------- | ------------------------------------------------------------- | -| `"none"` | Exception class name only (e.g., `AssertionError`) | -| `"message"` | Exception name + first line of message _(default when unset)_ | -| `"traceback"` | Exception name + message + full traceback | -| `"friendly"` | Only the exception message text — no class name, no traceback | +| Level | Shows to student | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `"friendly"` | The exception message only, with no class name _(default)_. If a call to a student function raised the error, the message includes the traceback from the student's code. | +| `"none"` | Exception class name only (e.g., `AssertionError`) | +| `"message"` | Exception name + first line of message | +| `"traceback"` | Exception name + message + full traceback | -Use `"friendly"` with the assertion helpers. Set it on one test: +The default, `"friendly"`, works well with the assertion helpers. To show more, such as the full +traceback for advanced students, set a different level on one test: ```python -@pytest.mark.output(level="friendly") +@pytest.mark.output(level="traceback") @pytest.mark.grading_data(name="Test addition", points=5) def test_add(sandbox: StudentFixture) -> None: assert_fn_equal(sandbox, "add", args=(2, 3), expected=5) @@ -493,18 +492,17 @@ def test_add(sandbox: StudentFixture) -> None: On every test in a file, with a module-level `pytestmark`: ```python -pytestmark = pytest.mark.output(level="friendly") +pytestmark = pytest.mark.output(level="traceback") ``` Or in the `ConfigObject`: ```python -autograder_config = ConfigObject(output_level="friendly") +autograder_config = ConfigObject(output_level="traceback") ``` With the `ConfigObject` approach, individual tests can still override it with -`@pytest.mark.output(level="traceback")`. The `"message"` and `"traceback"` levels can help -advanced students, who can learn from reading tracebacks. +`@pytest.mark.output(level=...)`. ### Partial Credit and Feedback @@ -901,7 +899,7 @@ All parameters are optional: | Parameter | Default | Description | | -------------------------- | -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | | `sandbox_timeout` | `1.0` | Seconds allowed for starting the sandbox, which runs the setup code and student code. See [Timeout Configuration](#timeout-configuration). | -| `output_level` | `None` | Default [output level](#output-levels) for every test. | +| `output_level` | `None` | Default [output level](#output-levels) for every test. `None` means `"friendly"`. | | `names_for_user` | `None` | Variables to give student code. See [Giving Values to Student Code](#giving-values-to-student-code). | | `starting_vars` | `{}` | Values for names in `names_for_user`. | | `student_code_pattern` | `"student_code*.py"` | Glob pattern for the student's file. | @@ -1955,8 +1953,8 @@ To see more examples of what is possible in these test files, look at the test c 1. **Keep code in the tests**: See [Where Code Belongs](#where-code-belongs). 2. **Use descriptive test names**: The `name` in `@pytest.mark.grading_data` is shown to students. -3. **Use friendly messages that don't give away answers**: Use the assertion helpers with the - `friendly` output level, and `assert_true` for hidden test cases. +3. **Use friendly messages that don't give away answers**: Use the assertion helpers, and + `assert_true` for hidden test cases. 4. **Test incrementally**: Break complex problems into smaller tests with partial credit. 5. **Prefer `parametrize`**: Use `@pytest.mark.parametrize` for several inputs instead of a loop, unless the inputs share partial credit. diff --git a/src/pytest_prairielearn_grader/config.py b/src/pytest_prairielearn_grader/config.py index 36ec890..e59a171 100644 --- a/src/pytest_prairielearn_grader/config.py +++ b/src/pytest_prairielearn_grader/config.py @@ -253,12 +253,14 @@ def test_student_code(sandbox): Individual tests can override this with ``@pytest.mark.output(level="...")``. Levels: + - ``"friendly"``: Show only the exception message, with no class name + (default when None). Designed for use with the ``assert_equal`` / + ``assert_fn_equal`` helpers to produce clean, student-readable output. + If a call to a student function raised the error, the message includes + the traceback from the student's code. - ``"none"``: Show only the exception class name (e.g., ``AssertionError``). - - ``"message"``: Show exception name + first line of message (default when None). + - ``"message"``: Show exception name + first line of message. - ``"traceback"``: Show exception name + message + full traceback. - - ``"friendly"``: Show only the exception message text with no traceback or - class name. Designed for use with the ``assert_equal`` / ``assert_fn_equal`` - helpers to produce clean, student-readable output. When None, the plugin default of ``"message"`` is used unless overridden by a per-test ``@pytest.mark.output(level=...)`` marker. diff --git a/src/pytest_prairielearn_grader/utils.py b/src/pytest_prairielearn_grader/utils.py index a0b0827..a50715e 100644 --- a/src/pytest_prairielearn_grader/utils.py +++ b/src/pytest_prairielearn_grader/utils.py @@ -398,4 +398,4 @@ def get_output_level_marker(marker: pytest.Mark | None, config_level: str | None except ValueError: pass - return GradingOutputLevel.ExceptionMessage + return GradingOutputLevel.Friendly diff --git a/tests/scenario_root/test_fixture_exception/scenario.py b/tests/scenario_root/test_fixture_exception/scenario.py index ffda8c8..2cc7f98 100644 --- a/tests/scenario_root/test_fixture_exception/scenario.py +++ b/tests/scenario_root/test_fixture_exception/scenario.py @@ -14,7 +14,7 @@ def ans(sandbox: StudentFixture): @pytest.mark.grading_data(name="Test with fixture exception", points=1) -@pytest.mark.output("exception_message") +@pytest.mark.output(level="message") def test_with_fixture_exception(sandbox: StudentFixture, ans) -> None: """ This test uses a fixture that accesses student code. diff --git a/tests/scenario_root/test_global_output_level/expected_outcome.json b/tests/scenario_root/test_global_output_level/expected_outcome.json index 039c9d4..6b9416e 100644 --- a/tests/scenario_root/test_global_output_level/expected_outcome.json +++ b/tests/scenario_root/test_global_output_level/expected_outcome.json @@ -4,7 +4,7 @@ "tests": [ { "max_points": 2, - "name": "Test add (global friendly)", + "name": "Test add (global message)", "points_frac": 1.0, "points": 2.0, "test_id": "test_global_output_level.py::test_fn_equal[student_code]", @@ -12,8 +12,8 @@ }, { "max_points": 2, - "message": "Checking: add(2, 3)\nExpected output: 5\nYour code output: -1\nThe expected and actual output do not match.", - "name": "Test add (global friendly)", + "message": "Student code grading failed with an exception: AssertionError\nException Message: Checking: add(2, 3)", + "name": "Test add (global message)", "points_frac": 0.0, "points": 0.0, "test_id": "test_global_output_level.py::test_fn_equal[student_code_wrong]", @@ -21,7 +21,7 @@ }, { "max_points": 1, - "name": "Test variable x (global friendly)", + "name": "Test variable x (global message)", "points_frac": 1.0, "points": 1.0, "test_id": "test_global_output_level.py::test_equal_variable[student_code]", @@ -29,8 +29,8 @@ }, { "max_points": 1, - "message": "Checking: variable 'x'\nExpected output: 42\nYour code output: 99\nThe expected and actual output do not match.", - "name": "Test variable x (global friendly)", + "message": "Student code grading failed with an exception: AssertionError\nException Message: Checking: variable 'x'", + "name": "Test variable x (global message)", "points_frac": 0.0, "points": 0.0, "test_id": "test_global_output_level.py::test_equal_variable[student_code_wrong]", diff --git a/tests/scenario_root/test_global_output_level/scenario.py b/tests/scenario_root/test_global_output_level/scenario.py index 01d0076..608e7b6 100644 --- a/tests/scenario_root/test_global_output_level/scenario.py +++ b/tests/scenario_root/test_global_output_level/scenario.py @@ -1,4 +1,4 @@ -"""Test that output_level="friendly" in ConfigObject applies globally without per-test markers.""" +"""Test that output_level in ConfigObject applies globally without per-test markers.""" import pytest @@ -7,20 +7,19 @@ from pytest_prairielearn_grader.assertions import assert_fn_equal from pytest_prairielearn_grader.fixture import StudentFixture -# Global friendly output — NO per-test @pytest.mark.output needed autograder_config = ConfigObject( - output_level="friendly", + output_level="message", ) -@pytest.mark.grading_data(name="Test add (global friendly)", points=2) +@pytest.mark.grading_data(name="Test add (global message)", points=2) def test_fn_equal(sandbox: StudentFixture) -> None: - """Should show friendly message without per-test marker.""" + """Should use the global level without a per-test marker.""" assert_fn_equal(sandbox, "add", args=(2, 3), expected=5) -@pytest.mark.grading_data(name="Test variable x (global friendly)", points=1) +@pytest.mark.grading_data(name="Test variable x (global message)", points=1) def test_equal_variable(sandbox: StudentFixture) -> None: - """Should show friendly message without per-test marker.""" + """Should use the global level without a per-test marker.""" value = sandbox.query("x") assert_equal(value, 42, description="variable 'x'") diff --git a/tests/scenario_root/test_helper_modules_workspace_name_clash/expected_outcome.json b/tests/scenario_root/test_helper_modules_workspace_name_clash/expected_outcome.json index 554fa06..b8f39ed 100644 --- a/tests/scenario_root/test_helper_modules_workspace_name_clash/expected_outcome.json +++ b/tests/scenario_root/test_helper_modules_workspace_name_clash/expected_outcome.json @@ -5,7 +5,7 @@ { "test_id": "test_helper_modules_workspace_name_clash.py::test_query", "outcome": "error", - "message": "Student code execution failed with an exception: ImportError\nException Message: These names are reserved for modules that this question provides: helper_function.py. These names are already used by modules that the grader has loaded: random.py. Rename yours and update any imports that use them.", + "message": "These names are reserved for modules that this question provides: helper_function.py. These names are already used by modules that the grader has loaded: random.py. Rename yours and update any imports that use them.", "max_points": 1, "points": 0.0, "points_frac": 0.0, diff --git a/tests/scenario_root/test_private_namespace/expected_outcome.json b/tests/scenario_root/test_private_namespace/expected_outcome.json index 2f2879c..5d0f8b4 100644 --- a/tests/scenario_root/test_private_namespace/expected_outcome.json +++ b/tests/scenario_root/test_private_namespace/expected_outcome.json @@ -4,7 +4,7 @@ "tests": [ { "max_points": 1, - "message": "Student code execution failed with an exception: NameError\nException Message: name 'other_secret_value' is not defined", + "message": "name 'other_secret_value' is not defined", "name": "test_private_namespace.py::test_random[student_code_fail_other]", "outcome": "error", "points": 0.0, @@ -13,7 +13,7 @@ }, { "max_points": 1, - "message": "Student code execution failed with an exception: NameError\nException Message: name 'super_secret_value' is not defined", + "message": "name 'super_secret_value' is not defined", "name": "test_private_namespace.py::test_random[student_code_fail]", "outcome": "error", "points": 0.0, @@ -22,7 +22,7 @@ }, { "max_points": 1, - "message": "Student code execution failed with an exception: NameError\nException Message: name '__data_params' is not defined", + "message": "name '__data_params' is not defined", "name": "test_private_namespace.py::test_random[student_code_fail_read_builtin]", "outcome": "error", "points": 0.0, diff --git a/tests/scenario_root/test_quiz_2_1/expected_outcome.json b/tests/scenario_root/test_quiz_2_1/expected_outcome.json index 55b61de..41a6745 100644 --- a/tests/scenario_root/test_quiz_2_1/expected_outcome.json +++ b/tests/scenario_root/test_quiz_2_1/expected_outcome.json @@ -22,7 +22,7 @@ }, { "max_points": 6, - "message": "Student code execution failed with an exception: RuntimeError", + "message": "Calling sympy.series is not allowed in this question.", "name": "Test analyze_vibration function", "outcome": "error", "points_frac": 0.0, @@ -31,7 +31,7 @@ }, { "max_points": 6, - "message": "Student code execution failed with an exception: RuntimeError", + "message": "Calling sympy.series is not allowed in this question.", "name": "Test analyze_vibration function", "outcome": "error", "points_frac": 0.0, diff --git a/tests/scenario_root/test_startup_exception_message/expected_outcome.json b/tests/scenario_root/test_startup_exception_message/expected_outcome.json index 354a744..a3d53d6 100644 --- a/tests/scenario_root/test_startup_exception_message/expected_outcome.json +++ b/tests/scenario_root/test_startup_exception_message/expected_outcome.json @@ -4,7 +4,7 @@ "tests": [ { "test_id": "test_startup_exception_message.py::test_default_level[student_code]", - "message": "Student code execution failed with an exception: ValueError\nException Message: first line of the reason\nsecond line of the reason", + "message": "first line of the reason\nsecond line of the reason", "outcome": "error", "max_points": 1, "points": 0.0, diff --git a/tests/scenario_root/test_stdout_feature/expected_outcome.json b/tests/scenario_root/test_stdout_feature/expected_outcome.json index 0020f76..f9f25aa 100644 --- a/tests/scenario_root/test_stdout_feature/expected_outcome.json +++ b/tests/scenario_root/test_stdout_feature/expected_outcome.json @@ -60,7 +60,7 @@ }, { "test_id": "test_stdout_feature.py::test_global_scope_stdout[student_code]", - "message": "Student code grading failed with an exception: NameError\nException Message: Function 'get_global_message' wasn't found in the student code", + "message": "Function 'get_global_message' wasn't found in the student code", "max_points": 2, "points_frac": 0.0, "points": 0.0, diff --git a/tests/scenario_root/test_workspace_shadowed_module/expected_outcome.json b/tests/scenario_root/test_workspace_shadowed_module/expected_outcome.json index 5db5346..4e507dc 100644 --- a/tests/scenario_root/test_workspace_shadowed_module/expected_outcome.json +++ b/tests/scenario_root/test_workspace_shadowed_module/expected_outcome.json @@ -5,7 +5,7 @@ "tests": [ { "test_id": "test_workspace_shadowed_module.py::test_uses_random", - "message": "Student code execution failed with an exception: ImportError\nException Message: These names are already used by modules that the grader has loaded: json/, random.py. Rename yours and update any imports that use them.", + "message": "These names are already used by modules that the grader has loaded: json/, random.py. Rename yours and update any imports that use them.", "outcome": "error", "max_points": 1, "points": 0.0, diff --git a/tests/scenario_root/test_workspace_startup_exception/expected_outcome.json b/tests/scenario_root/test_workspace_startup_exception/expected_outcome.json index e317c5b..a57e921 100644 --- a/tests/scenario_root/test_workspace_startup_exception/expected_outcome.json +++ b/tests/scenario_root/test_workspace_startup_exception/expected_outcome.json @@ -4,7 +4,7 @@ "tests": [ { "test_id": "test_workspace_startup_exception.py::test_entry_raises", - "message": "Student code execution failed with an exception: ImportError\nException Message: main.py could not find the data it needs", + "message": "main.py could not find the data it needs", "outcome": "error", "max_points": 1, "points": 0.0, From 2d17a5fb12d08d685356943ca84cd94cf44ebc5b Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Tue, 17 Nov 2026 11:43:58 -0500 Subject: [PATCH 2/3] Score subtests and stop the sandbox after a timeout A test that uses pytest's subtests fixture gets the fraction of its subtests that passed, and its message shows the first 5 failures. After any query times out, later queries on the sandbox raise SandboxStopped, so a late reply can't be read as the answer to the next call. Co-Authored-By: Claude Opus 5.5 --- README.md | 16 ++ converting.md | 9 +- pyproject.toml | 2 +- quick_start.md | 62 +++++- src/pytest_prairielearn_grader/assertions.py | 26 +-- src/pytest_prairielearn_grader/fixture.py | 44 ++++- src/pytest_prairielearn_grader/plugin.py | 169 +++++++++++----- src/pytest_prairielearn_grader/utils.py | 20 +- tests/test_subtests.py | 197 +++++++++++++++++++ uv.lock | 2 +- 10 files changed, 460 insertions(+), 87 deletions(-) create mode 100644 tests/test_subtests.py diff --git a/README.md b/README.md index 8da264e..5becb0a 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,22 @@ Available helpers: - `assert_true(condition)` / `assert_false(condition)` — boolean checks - `assert_plot_equal(plot, expected)` — compare a matplotlib plot with a reference plot +### Partial Credit Across Inputs + +Check each input in a `subtests.test()` block, using pytest's built-in `subtests` fixture. The +test gets the fraction of blocks that passed, and the student sees the first few failures: + +```python +@pytest.mark.grading_data(name="Check random values", points=3) +def test_random_values(sandbox: StudentFixture, subtests: pytest.Subtests) -> None: + for n in [2, 3, 5, 8, 13]: + with subtests.test(): + assert_fn_equal(sandbox, "fib", args=(n,), expected=fib(n)) +``` + +After a call to student code times out, the sandbox stops answering queries, so the remaining +inputs fail without running. See [Checking Many Inputs](quick_start.md#checking-many-inputs). + ### Ungradable Submission Detection When student code has a `SyntaxError`, the submission is automatically marked as **ungradable** (`"gradable": false` in results). This means the student does not lose a grading attempt and sees a clear error message. diff --git a/converting.md b/converting.md index ccdfa81..873217d 100644 --- a/converting.md +++ b/converting.md @@ -187,7 +187,7 @@ regrade of a variant, and `question.html` could show it with `{{params.x}}`. | `Feedback.set_score(s)` | `feedback.set_score(s)` with the `feedback` fixture. See [Scoring](#scoring). | | `Feedback.add_feedback("...")` | `feedback.add_message("...")` | | `accuracy_critical=True` | A failed assertion ends the test. Other tests still run. | -| `total_iters = 3` and `@not_repeated` | `@pytest.mark.parametrize` over the inputs, or a loop with partial credit | +| `total_iters = 3` and `@not_repeated` | `@pytest.mark.parametrize` over the inputs, or a loop of `subtests.test()` blocks for partial credit | | `Feedback.not_allowed` | `import_blacklist=["numpy.linalg.inv"]` in the `ConfigObject`. See [Update the setup code](#4-update-the-setup-code). | | `repeated_setup()` | Not needed: setup code runs again for each sandbox. | @@ -246,6 +246,9 @@ legacy feedback where you can, such as `"x is inaccurate"`. - A test that fails gets the last score it set with `feedback.set_score`, or 0. Call `set_score` after each check that earns partial credit. - `feedback.set_score_final(s)` sets the score even if the test then fails. +- A legacy loop that adds up how many inputs were correct becomes a loop of `subtests.test()` + blocks. The test gets the fraction of blocks that passed. See + [Checking Many Inputs](quick_start.md#checking-many-inputs). - The legacy pattern of `if check: set_score(1) else: set_score(0)` becomes a plain assertion. ### Querying @@ -254,8 +257,8 @@ legacy feedback where you can, such as `"x is inaccurate"`. the test with a message that says so. You don't need to check this yourself. - Each call to `query_function` has a 1 second timeout by default. Pass `query_timeout=` for slower functions. -- When a test calls a student function several times, use `query_function_raw` and stop after the - first `"timeout"` status: see the `randomFunctionName` example question. +- After a call times out, later queries on the sandbox fail without running student code, so a + slow function can't use up the grading job's time. You don't need to stop a loop yourself. ## 4. Update the setup code diff --git a/pyproject.toml b/pyproject.toml index f8e8927..246f381 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "A pytest plugin for autograding Python code. Designed for use wit readme = "README.md" authors = [{ name = "Eliot Robson", email = "eliot.robson24@gmail.com" }] requires-python = ">=3.11" -dependencies = ["dill>=0.4.0", "pytest", "prettytable>=3.8.0"] +dependencies = ["dill>=0.4.0", "pytest>=9.0,<10", "prettytable>=3.8.0"] license = { file = "LICENSE" } classifiers = [ "Development Status :: 3 - Alpha", diff --git a/quick_start.md b/quick_start.md index 246543d..f0270ea 100644 --- a/quick_start.md +++ b/quick_start.md @@ -260,6 +260,9 @@ These fixtures are provided by the `pytest-prairielearn-grader` package: 6. **`workspace_sandbox: WorkspaceFixture`**: For multi-file workspace projects. See [Workspace Grading](#workspace-grading). +pytest's built-in **`subtests: pytest.Subtests`** fixture splits a test's points across many +inputs. See [Checking Many Inputs](#checking-many-inputs). + For implementation details, see the [fixture source code](https://github.com/prairielearn/grader-python-v2/blob/main/src/pytest_prairielearn_grader/fixture.py). ### Querying Student Code @@ -310,9 +313,8 @@ def test_function(sandbox: StudentFixture) -> None: You don't need to catch these errors: they fail the test with a message saying what went wrong. -Each call has a 1 second timeout by default. See [Function Timeout](#4-function-timeout). When a -test calls a student function many times, use `query_function_raw` and stop after the first -`"timeout"` status, so that a slow function doesn't use up the grading job's time. +Each call has a 1 second timeout by default. See [Function Timeout](#4-function-timeout). To give +partial credit across many calls, see [Checking Many Inputs](#checking-many-inputs). **Testing for specific exceptions with `query_function_raw`:** @@ -339,7 +341,7 @@ def test_raises_value_error(sandbox: StudentFixture, feedback: FeedbackFixture) The `query_function_raw` response contains: -- `status`: One of `"success"`, `"exception"`, `"timeout"`, `"not_found"`, or `"not_serializable"`. `"not_found"` means the function doesn't exist, and only the lookup can cause it: an error raised while the function runs, even a `KeyError` or `ImportError`, is `"exception"`. `"not_serializable"` means the function finished, but its return value can't be sent back to the grader. `exception_message` explains this, and later queries on the sandbox still work. +- `status`: One of `"success"`, `"exception"`, `"timeout"`, `"not_found"`, or `"not_serializable"`. After a `"timeout"`, later queries on the sandbox raise an error: see [Function Timeout](#4-function-timeout). `"not_found"` means the function doesn't exist, and only the lookup can cause it: an error raised while the function runs, even a `KeyError` or `ImportError`, is `"exception"`. `"not_serializable"` means the function finished, but its return value can't be sent back to the grader. `exception_message` explains this, and later queries on the sandbox still work. - `value`: The return value (when status is `"success"`) - `stdout` / `stderr`: Captured output from the function call - `exception_name`: The exception class name (e.g., `"ValueError"`) @@ -373,6 +375,12 @@ Control execution time for individual function calls: result = sandbox.query_function("slow_computation", data, query_timeout=2.0) ``` +After any query times out, every later query on that sandbox raises `SandboxStopped`. The student +code may still send the late answer, which would otherwise be read as the answer to the next call. +So a timeout fails the rest of the test, and with a [module-scoped sandbox](#module-scoped-sandbox), +the later tests that use it too. In a [subtests loop](#checking-many-inputs), the inputs before the +timeout keep their credit. + #### 5. Query Setup Variables ```python @@ -538,6 +546,52 @@ def test_with_partial_credit(sandbox: StudentFixture, feedback: FeedbackFixture) - `feedback.add_message(msg)`: Add custom feedback message - `feedback.add_image(image, label=None)`: Show an image with the test's results. See [Showing Plots to Students](#showing-plots-to-students) +### Checking Many Inputs + +To split a test's points across many inputs, check each input in a `subtests.test()` block, using +pytest's built-in `subtests` fixture: + +```python +@pytest.mark.grading_data(name="Check random values", points=3) +def test_random_values(sandbox: StudentFixture, subtests: pytest.Subtests) -> None: + for n in [2, 3, 5, 8, 13, 17, 19, 23, 27, 29]: + with subtests.test(): + assert_fn_equal(sandbox, "fib", args=(n,), expected=fib(n)) +``` + +A failed assertion or exception inside a block fails only that block, and the loop goes on. The +test's score is the fraction of blocks that passed: + +- A block that calls `pytest.skip` counts as failed. +- If the test fails outside any block, its score is 0, or the last score it set with + `feedback.set_score`. +- `feedback.set_score_final` replaces the fraction. + +The test's message starts with how many blocks passed, such as `7/10 checks passed.`, followed by +the messages of the first 5 failed blocks. Passed blocks aren't listed. Pass a label to show it +before a block's message, such as `subtests.test(f"last_element({lst})")`. The `assert_fn_*` +helpers already name the call, so they don't need one. + +A block can check several things about one input: + +```python +@pytest.mark.grading_data(name="last_element", points=10) +def test_last_element(sandbox: StudentFixture, subtests: pytest.Subtests) -> None: + for _ in range(10): + lst = random.sample(range(1000), k=random.randint(1, 20)) + with subtests.test(f"last_element({lst})"): + value, lst_after = sandbox.query_function("last_element_and_list", lst) + assert_equal(value, lst[-1]) + assert_equal(lst_after, lst, msg="Your function should not modify the input list.") +``` + +When a call times out, the remaining blocks fail without running student code, and the message +says so once: `The remaining 3 checks didn't run because fib(23) timed out after 1.0 seconds.` +See [Function Timeout](#4-function-timeout). + +Use `@pytest.mark.parametrize` instead when each input should be its own test, with its own name +and points. + ### Capturing and Testing Output Control whether student code output appears in feedback: diff --git a/src/pytest_prairielearn_grader/assertions.py b/src/pytest_prairielearn_grader/assertions.py index ea442da..9aaaaf0 100644 --- a/src/pytest_prairielearn_grader/assertions.py +++ b/src/pytest_prairielearn_grader/assertions.py @@ -21,22 +21,8 @@ def test_add(sandbox): from typing import Any from .fixture import _SandboxBase - - -def _format_value(value: Any, *, max_len: int = 200) -> str: - """Return a repr of *value*, truncated for readability.""" - text = repr(value) - if len(text) > max_len: - text = text[: max_len - 3] + "..." - return text - - -def _format_call(func_name: str, args: tuple[Any, ...], kwargs: dict[str, Any] | None) -> str: - """Format ``func_name(args, kwargs)`` for display.""" - parts: list[str] = [_format_value(a, max_len=60) for a in args] - if kwargs: - parts.extend(f"{k}={_format_value(v, max_len=60)}" for k, v in kwargs.items()) - return f"{func_name}({', '.join(parts)})" +from .utils import format_call +from .utils import format_value def _build_failure_lines( @@ -50,8 +36,8 @@ def _build_failure_lines( lines: list[str] = [] if description: lines.append(f"Checking: {description}") - lines.append(f"Expected output: {_format_value(expected)}") - lines.append(f"Your code output: {_format_value(actual)}") + lines.append(f"Expected output: {format_value(expected)}") + lines.append(f"Your code output: {format_value(actual)}") if msg: lines.append(msg) else: @@ -350,7 +336,7 @@ def assert_fn_equal( """ call_kwargs = kwargs or {} actual = sandbox.query_function(func_name, *args, query_timeout=query_timeout, **call_kwargs) - call_description = description or _format_call(func_name, args, kwargs) + call_description = description or format_call(func_name, args, kwargs) assert_equal(actual, expected, msg=msg, description=call_description) return actual @@ -380,6 +366,6 @@ def assert_fn_approx_equal( """ call_kwargs = kwargs or {} actual = sandbox.query_function(func_name, *args, query_timeout=query_timeout, **call_kwargs) - call_description = description or _format_call(func_name, args, kwargs) + call_description = description or format_call(func_name, args, kwargs) assert_approx_equal(actual, expected, rtol=rtol, atol=atol, msg=msg, description=call_description) return actual diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index dd2a519..4160cd1 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -30,6 +30,7 @@ from .utils import WorkspaceStartRequest from .utils import deserialize_object_unsafe from .utils import drop_privileges +from .utils import format_call from .utils import serialize_object_unsafe DataFixture = dict[str, Any] @@ -50,6 +51,21 @@ logger = logging.getLogger(__name__) +class SandboxStopped(RuntimeError): + """ + Raised by a query on a sandbox where an earlier query timed out. + + The student code may still send the reply to the call that timed out, and a later query would + read it as its own answer. So after a timeout, the sandbox refuses all further queries. + """ + + timed_out_call: str + + def __init__(self, timed_out_call: str) -> None: + super().__init__(f"This check didn't run because {timed_out_call}.") + self.timed_out_call = timed_out_call + + def _extract_notebook_code(notebook_path: Path, cell_tag: str | None = None) -> str: """Extract Python source code from a Jupyter notebook (.ipynb) file. @@ -209,6 +225,7 @@ class _SandboxBase: _data_files_dir: Path | None _accumulated_stdout: list[str] _server_failure: str | None + _timed_out_call: str | None def __init__( self, @@ -232,6 +249,7 @@ def __init__( self.student_socket = None self._accumulated_stdout = [] self._server_failure = None + self._timed_out_call = None def _assert_process_running(self) -> None: assert self.process is not None, "Sandbox process is not running. Please start it first." @@ -268,6 +286,8 @@ def _read_from_socket(self) -> bytes: def _assert_no_server_failure(self) -> None: if self._server_failure is not None: raise RuntimeError(self._server_failure) + if self._timed_out_call is not None: + raise SandboxStopped(self._timed_out_call) def _read_query_response(self) -> Any: """Read a response to a query, raising a clear error if the server failed and closed the connection.""" @@ -335,7 +355,11 @@ def query_raw(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT assert self.student_socket is not None self.student_socket.settimeout(query_timeout) self._send_json_object(json_message) - data: StudentQueryResponse = self._read_query_response() + try: + data: StudentQueryResponse = self._read_query_response() + except TimeoutError as error: + self._timed_out_call = f"the query for '{var_to_query}' timed out after {query_timeout} seconds" + raise TimeoutError(f"The query for '{var_to_query}' timed out after {query_timeout} seconds.") from error return data def query(self, var_to_query: str, *, query_timeout: float = DEFAULT_TIMEOUT) -> Any: @@ -354,9 +378,9 @@ def query_function_raw( """Call a student function and return its protocol response. A socket timeout is returned as ``FunctionStatusCode.TIMEOUT`` rather - than raised. Callers must stop issuing requests on this sandbox after a - timeout because the student process may still write the timed-out - request's response. + than raised. After a timeout, every later query on this sandbox raises + SandboxStopped, because the student process may still write the + timed-out request's response. Raises a RuntimeError if the sandbox server failed, for example because it couldn't load the arguments. The sandbox can't be used after that. @@ -375,13 +399,14 @@ def query_function_raw( try: data: StudentFunctionResponse = self._read_query_response() except TimeoutError as error: + self._timed_out_call = f"{format_call(function_name, args, kwargs)} timed out after {query_timeout} seconds" data = { "status": FunctionStatusCode.TIMEOUT, "value": None, "stdout": "", "stderr": "", "exception_name": type(error).__name__, - "exception_message": str(error), + "exception_message": f"{self._timed_out_call}.", "traceback": None, } if data.get("stdout"): @@ -393,12 +418,10 @@ def query_function(self, function_name: str, *args: Any, query_timeout: float = response = self.query_function_raw(function_name, *args, query_timeout=query_timeout, **kwargs) match response["status"]: case FunctionStatusCode.EXCEPTION: - raise RuntimeError( - f"Function '{function_name}' raised an exception " - f"{response['exception_name']}: {response['exception_message']}\n{response['traceback']}" - ) + call = format_call(function_name, args, kwargs) + raise RuntimeError(f"{call} raised {response['exception_name']}: {response['exception_message']}\n{response['traceback']}") case FunctionStatusCode.TIMEOUT: - raise TimeoutError(f"Query for function '{function_name}' timed out after {query_timeout} seconds.") + raise TimeoutError(response["exception_message"]) case FunctionStatusCode.NOT_FOUND: raise NameError(response["exception_message"]) case FunctionStatusCode.NOT_SERIALIZABLE: @@ -496,6 +519,7 @@ def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_T return self._read_start_response(initialization_timeout) def query_setup_raw(self, var_to_query: str) -> SetupQueryResponse: + self._assert_no_server_failure() self._assert_process_running() json_message: SetupQueryRequest = {"message_type": MessageType.QUERY_SETUP, "var": var_to_query} assert self.student_socket is not None, "Student socket is not connected. Please start the student code server first." diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index d9f1c4c..67d7019 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -4,6 +4,7 @@ import sys from collections.abc import Iterable from copy import deepcopy +from dataclasses import dataclass from pathlib import Path from types import ModuleType from typing import Any @@ -20,6 +21,7 @@ from .config import ConfigObject from .fixture import FeedbackFixture +from .fixture import SandboxStopped from .fixture import StudentFiles from .fixture import StudentFixture from .fixture import WorkspaceFixture @@ -33,6 +35,8 @@ # Default configuration constants DEFAULT_SANDBOX_TIMEOUT = 1.0 DEFAULT_IMPORT_BLACKLIST = ["os", "sys", "subprocess", "pathlib", "shutil"] +# Failing subtests shown in a test's message. The rest are only counted, to keep the message readable. +MAX_SHOWN_SUBTEST_FAILURES = 5 class SandboxStartupFailed(Failed): @@ -56,6 +60,83 @@ class TestResult(NamedTuple): stdout: str +@dataclass(frozen=True, slots=True) +class SubtestResult: + """The result of one `subtests.test()` block in a test.""" + + label: str | None + outcome: str + excinfo: pytest.ExceptionInfo[BaseException] | None + + +def format_failure(excinfo: pytest.ExceptionInfo[BaseException], output_level: GradingOutputLevel, when: str) -> str: + """Format the exception that failed a test, or one of its subtests, for the test's output level.""" + if isinstance(excinfo.value, SandboxStartupFailed): + # The sandbox startup handler already formatted this message for the + # output level, so show it in full without another exception header. + return str(excinfo.value) + + if output_level == GradingOutputLevel.Friendly: + # Friendly mode: show only the exception message text with + # no exception class name, traceback, or "failed with" prefix. + return str(excinfo.value) + + # Customize the message based on the failure phase + if when == "setup": + phase_message = "Student code execution failed with an exception" + elif when == "teardown": + phase_message = "Student code teardown failed with an exception" + else: + phase_message = "Student code grading failed with an exception" + + if output_level == GradingOutputLevel.ExceptionName: + return f"{phase_message}: {excinfo.type.__name__}" + + if output_level == GradingOutputLevel.ExceptionMessage: + # TODO make this work with multiline messages somehow? + exception_message = str(excinfo.value).split(os.linesep)[0] + return f"{phase_message}: {excinfo.type.__name__}{os.linesep}Exception Message: {exception_message}" + + # If showing more than the exception name, show the message + full traceback + return str(excinfo.getrepr(style="no")) + + +def format_subtest_results(subtests: list[SubtestResult], output_level: GradingOutputLevel) -> str: + """ + Summarize a test's subtests: how many passed, the first few failures, and the checks that + didn't run because the sandbox stopped after a timeout. + """ + passed = sum(subtest.outcome == "passed" for subtest in subtests) + blocks = [f"{passed}/{len(subtests)} checks passed."] + + stopped: list[SandboxStopped] = [] + shown: list[SubtestResult] = [] + for subtest in subtests: + if subtest.excinfo is not None and isinstance(subtest.excinfo.value, SandboxStopped): + stopped.append(subtest.excinfo.value) + elif subtest.outcome != "passed": + shown.append(subtest) + + for subtest in shown[:MAX_SHOWN_SUBTEST_FAILURES]: + message = format_failure(subtest.excinfo, output_level, "call") if subtest.excinfo is not None else subtest.outcome + if subtest.label is None: + blocks.append(message) + elif "\n" in message: + blocks.append(f"{subtest.label}:{os.linesep}{message}") + else: + blocks.append(f"{subtest.label}: {message}") + + if len(shown) > MAX_SHOWN_SUBTEST_FAILURES: + hidden = len(shown) - MAX_SHOWN_SUBTEST_FAILURES + blocks.append(f"...and {hidden} more {'check' if hidden == 1 else 'checks'} failed.") + + if stopped: + remaining = "The remaining check" if len(stopped) == 1 else f"The remaining {len(stopped)} checks" + blocks.append(f"{remaining} didn't run because {stopped[0].timed_out_call}.") + + return (os.linesep * 2).join(blocks) + + def get_tests_dir(test_module: ModuleType) -> Path | None: """ Get the tests folder for a test module: the folder named after the module, next to it. @@ -867,6 +948,10 @@ def pytest_configure(config: Config) -> None: class ResultCollectorPlugin: collected_results: dict[str, TestResult] + # The call result made for each test and each of its subtests, waiting for + # pytest_runtest_logreport to say which one it was. + pending_calls: dict[str, TestResult] + subtest_results: dict[str, list[SubtestResult]] student_feedback_data: dict[str, FeedbackFixture] grading_data: dict[str, Any] module_sandbox_cache: dict[tuple[str, str], StudentFixture] @@ -876,6 +961,8 @@ class ResultCollectorPlugin: def __init__(self) -> None: self.collected_results = {} + self.pending_calls = {} + self.subtest_results = {} self.student_feedback_data = {} self.grading_data = {} self.module_sandbox_cache = {} @@ -931,12 +1018,9 @@ def pytest_runtest_makereport(self, item: pytest.Item, call: pytest.CallInfo) -> if stdout_content.strip(): # Only store if there's actual content accumulated_stdout = stdout_content - self.collected_results[report.nodeid] = TestResult(report=report, call=call, stdout=accumulated_stdout) - # You could store more details here if needed - # item.config.my_test_results[report.nodeid] = { - # "outcome": report.outcome, - # "duration": report.duration, - # } + # pytest also calls this hook for each subtest, with the subtest's call info, + # so pytest_runtest_logreport decides where the result goes. + self.pending_calls[report.nodeid] = TestResult(report=report, call=call, stdout=accumulated_stdout) fixture = None # This code section appears to be unused legacy code @@ -951,6 +1035,21 @@ def pytest_runtest_makereport(self, item: pytest.Item, call: pytest.CallInfo) -> # if fixture: # fixture.skipped = outcome.get_result().outcome == "skipped" + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + """ + Sort each call result into the test's own result or one of its subtests' results. + """ + if report.when != "call" or report.nodeid not in self.pending_calls: + return + + result = self.pending_calls.pop(report.nodeid) + if isinstance(report, pytest.SubtestReport): + self.subtest_results.setdefault(report.nodeid, []).append( + SubtestResult(label=report.context.msg, outcome=report.outcome, excinfo=result.call.excinfo) + ) + else: + self.collected_results[report.nodeid] = result + @pytest.fixture def feedback(self, request: pytest.FixtureRequest) -> FeedbackFixture: """ @@ -1018,48 +1117,18 @@ def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> Iter # Create an empty feedback object if none was created during the test feedback_obj = FeedbackFixture(test_id=nodeid) - # If the test failed (in any phase), add the exception message to the feedback - if report.outcome == "failed" and call.excinfo is not None: - config_level = _get_config_output_level(getattr(item, "module", None)) - output_level: GradingOutputLevel = get_output_level_marker(item.get_closest_marker("output"), config_level) - - logger.debug(f"Grading output level set to: {output_level}") - - if isinstance(call.excinfo.value, SandboxStartupFailed): - # The sandbox startup handler already formatted this message for the - # output level, so show it in full without another exception header. - feedback_obj.add_message(str(call.excinfo.value)) + config_level = _get_config_output_level(getattr(item, "module", None)) + output_level: GradingOutputLevel = get_output_level_marker(item.get_closest_marker("output"), config_level) + logger.debug(f"Grading output level set to: {output_level}") - elif output_level == GradingOutputLevel.Friendly: - # Friendly mode: show only the exception message text with - # no exception class name, traceback, or "failed with" prefix. - exception_message = str(call.excinfo.value) - feedback_obj.add_message(exception_message) + subtests = self.subtest_results.get(nodeid, []) + if subtests: + feedback_obj.add_message(format_subtest_results(subtests, output_level)) - else: - # Customize the message based on the failure phase - if report.when == "setup": - phase_message = "Student code execution failed with an exception" - elif report.when == "teardown": - phase_message = "Student code teardown failed with an exception" - else: - phase_message = "Student code grading failed with an exception" - - if output_level == GradingOutputLevel.ExceptionName: - exception_name = call.excinfo.type.__name__ - fail_message = f"{phase_message}: {exception_name}" - feedback_obj.add_message(fail_message) - - elif output_level == GradingOutputLevel.ExceptionMessage: - exception_name = call.excinfo.type.__name__ - # TODO make this work with multiline messages somehow? - exception_message = str(call.excinfo.value).split(os.linesep)[0] - fail_message = f"{phase_message}: {exception_name}{os.linesep}Exception Message: {exception_message}" - feedback_obj.add_message(fail_message) - - # If showing more than the exception name, show the message + full traceback - else: - feedback_obj.add_message(str(call.excinfo.getrepr(style="no"))) + # If the test failed (in any phase), add the exception message to the feedback. + # A test whose subtests failed is also marked failed, but without exception info. + if report.outcome == "failed" and call.excinfo is not None: + feedback_obj.add_message(format_failure(call.excinfo, output_level, report.when)) # Check if stdout feedback should be included (default True) include_stdout_feedback = grading_data.get("include_stdout_feedback", True) @@ -1101,6 +1170,14 @@ def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> Iter # For now, we raise an error for unexpected outcomes raise ValueError(f"Unexpected outcome '{outcome}' for test '{nodeid}'.") + # A test with subtests gets the fraction of them that passed, and skipped subtests count + # as failed. pytest marks the test failed when any subtest fails, so check whether the + # test body itself raised instead. If it did, the score from above (0 or set_score) stays. + if subtests and report.when == "call" and call.excinfo is None and not feedback_obj.final_score_override: + passed = sum(subtest.outcome == "passed" for subtest in subtests) + res_obj["points_frac"] = passed / len(subtests) + res_obj["outcome"] = "passed" if passed == len(subtests) else "failed" + res_obj["points"] = res_obj["points_frac"] * res_obj["max_points"] final_results.append(res_obj) diff --git a/src/pytest_prairielearn_grader/utils.py b/src/pytest_prairielearn_grader/utils.py index a50715e..04da00d 100644 --- a/src/pytest_prairielearn_grader/utils.py +++ b/src/pytest_prairielearn_grader/utils.py @@ -11,6 +11,22 @@ import pytest +def format_value(value: Any, *, max_len: int = 200) -> str: + """Return a repr of *value*, truncated for readability.""" + text = repr(value) + if len(text) > max_len: + text = text[: max_len - 3] + "..." + return text + + +def format_call(func_name: str, args: tuple[Any, ...], kwargs: dict[str, Any] | None) -> str: + """Format ``func_name(args, kwargs)`` for display.""" + parts: list[str] = [format_value(a, max_len=60) for a in args] + if kwargs: + parts.extend(f"{k}={format_value(v, max_len=60)}" for k, v in kwargs.items()) + return f"{func_name}({', '.join(parts)})" + + class GradingOutputLevel(StrEnum): ExceptionName = "none" ExceptionMessage = "message" @@ -33,8 +49,8 @@ class FunctionStatusCode(StrEnum): Status codes for a function execution. A timeout indicates that the function call did not produce a response within - the requested interval. Callers must stop issuing requests on that sandbox - after receiving this status. + the requested interval. After this status, every later query on that sandbox + raises SandboxStopped. NOT_SERIALIZABLE means the function finished but its return value can't be sent to the grader. The sandbox can still be used after this status. diff --git a/tests/test_subtests.py b/tests/test_subtests.py new file mode 100644 index 0000000..6df6857 --- /dev/null +++ b/tests/test_subtests.py @@ -0,0 +1,197 @@ +import json +import textwrap +from typing import Any + +import pytest + +pytest_plugins = ("pytester",) + +STUDENT_CODE = """ +import time + + +def double(x): + if x == 3: + return 0 + if x == 4: + time.sleep(1) + if x == 6: + raise ValueError("six is not allowed") + return 2 * x +""" + +TEST_HEADER = """ +import pytest + +from pytest_prairielearn_grader import assert_fn_equal +from pytest_prairielearn_grader import assert_true +""" + + +def run_grader(pytester: pytest.Pytester, tests: str) -> dict[str, dict[str, Any]]: + """Grade STUDENT_CODE with the given tests, and return each test's result by name.""" + pytester.path.joinpath("test_grader.py").write_text(TEST_HEADER + textwrap.dedent(tests)) + pytester.mkdir("test_grader").joinpath("student_code.py").write_text(STUDENT_CODE) + pytester.runpytest("test_grader.py") + results = json.loads(pytester.path.joinpath("autograder_results.json").read_text()) + return {result["name"]: result for result in results["tests"]} + + +@pytest.mark.parametrize( + ("test_case", "points_frac", "messages"), + [ + pytest.param( + """ + def test_case(sandbox, subtests): + for x in [1, 2]: + with subtests.test(): + assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x) + """, + 1.0, + ["2/2 checks passed."], + id="all_pass", + ), + pytest.param( + """ + def test_case(sandbox, subtests): + for x in [1, 2, 3]: + with subtests.test(): + assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x) + """, + 2 / 3, + ["2/3 checks passed.", "Checking: double(3)"], + id="fraction", + ), + pytest.param( + """ + def test_case(sandbox, subtests): + for x in [1, 4, 2, 5]: + with subtests.test(): + assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x, query_timeout=0.2) + """, + 0.25, + [ + "1/4 checks passed.", + "double(4) timed out after 0.2 seconds.", + "The remaining 2 checks didn't run because double(4) timed out after 0.2 seconds.", + ], + id="timeout_stops_remaining", + ), + pytest.param( + """ + def test_case(sandbox, subtests): + for x in [6, 1]: + with subtests.test(): + assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x) + """, + 0.5, + ["double(6) raised ValueError: six is not allowed"], + id="exception", + ), + pytest.param( + """ + def test_case(sandbox, subtests): + for x in [1, 2]: + with subtests.test(): + if x == 2: + pytest.skip("not checked") + assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x) + """, + 0.5, + ["1/2 checks passed.", "not checked"], + id="skipped_counts_as_failed", + ), + pytest.param( + """ + def test_case(sandbox, subtests): + with subtests.test(): + assert_fn_equal(sandbox, "double", args=(1,), expected=2) + assert_true(False, msg="The test body failed.") + """, + 0.0, + ["1/1 checks passed.", "The test body failed."], + id="body_failure", + ), + pytest.param( + """ + def test_case(sandbox, subtests, feedback): + for x in [1, 3]: + with subtests.test(): + assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x) + feedback.set_score_final(0.9) + """, + 0.9, + ["1/2 checks passed."], + id="set_score_final", + ), + pytest.param( + """ + def test_case(sandbox, subtests): + for x in range(10, 18): + with subtests.test(): + assert_fn_equal(sandbox, "double", args=(x,), expected=-1) + """, + 0.0, + ["0/8 checks passed.", "...and 3 more checks failed."], + id="long_output", + ), + pytest.param( + """ + def test_case(sandbox, subtests): + with subtests.test("first"): + assert_true(False, msg="nope") + with subtests.test("second"): + assert_fn_equal(sandbox, "double", args=(3,), expected=6) + """, + 0.0, + ["first: nope", "second:\nChecking: double(3)"], + id="labels", + ), + ], +) +def test_subtests_score(pytester: pytest.Pytester, test_case: str, points_frac: float, messages: list[str]) -> None: + test_case = '@pytest.mark.grading_data(name="case", points=2)\n' + textwrap.dedent(test_case).lstrip() + result = run_grader(pytester, test_case)["case"] + + assert result["points_frac"] == pytest.approx(points_frac) + assert result["points"] == pytest.approx(2 * points_frac) + assert result["outcome"] == ("passed" if points_frac == 1 else "failed") + for message in messages: + assert message in result["message"] + assert "This check didn't run" not in result["message"] + + +def test_shown_failures_are_capped(pytester: pytest.Pytester) -> None: + message = run_grader( + pytester, + """ + @pytest.mark.grading_data(name="case", points=1) + def test_case(sandbox, subtests): + for x in range(10, 18): + with subtests.test(): + assert_fn_equal(sandbox, "double", args=(x,), expected=-1) + """, + )["case"]["message"] + + assert "Checking: double(14)" in message + assert "Checking: double(15)" not in message + + +def test_timeout_stops_later_tests_on_module_sandbox(pytester: pytest.Pytester) -> None: + results = run_grader( + pytester, + """ + @pytest.mark.grading_data(name="first", points=1) + def test_first(module_sandbox): + assert_fn_equal(module_sandbox, "double", args=(4,), expected=8, query_timeout=0.2) + + + @pytest.mark.grading_data(name="second", points=1) + def test_second(module_sandbox): + assert_fn_equal(module_sandbox, "double", args=(1,), expected=2) + """, + ) + + assert results["first"]["message"] == "double(4) timed out after 0.2 seconds." + assert results["second"]["message"] == "This check didn't run because double(4) timed out after 0.2 seconds." + assert results["second"]["points_frac"] == 0 diff --git a/uv.lock b/uv.lock index 60aa127..2842644 100644 --- a/uv.lock +++ b/uv.lock @@ -907,7 +907,7 @@ requires-dist = [ { name = "dill", specifier = ">=0.4.0" }, { name = "nbformat", marker = "extra == 'notebook'", specifier = ">=5.0" }, { name = "prettytable", specifier = ">=3.8.0" }, - { name = "pytest" }, + { name = "pytest", specifier = ">=9.0,<10" }, ] provides-extras = ["notebook"] From 9e6e0e0edb1296f88b9a9e226476a9177cfa8ab8 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Tue, 17 Nov 2026 12:25:18 -0500 Subject: [PATCH 3/3] Narrow format_failure's when argument to pytest's test phases Co-Authored-By: Claude Opus 5.5 --- src/pytest_prairielearn_grader/plugin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 67d7019..969d94e 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -8,6 +8,7 @@ from pathlib import Path from types import ModuleType from typing import Any +from typing import Literal from typing import NamedTuple from typing import cast @@ -69,7 +70,9 @@ class SubtestResult: excinfo: pytest.ExceptionInfo[BaseException] | None -def format_failure(excinfo: pytest.ExceptionInfo[BaseException], output_level: GradingOutputLevel, when: str) -> str: +def format_failure( + excinfo: pytest.ExceptionInfo[BaseException], output_level: GradingOutputLevel, when: Literal["setup", "call", "teardown"] +) -> str: """Format the exception that failed a test, or one of its subtests, for the test's output level.""" if isinstance(excinfo.value, SandboxStartupFailed): # The sandbox startup handler already formatted this message for the