Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions quick_start.md
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,18 @@ test's score is the fraction of blocks that passed:
`feedback.set_score`.
- `feedback.set_score_final` replaces the fraction.

To make some blocks count for more, pass a `weight`, which is 1 by default:

```python
for n, weight in [(2, 1), (10, 1), (29, 2)]:
with subtests.test(weight=weight):
assert_fn_equal(sandbox, "fib", args=(n,), expected=fib(n))
```

The score is then the weight of the blocks that passed divided by the total weight: here, passing
only `fib(29)` gives half the points. Weights are non-negative ints or floats. A block with weight 0
is still checked and shown, but doesn't change the score.

The test's message starts with how many blocks passed, such as `7/10 checks passed.`, followed by
the messages of the first 5 failed blocks. Passed blocks aren't listed. Pass a label to show it
before a block's message, such as `subtests.test(f"last_element({lst})")`. The `assert_fn_*`
Expand Down
58 changes: 50 additions & 8 deletions src/pytest_prairielearn_grader/plugin.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import json
import logging
import math
import os
import sys
from collections.abc import Iterable
Expand Down Expand Up @@ -68,6 +69,32 @@ class SubtestResult:
label: str | None
outcome: str
excinfo: pytest.ExceptionInfo[BaseException] | None
# The `weight` keyword argument. Newer pytest versions keep its repr, such as "2", instead.
weight: object = None


def get_subtest_weights(subtests: list[SubtestResult]) -> list[float]:
"""
Each subtest's weight from `subtests.test(weight=...)`, or 1 without one. Raises ValueError,
with a message for the instructor, if a weight isn't a non-negative number or they're all 0.
"""
weights = []
for subtest in subtests:
if subtest.weight is None:
weights.append(1.0)
continue
weight = math.nan
if isinstance(subtest.weight, int | float | str) and not isinstance(subtest.weight, bool):
try:
weight = float(subtest.weight)
except ValueError:
pass
if not math.isfinite(weight) or weight < 0:
raise ValueError(f"A subtest's weight must be a non-negative int or float, not {subtest.weight}.")
weights.append(weight)
if sum(weights) == 0:
raise ValueError("At least one subtest needs a weight greater than 0.")
return weights


def format_failure(
Expand Down Expand Up @@ -1048,7 +1075,12 @@ def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
result = self.pending_calls.pop(report.nodeid)
if isinstance(report, pytest.SubtestReport):
self.subtest_results.setdefault(report.nodeid, []).append(
SubtestResult(label=report.context.msg, outcome=report.outcome, excinfo=result.call.excinfo)
SubtestResult(
label=report.context.msg,
outcome=report.outcome,
excinfo=result.call.excinfo,
weight=report.context.kwargs.get("weight"),
)
)
else:
self.collected_results[report.nodeid] = result
Expand Down Expand Up @@ -1125,8 +1157,13 @@ def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> Iter
logger.debug(f"Grading output level set to: {output_level}")

subtests = self.subtest_results.get(nodeid, [])
subtest_weights: list[float] | None = None
if subtests:
feedback_obj.add_message(format_subtest_results(subtests, output_level))
try:
subtest_weights = get_subtest_weights(subtests)
feedback_obj.add_message(format_subtest_results(subtests, output_level))
except ValueError as e:
feedback_obj.add_message(str(e))

# If the test failed (in any phase), add the exception message to the feedback.
# A test whose subtests failed is also marked failed, but without exception info.
Expand Down Expand Up @@ -1173,13 +1210,18 @@ def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> Iter
# For now, we raise an error for unexpected outcomes
raise ValueError(f"Unexpected outcome '{outcome}' for test '{nodeid}'.")

# A test with subtests gets the fraction of them that passed, and skipped subtests count
# as failed. pytest marks the test failed when any subtest fails, so check whether the
# test body itself raised instead. If it did, the score from above (0 or set_score) stays.
# A test with subtests gets the weighted fraction of them that passed, and skipped
# subtests count as failed. pytest marks the test failed when any subtest fails, so check
# whether the test body itself raised instead. If it did, the score from above (0 or
# set_score) stays. A test with an invalid weight gets 0.
if subtests and report.when == "call" and call.excinfo is None and not feedback_obj.final_score_override:
passed = sum(subtest.outcome == "passed" for subtest in subtests)
res_obj["points_frac"] = passed / len(subtests)
res_obj["outcome"] = "passed" if passed == len(subtests) else "failed"
if subtest_weights is None:
res_obj["points_frac"] = 0.0
res_obj["outcome"] = "failed"
else:
passed_weight = sum(w for subtest, w in zip(subtests, subtest_weights, strict=True) if subtest.outcome == "passed")
res_obj["points_frac"] = passed_weight / sum(subtest_weights)
res_obj["outcome"] = "passed" if all(subtest.outcome == "passed" for subtest in subtests) else "failed"

res_obj["points"] = res_obj["points_frac"] * res_obj["max_points"]
final_results.append(res_obj)
Expand Down
48 changes: 48 additions & 0 deletions tests/test_subtests.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,37 @@ def test_case(sandbox, subtests):
["first: nope", "second:\nChecking: double(3)"],
id="labels",
),
pytest.param(
"""
def test_case(sandbox, subtests):
for x, weight in [(1, 1), (2, 2), (3, 1)]:
with subtests.test(weight=weight):
assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x)
""",
3 / 4,
["2/3 checks passed.", "Checking: double(3)"],
id="weights",
),
pytest.param(
"""
def test_case(sandbox, subtests):
with subtests.test(weight="heavy"):
assert_fn_equal(sandbox, "double", args=(1,), expected=2)
""",
0.0,
["A subtest's weight must be a non-negative int or float, not"],
id="invalid_weight",
),
pytest.param(
"""
def test_case(sandbox, subtests):
with subtests.test(weight=0):
assert_fn_equal(sandbox, "double", args=(1,), expected=2)
""",
0.0,
["At least one subtest needs a weight greater than 0."],
id="all_zero_weights",
),
],
)
def test_subtests_score(pytester: pytest.Pytester, test_case: str, points_frac: float, messages: list[str]) -> None:
Expand Down Expand Up @@ -226,3 +257,20 @@ def test_case(sandbox, subtests, monkeypatch):
assert result["points_frac"] == pytest.approx(0.25)
assert "double(4) timed out after 0.2 seconds." in result["message"]
assert "The remaining 2 checks didn't run because double(4) timed out after 0.2 seconds." in result["message"]


def test_zero_weight_failure_keeps_full_points(pytester: pytest.Pytester) -> None:
result = run_grader(
pytester,
"""
@pytest.mark.grading_data(name="case", points=1)
def test_case(sandbox, subtests):
for x, weight in [(1, 1), (3, 0)]:
with subtests.test(weight=weight):
assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x)
""",
)["case"]

assert result["points_frac"] == 1
assert result["outcome"] == "failed"
assert "Checking: double(3)" in result["message"]
Loading