diff --git a/quick_start.md b/quick_start.md index f0270ea..220051f 100644 --- a/quick_start.md +++ b/quick_start.md @@ -567,6 +567,18 @@ test's score is the fraction of blocks that passed: `feedback.set_score`. - `feedback.set_score_final` replaces the fraction. +To make some blocks count for more, pass a `weight`, which is 1 by default: + +```python +for n, weight in [(2, 1), (10, 1), (29, 2)]: + with subtests.test(weight=weight): + assert_fn_equal(sandbox, "fib", args=(n,), expected=fib(n)) +``` + +The score is then the weight of the blocks that passed divided by the total weight: here, passing +only `fib(29)` gives half the points. Weights are non-negative ints or floats. A block with weight 0 +is still checked and shown, but doesn't change the score. + The test's message starts with how many blocks passed, such as `7/10 checks passed.`, followed by the messages of the first 5 failed blocks. Passed blocks aren't listed. Pass a label to show it before a block's message, such as `subtests.test(f"last_element({lst})")`. The `assert_fn_*` diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 969d94e..07c0c13 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -1,5 +1,6 @@ import json import logging +import math import os import sys from collections.abc import Iterable @@ -68,6 +69,32 @@ class SubtestResult: label: str | None outcome: str excinfo: pytest.ExceptionInfo[BaseException] | None + # The `weight` keyword argument. Newer pytest versions keep its repr, such as "2", instead. + weight: object = None + + +def get_subtest_weights(subtests: list[SubtestResult]) -> list[float]: + """ + Each subtest's weight from `subtests.test(weight=...)`, or 1 without one. Raises ValueError, + with a message for the instructor, if a weight isn't a non-negative number or they're all 0. + """ + weights = [] + for subtest in subtests: + if subtest.weight is None: + weights.append(1.0) + continue + weight = math.nan + if isinstance(subtest.weight, int | float | str) and not isinstance(subtest.weight, bool): + try: + weight = float(subtest.weight) + except ValueError: + pass + if not math.isfinite(weight) or weight < 0: + raise ValueError(f"A subtest's weight must be a non-negative int or float, not {subtest.weight}.") + weights.append(weight) + if sum(weights) == 0: + raise ValueError("At least one subtest needs a weight greater than 0.") + return weights def format_failure( @@ -1048,7 +1075,12 @@ def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: result = self.pending_calls.pop(report.nodeid) if isinstance(report, pytest.SubtestReport): self.subtest_results.setdefault(report.nodeid, []).append( - SubtestResult(label=report.context.msg, outcome=report.outcome, excinfo=result.call.excinfo) + SubtestResult( + label=report.context.msg, + outcome=report.outcome, + excinfo=result.call.excinfo, + weight=report.context.kwargs.get("weight"), + ) ) else: self.collected_results[report.nodeid] = result @@ -1125,8 +1157,13 @@ def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> Iter logger.debug(f"Grading output level set to: {output_level}") subtests = self.subtest_results.get(nodeid, []) + subtest_weights: list[float] | None = None if subtests: - feedback_obj.add_message(format_subtest_results(subtests, output_level)) + try: + subtest_weights = get_subtest_weights(subtests) + feedback_obj.add_message(format_subtest_results(subtests, output_level)) + except ValueError as e: + feedback_obj.add_message(str(e)) # If the test failed (in any phase), add the exception message to the feedback. # A test whose subtests failed is also marked failed, but without exception info. @@ -1173,13 +1210,18 @@ def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> Iter # For now, we raise an error for unexpected outcomes raise ValueError(f"Unexpected outcome '{outcome}' for test '{nodeid}'.") - # A test with subtests gets the fraction of them that passed, and skipped subtests count - # as failed. pytest marks the test failed when any subtest fails, so check whether the - # test body itself raised instead. If it did, the score from above (0 or set_score) stays. + # A test with subtests gets the weighted fraction of them that passed, and skipped + # subtests count as failed. pytest marks the test failed when any subtest fails, so check + # whether the test body itself raised instead. If it did, the score from above (0 or + # set_score) stays. A test with an invalid weight gets 0. if subtests and report.when == "call" and call.excinfo is None and not feedback_obj.final_score_override: - passed = sum(subtest.outcome == "passed" for subtest in subtests) - res_obj["points_frac"] = passed / len(subtests) - res_obj["outcome"] = "passed" if passed == len(subtests) else "failed" + if subtest_weights is None: + res_obj["points_frac"] = 0.0 + res_obj["outcome"] = "failed" + else: + passed_weight = sum(w for subtest, w in zip(subtests, subtest_weights, strict=True) if subtest.outcome == "passed") + res_obj["points_frac"] = passed_weight / sum(subtest_weights) + res_obj["outcome"] = "passed" if all(subtest.outcome == "passed" for subtest in subtests) else "failed" res_obj["points"] = res_obj["points_frac"] * res_obj["max_points"] final_results.append(res_obj) diff --git a/tests/test_subtests.py b/tests/test_subtests.py index b8b46f7..a6d4462 100644 --- a/tests/test_subtests.py +++ b/tests/test_subtests.py @@ -147,6 +147,37 @@ def test_case(sandbox, subtests): ["first: nope", "second:\nChecking: double(3)"], id="labels", ), + pytest.param( + """ + def test_case(sandbox, subtests): + for x, weight in [(1, 1), (2, 2), (3, 1)]: + with subtests.test(weight=weight): + assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x) + """, + 3 / 4, + ["2/3 checks passed.", "Checking: double(3)"], + id="weights", + ), + pytest.param( + """ + def test_case(sandbox, subtests): + with subtests.test(weight="heavy"): + assert_fn_equal(sandbox, "double", args=(1,), expected=2) + """, + 0.0, + ["A subtest's weight must be a non-negative int or float, not"], + id="invalid_weight", + ), + pytest.param( + """ + def test_case(sandbox, subtests): + with subtests.test(weight=0): + assert_fn_equal(sandbox, "double", args=(1,), expected=2) + """, + 0.0, + ["At least one subtest needs a weight greater than 0."], + id="all_zero_weights", + ), ], ) def test_subtests_score(pytester: pytest.Pytester, test_case: str, points_frac: float, messages: list[str]) -> None: @@ -226,3 +257,20 @@ def test_case(sandbox, subtests, monkeypatch): assert result["points_frac"] == pytest.approx(0.25) assert "double(4) timed out after 0.2 seconds." in result["message"] assert "The remaining 2 checks didn't run because double(4) timed out after 0.2 seconds." in result["message"] + + +def test_zero_weight_failure_keeps_full_points(pytester: pytest.Pytester) -> None: + result = run_grader( + pytester, + """ + @pytest.mark.grading_data(name="case", points=1) + def test_case(sandbox, subtests): + for x, weight in [(1, 1), (3, 0)]: + with subtests.test(weight=weight): + assert_fn_equal(sandbox, "double", args=(x,), expected=2 * x) + """, + )["case"] + + assert result["points_frac"] == 1 + assert result["outcome"] == "failed" + assert "Checking: double(3)" in result["message"]