diff --git a/converting.md b/converting.md index 2b0035e..1ec1616 100644 --- a/converting.md +++ b/converting.md @@ -339,6 +339,9 @@ For questions where students submit a Jupyter notebook from a workspace: - Legacy notebook questions grade only the cells that start with `#grade`. Keep that with `notebook_cell_tag="#grade"`. Without it, every code cell runs, including plots and scratch work. - Set `sandbox_timeout=15` if the notebook imports numpy or similar libraries. +- For a notebook in parts, mark each test with `@pytest.mark.notebook_stop_tag("# Part 2")` so + an error in a later part doesn't fail the tests for earlier parts. See + [Grading a Notebook in Parts](quick_start.md#grading-a-notebook-in-parts-with-notebook_stop_tag). - Notebook questions use the normal `sandbox` fixture, not `workspace_mode`. - The grader runs the graded cells as a script, so anything a graded cell needs must be defined in a graded cell. Tag the cells with the notebook's imports `#grade`. Use `leading_code.py` only diff --git a/quick_start.md b/quick_start.md index 7491d70..86594fa 100644 --- a/quick_start.md +++ b/quick_start.md @@ -1646,6 +1646,32 @@ never reaches the sandbox. > `notebook_cell_tag=None` (the default) includes **all** code cells. Use this when you control > the notebook template and don't need students to tag their answers. +### Grading a Notebook in Parts with `notebook_stop_tag` + +Student code runs when the sandbox starts, so an exception in any cell fails every test. For a +notebook with several parts, the `notebook_stop_tag` marker gives a test only the cells before the +first code cell whose first non-empty line starts with the tag. An unfinished Part 3 then can't +cost the student the points for Parts 1 and 2: + +```python +@pytest.mark.notebook_stop_tag("# Part 2") +def test_part_1(sandbox: StudentFixture) -> None: + assert sandbox.query_function("double", 4) == 8 + + +@pytest.mark.notebook_stop_tag("# Part 3") +def test_part_2(sandbox: StudentFixture) -> None: + assert sandbox.query_function("triple", 4) == 12 +``` + +- Each test still gets every cell before its stop cell, so Part 2 can use Part 1's functions. +- The marker works on a test, a class, or a whole module with `pytestmark`. Use the `sandbox` + fixture, since `module_sandbox` starts one sandbox for every test in the module. +- It works with `notebook_cell_tag`: the stop cell is found among all code cells, and only tagged + cells before it run. +- Matching is by prefix, like `notebook_cell_tag`, so `# Part 1` also matches `# Part 10`. +- If a student deletes the stop cell's tag, the test runs every cell. + ### `info.json` for a JupyterLab Notebook Question ```json diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 9b3e015..69d653e 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -66,7 +66,13 @@ def __init__(self, timed_out_call: str) -> None: self.timed_out_call = timed_out_call -def _extract_notebook_code(notebook_path: Path, cell_tag: str | None = None) -> str: +def _starts_with_tag(source: str, tag: str) -> bool: + """Whether the first non-empty line of a cell's source starts with the tag.""" + lines = source.strip().splitlines() + return bool(lines) and lines[0].strip().startswith(tag) + + +def _extract_notebook_code(notebook_path: Path, cell_tag: str | None = None, stop_tag: str | None = None) -> str: """Extract Python source code from a Jupyter notebook (.ipynb) file. Args: @@ -74,6 +80,10 @@ def _extract_notebook_code(notebook_path: Path, cell_tag: str | None = None) -> cell_tag: When provided, only code cells whose first non-empty line starts with this string are included. When ``None`` all code cells are included. + stop_tag: When provided, extraction stops at the first code cell whose + first non-empty line starts with this string, so that cell and every + cell after it are left out. When the notebook has no such cell, every + cell is considered. Returns: A single Python source string containing all selected cell contents @@ -97,10 +107,10 @@ def _extract_notebook_code(notebook_path: Path, cell_tag: str | None = None) -> if cell.cell_type != "code": continue source: str = cell.source - if cell_tag is not None: - lines = source.strip().splitlines() - if not lines or not lines[0].strip().startswith(cell_tag): - continue + if stop_tag is not None and _starts_with_tag(source, stop_tag): + break + if cell_tag is not None and not _starts_with_tag(source, cell_tag): + continue code_parts.append(source) return "\n\n".join(code_parts) @@ -465,6 +475,7 @@ def __init__( names_for_user_list: list[str] | None, worker_username: str | None, notebook_cell_tag: str | None = None, + notebook_stop_tag: str | None = None, helper_modules: dict[str, str] | None = None, data_files: dict[str, Path] | None = None, ) -> None: @@ -476,6 +487,7 @@ def __init__( self.student_code_file = file_names.student_code_file self.setup_code_file = file_names.setup_code_file self.notebook_cell_tag = notebook_cell_tag + self.notebook_stop_tag = notebook_stop_tag self.helper_modules = helper_modules def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_TIMEOUT) -> ProcessStartResponse: @@ -492,7 +504,7 @@ def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_T student_code += os.linesep if self.student_code_file.is_file(): if self.student_code_file.suffix.lower() == ".ipynb": - student_code += _extract_notebook_code(self.student_code_file, self.notebook_cell_tag) + student_code += _extract_notebook_code(self.student_code_file, self.notebook_cell_tag, self.notebook_stop_tag) else: student_code += self.student_code_file.read_text(encoding="utf-8") if self.trailing_file.is_file(): diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 6352b15..2c9f644 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -350,6 +350,17 @@ def _initialize_sandbox_fixture( config_starting_vars = {} + # Read from the test, its class or the module (pytestmark), so module_sandbox can use it too. + notebook_stop_tag: str | None = None + stop_marker = request.node.get_closest_marker("notebook_stop_tag") + if stop_marker is not None: + if len(stop_marker.args) != 1 or not isinstance(stop_marker.args[0], str) or not stop_marker.args[0].strip(): + pytest.fail( + f"notebook_stop_tag takes one non-empty string, the tag that starts the first cell to leave out, got: {stop_marker.args!r}", + pytrace=False, + ) + notebook_stop_tag = stop_marker.args[0] + # Handle names_for_user before merging - ConfigObject has list[str], data.json has list[dict] names_for_user_list: list[str] | None = None if "names_for_user" in config_dict: @@ -396,6 +407,7 @@ def _initialize_sandbox_fixture( names_for_user_list=names_for_user_list, worker_username=request.config.getoption("--worker-username"), notebook_cell_tag=notebook_cell_tag, + notebook_stop_tag=notebook_stop_tag, helper_modules=_load_helper_modules(request, helper_module_names, import_blacklist), data_files=_resolve_data_files(request, data_file_names), ) @@ -975,6 +987,9 @@ def pytest_configure(config: Config) -> None: # Add a marker for the sandbox fixture to set the initialization timeout config.addinivalue_line("markers", "sandbox_timeout(timeout_value): sets a timeout for initialization of the sandbox fixture") config.addinivalue_line("markers", "output(level): controls the detail level of grading failure output") + config.addinivalue_line( + "markers", "notebook_stop_tag(tag): run only the notebook cells before the first code cell that starts with tag" + ) # Every question's tests are in a file named test_student.py. pytest's default import mode # imports each one as the module `test_student`, so a run over several questions stops with an diff --git a/tests/scenario_root/test_notebook_stop_tag/__init__.py b/tests/scenario_root/test_notebook_stop_tag/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_notebook_stop_tag/expected_outcome.json b/tests/scenario_root/test_notebook_stop_tag/expected_outcome.json new file mode 100644 index 0000000..984b345 --- /dev/null +++ b/tests/scenario_root/test_notebook_stop_tag/expected_outcome.json @@ -0,0 +1,29 @@ +{ + "expected_data_object": { + "score": 0.6666666666666666, + "tests": [ + { + "test_id": "test_notebook_stop_tag.py::test_part_1[student_code]", + "outcome": "passed", + "max_points": 1, + "points": 1.0, + "points_frac": 1.0 + }, + { + "test_id": "test_notebook_stop_tag.py::TestPart2::test_part_2[student_code]", + "outcome": "passed", + "max_points": 1, + "points": 1.0, + "points_frac": 1.0 + }, + { + "test_id": "test_notebook_stop_tag.py::test_whole_notebook[student_code]", + "message": "Part 3 is unfinished", + "outcome": "error", + "max_points": 1, + "points": 0.0, + "points_frac": 0.0 + } + ] + } +} diff --git a/tests/scenario_root/test_notebook_stop_tag/scenario.py b/tests/scenario_root/test_notebook_stop_tag/scenario.py new file mode 100644 index 0000000..5df3535 --- /dev/null +++ b/tests/scenario_root/test_notebook_stop_tag/scenario.py @@ -0,0 +1,35 @@ +""" +Notebook stop-tag scenario. + +The notebook has three parts, and Part 3 raises ``ValueError``. Tests marked with +``notebook_stop_tag`` run only the cells before the cell that starts with the tag, so +the Part 1 and Part 2 tests still pass. The unmarked test runs every cell and fails. +""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +autograder_config = ConfigObject( + student_code_pattern="student_code*.ipynb", + sandbox_timeout=5.0, +) + + +@pytest.mark.notebook_stop_tag("# Part 2") +@pytest.mark.grading_data(name="Part 1", points=1) +def test_part_1(sandbox: StudentFixture) -> None: + assert sandbox.query_function("double", 4) == 8 + + +@pytest.mark.notebook_stop_tag("# Part 3") +class TestPart2: + @pytest.mark.grading_data(name="Part 2", points=1) + def test_part_2(self, sandbox: StudentFixture) -> None: + assert sandbox.query_function("triple", 4) == 12 + + +@pytest.mark.grading_data(name="Whole notebook", points=1) +def test_whole_notebook(sandbox: StudentFixture) -> None: + assert sandbox.query_function("double", 4) == 8 diff --git a/tests/scenario_root/test_notebook_stop_tag/student_code.ipynb b/tests/scenario_root/test_notebook_stop_tag/student_code.ipynb new file mode 100644 index 0000000..2d2239e --- /dev/null +++ b/tests/scenario_root/test_notebook_stop_tag/student_code.ipynb @@ -0,0 +1,57 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Homework\n", + "Start each part's first cell with its `# Part` comment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Part 1\n", + "def double(x):\n", + " return 2 * x" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Part 2\n", + "def triple(x):\n", + " return 3 * x" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Part 3\n", + "raise ValueError('Part 3 is unfinished')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/test_notebook_extraction.py b/tests/test_notebook_extraction.py new file mode 100644 index 0000000..c5bc462 --- /dev/null +++ b/tests/test_notebook_extraction.py @@ -0,0 +1,32 @@ +import json +from pathlib import Path + +import pytest + +from pytest_prairielearn_grader.fixture import _extract_notebook_code + +CELLS = ["#grade\na = 1", "scratch = 0", "#grade part 2\nb = 2", "#grade\nc = 3"] + + +@pytest.fixture +def notebook(tmp_path: Path) -> Path: + path = tmp_path / "student_code.ipynb" + cells: list[dict[str, object]] = [ + {"cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": source} for source in CELLS + ] + path.write_text(json.dumps({"cells": cells, "metadata": {}, "nbformat": 4, "nbformat_minor": 4})) + return path + + +@pytest.mark.parametrize( + ("cell_tag", "stop_tag", "expected"), + [ + (None, None, CELLS), + (None, "#grade part 2", CELLS[:2]), + (None, "#part 9", CELLS), + ("#grade", "#grade part 2", CELLS[:1]), + ("#grade", "scratch", CELLS[:1]), + ], +) +def test_stop_tag(notebook: Path, cell_tag: str | None, stop_tag: str | None, expected: list[str]) -> None: + assert _extract_notebook_code(notebook, cell_tag, stop_tag) == "\n\n".join(expected)