Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions converting.md
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,9 @@ For questions where students submit a Jupyter notebook from a workspace:
- Legacy notebook questions grade only the cells that start with `#grade`. Keep that with
`notebook_cell_tag="#grade"`. Without it, every code cell runs, including plots and scratch work.
- Set `sandbox_timeout=15` if the notebook imports numpy or similar libraries.
- For a notebook in parts, mark each test with `@pytest.mark.notebook_stop_tag("# Part 2")` so
an error in a later part doesn't fail the tests for earlier parts. See
[Grading a Notebook in Parts](quick_start.md#grading-a-notebook-in-parts-with-notebook_stop_tag).
- Notebook questions use the normal `sandbox` fixture, not `workspace_mode`.
- The grader runs the graded cells as a script, so anything a graded cell needs must be defined in
a graded cell. Tag the cells with the notebook's imports `#grade`. Use `leading_code.py` only
Expand Down
26 changes: 26 additions & 0 deletions quick_start.md
Original file line number Diff line number Diff line change
Expand Up @@ -1646,6 +1646,32 @@ never reaches the sandbox.
> `notebook_cell_tag=None` (the default) includes **all** code cells. Use this when you control
> the notebook template and don't need students to tag their answers.

### Grading a Notebook in Parts with `notebook_stop_tag`

Student code runs when the sandbox starts, so an exception in any cell fails every test. For a
notebook with several parts, the `notebook_stop_tag` marker gives a test only the cells before the
first code cell whose first non-empty line starts with the tag. An unfinished Part 3 then can't
cost the student the points for Parts 1 and 2:

```python
@pytest.mark.notebook_stop_tag("# Part 2")
def test_part_1(sandbox: StudentFixture) -> None:
assert sandbox.query_function("double", 4) == 8


@pytest.mark.notebook_stop_tag("# Part 3")
def test_part_2(sandbox: StudentFixture) -> None:
assert sandbox.query_function("triple", 4) == 12
```

- Each test still gets every cell before its stop cell, so Part 2 can use Part 1's functions.
- The marker works on a test, a class, or a whole module with `pytestmark`. Use the `sandbox`
fixture, since `module_sandbox` starts one sandbox for every test in the module.
- It works with `notebook_cell_tag`: the stop cell is found among all code cells, and only tagged
cells before it run.
- Matching is by prefix, like `notebook_cell_tag`, so `# Part 1` also matches `# Part 10`.
- If a student deletes the stop cell's tag, the test runs every cell.

### `info.json` for a JupyterLab Notebook Question

```json
Expand Down
24 changes: 18 additions & 6 deletions src/pytest_prairielearn_grader/fixture.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,24 @@ def __init__(self, timed_out_call: str) -> None:
self.timed_out_call = timed_out_call


def _extract_notebook_code(notebook_path: Path, cell_tag: str | None = None) -> str:
def _starts_with_tag(source: str, tag: str) -> bool:
"""Whether the first non-empty line of a cell's source starts with the tag."""
lines = source.strip().splitlines()
return bool(lines) and lines[0].strip().startswith(tag)


def _extract_notebook_code(notebook_path: Path, cell_tag: str | None = None, stop_tag: str | None = None) -> str:
"""Extract Python source code from a Jupyter notebook (.ipynb) file.

Args:
notebook_path: Path to the ``.ipynb`` file.
cell_tag: When provided, only code cells whose first non-empty line
starts with this string are included. When ``None`` all code cells
are included.
stop_tag: When provided, extraction stops at the first code cell whose
first non-empty line starts with this string, so that cell and every
cell after it are left out. When the notebook has no such cell, every
cell is considered.

Returns:
A single Python source string containing all selected cell contents
Expand All @@ -97,10 +107,10 @@ def _extract_notebook_code(notebook_path: Path, cell_tag: str | None = None) ->
if cell.cell_type != "code":
continue
source: str = cell.source
if cell_tag is not None:
lines = source.strip().splitlines()
if not lines or not lines[0].strip().startswith(cell_tag):
continue
if stop_tag is not None and _starts_with_tag(source, stop_tag):
break
if cell_tag is not None and not _starts_with_tag(source, cell_tag):
continue
code_parts.append(source)

return "\n\n".join(code_parts)
Expand Down Expand Up @@ -465,6 +475,7 @@ def __init__(
names_for_user_list: list[str] | None,
worker_username: str | None,
notebook_cell_tag: str | None = None,
notebook_stop_tag: str | None = None,
helper_modules: dict[str, str] | None = None,
data_files: dict[str, Path] | None = None,
) -> None:
Expand All @@ -476,6 +487,7 @@ def __init__(
self.student_code_file = file_names.student_code_file
self.setup_code_file = file_names.setup_code_file
self.notebook_cell_tag = notebook_cell_tag
self.notebook_stop_tag = notebook_stop_tag
self.helper_modules = helper_modules

def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_TIMEOUT) -> ProcessStartResponse:
Expand All @@ -492,7 +504,7 @@ def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_T
student_code += os.linesep
if self.student_code_file.is_file():
if self.student_code_file.suffix.lower() == ".ipynb":
student_code += _extract_notebook_code(self.student_code_file, self.notebook_cell_tag)
student_code += _extract_notebook_code(self.student_code_file, self.notebook_cell_tag, self.notebook_stop_tag)
else:
student_code += self.student_code_file.read_text(encoding="utf-8")
if self.trailing_file.is_file():
Expand Down
15 changes: 15 additions & 0 deletions src/pytest_prairielearn_grader/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,17 @@ def _initialize_sandbox_fixture(

config_starting_vars = {}

# Read from the test, its class or the module (pytestmark), so module_sandbox can use it too.
notebook_stop_tag: str | None = None
stop_marker = request.node.get_closest_marker("notebook_stop_tag")
if stop_marker is not None:
if len(stop_marker.args) != 1 or not isinstance(stop_marker.args[0], str) or not stop_marker.args[0].strip():
pytest.fail(
f"notebook_stop_tag takes one non-empty string, the tag that starts the first cell to leave out, got: {stop_marker.args!r}",
pytrace=False,
)
notebook_stop_tag = stop_marker.args[0]

# Handle names_for_user before merging - ConfigObject has list[str], data.json has list[dict]
names_for_user_list: list[str] | None = None
if "names_for_user" in config_dict:
Expand Down Expand Up @@ -396,6 +407,7 @@ def _initialize_sandbox_fixture(
names_for_user_list=names_for_user_list,
worker_username=request.config.getoption("--worker-username"),
notebook_cell_tag=notebook_cell_tag,
notebook_stop_tag=notebook_stop_tag,
helper_modules=_load_helper_modules(request, helper_module_names, import_blacklist),
data_files=_resolve_data_files(request, data_file_names),
)
Expand Down Expand Up @@ -975,6 +987,9 @@ def pytest_configure(config: Config) -> None:
# Add a marker for the sandbox fixture to set the initialization timeout
config.addinivalue_line("markers", "sandbox_timeout(timeout_value): sets a timeout for initialization of the sandbox fixture")
config.addinivalue_line("markers", "output(level): controls the detail level of grading failure output")
config.addinivalue_line(
"markers", "notebook_stop_tag(tag): run only the notebook cells before the first code cell that starts with tag"
)

# Every question's tests are in a file named test_student.py. pytest's default import mode
# imports each one as the module `test_student`, so a run over several questions stops with an
Expand Down
Empty file.
29 changes: 29 additions & 0 deletions tests/scenario_root/test_notebook_stop_tag/expected_outcome.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{
"expected_data_object": {
"score": 0.6666666666666666,
"tests": [
{
"test_id": "test_notebook_stop_tag.py::test_part_1[student_code]",
"outcome": "passed",
"max_points": 1,
"points": 1.0,
"points_frac": 1.0
},
{
"test_id": "test_notebook_stop_tag.py::TestPart2::test_part_2[student_code]",
"outcome": "passed",
"max_points": 1,
"points": 1.0,
"points_frac": 1.0
},
{
"test_id": "test_notebook_stop_tag.py::test_whole_notebook[student_code]",
"message": "Part 3 is unfinished",
"outcome": "error",
"max_points": 1,
"points": 0.0,
"points_frac": 0.0
}
]
}
}
35 changes: 35 additions & 0 deletions tests/scenario_root/test_notebook_stop_tag/scenario.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
"""
Notebook stop-tag scenario.

The notebook has three parts, and Part 3 raises ``ValueError``. Tests marked with
``notebook_stop_tag`` run only the cells before the cell that starts with the tag, so
the Part 1 and Part 2 tests still pass. The unmarked test runs every cell and fails.
"""

import pytest

from pytest_prairielearn_grader import ConfigObject
from pytest_prairielearn_grader.fixture import StudentFixture

autograder_config = ConfigObject(
student_code_pattern="student_code*.ipynb",
sandbox_timeout=5.0,
)


@pytest.mark.notebook_stop_tag("# Part 2")
@pytest.mark.grading_data(name="Part 1", points=1)
def test_part_1(sandbox: StudentFixture) -> None:
assert sandbox.query_function("double", 4) == 8


@pytest.mark.notebook_stop_tag("# Part 3")
class TestPart2:
@pytest.mark.grading_data(name="Part 2", points=1)
def test_part_2(self, sandbox: StudentFixture) -> None:
assert sandbox.query_function("triple", 4) == 12


@pytest.mark.grading_data(name="Whole notebook", points=1)
def test_whole_notebook(sandbox: StudentFixture) -> None:
assert sandbox.query_function("double", 4) == 8
57 changes: 57 additions & 0 deletions tests/scenario_root/test_notebook_stop_tag/student_code.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Homework\n",
"Start each part's first cell with its `# Part` comment."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Part 1\n",
"def double(x):\n",
" return 2 * x"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Part 2\n",
"def triple(x):\n",
" return 3 * x"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Part 3\n",
"raise ValueError('Part 3 is unfinished')"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python",
"version": "3.11.0"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
32 changes: 32 additions & 0 deletions tests/test_notebook_extraction.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import json
from pathlib import Path

import pytest

from pytest_prairielearn_grader.fixture import _extract_notebook_code

CELLS = ["#grade\na = 1", "scratch = 0", "#grade part 2\nb = 2", "#grade\nc = 3"]


@pytest.fixture
def notebook(tmp_path: Path) -> Path:
path = tmp_path / "student_code.ipynb"
cells: list[dict[str, object]] = [
{"cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": source} for source in CELLS
]
path.write_text(json.dumps({"cells": cells, "metadata": {}, "nbformat": 4, "nbformat_minor": 4}))
return path


@pytest.mark.parametrize(
("cell_tag", "stop_tag", "expected"),
[
(None, None, CELLS),
(None, "#grade part 2", CELLS[:2]),
(None, "#part 9", CELLS),
("#grade", "#grade part 2", CELLS[:1]),
("#grade", "scratch", CELLS[:1]),
],
)
def test_stop_tag(notebook: Path, cell_tag: str | None, stop_tag: str | None, expected: list[str]) -> None:
assert _extract_notebook_code(notebook, cell_tag, stop_tag) == "\n\n".join(expected)
Loading