From 54ae3c6f5ab9eba5c570b27d7d78b0b9871c638e Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Fri, 13 Nov 2026 17:23:29 -0500 Subject: [PATCH 1/2] Support text, annotations, axhline and axvline in plots Plain text and annotations in data or axes coordinates, and lines from axhline and axvline, are now sent to the test instead of rejecting the whole plot. assert_plot_equal compares the new lines with the other data, and a new check_texts option compares text and annotations. Extra text is ignored unless check_texts is set. Co-Authored-By: Claude Opus 5.5 --- quick_start.md | 32 ++-- src/pytest_prairielearn_grader/assertions.py | 14 +- .../matplotlib_json.py | 166 +++++++++++++++++- src/pytest_prairielearn_grader/plot_checks.py | 112 +++++++++++- tests/test_matplotlib_json.py | 90 ++++++++-- tests/test_plot_assertions.py | 68 ++++++- 6 files changed, 445 insertions(+), 37 deletions(-) diff --git a/quick_start.md b/quick_start.md index bb14f43..c3df74a 100644 --- a/quick_start.md +++ b/quick_start.md @@ -627,14 +627,17 @@ An `Axes` value brings its whole figure, so subplots are never lost: use `figure What's sent to the test: - **Lines**, with their data, label, color, width, style, marker and draw style. This includes `plot`, `semilogy`, `loglog` and `step`. +- **Horizontal and vertical lines** from `axhline` and `axvline`, with their position, extent, label, color, width and style. +- **Text** from `text`, in data or axes coordinates, with its position, alignment and rotation. +- **Annotations** from `annotate`, with the annotated point in data or axes coordinates, the text position in data or axes coordinates or offset points, and whether it has an arrow. - **Scatter points**, with their positions, sizes and colors. - **Bars and histograms**, with their positions, sizes, colors and orientation. - **Axes details:** scales, labels, titles, limits, position and legend labels. Twin axes come back as separate axes in `figure.axes`. - **Figure details:** size and overall title. -Other plot content makes the query fail with a message saying what isn't supported, rather than losing data. This includes `errorbar`, `stem`, `fill_between`, `pie`, `imshow`, `contour`, `vlines` and `hlines`, `axhline` and `axvline`, `axvspan`, text and annotations, colormapped scatter plots, polar axes, inset axes and subfigures. +Other plot content makes the query fail with a message saying what isn't supported, rather than losing data. This includes `errorbar`, `stem`, `fill_between`, `pie`, `imshow`, `contour`, `vlines` and `hlines`, `axline`, `axvspan`, text in figure coordinates, colormapped scatter plots, polar axes, inset axes and subfigures. -Some details aren't sent, so don't grade them: grid lines, tick positions and labels, line and marker transparency set separately from color, scatter marker shapes, the base of a log scale, and `symlog` settings. Values are sent as decimal numbers, and dates come back as matplotlib date numbers (use `matplotlib.dates.num2date` to convert them). Categories on a line come back as strings. +Some details aren't sent, so don't grade them: grid lines, tick positions and labels, line and marker transparency set separately from color, scatter marker shapes, the base of a log scale, `symlog` settings, text fonts and colors, and arrow styles, since every annotation arrow comes back as a plain arrow. Values are sent as decimal numbers, and dates come back as matplotlib date numbers (use `matplotlib.dates.num2date` to convert them). Categories on a line come back as strings. Importing matplotlib can take most of the default 1-second sandbox timeout, so set a longer one in plot questions, such as `sandbox_timeout = 15` at the top of the test file. @@ -672,24 +675,25 @@ By default, it checks: - **Subplots:** the number of axes. - **Scales:** the x and y axis scales, such as linear or log. -- **Data:** the lines, scatter plots and bar plots, which must have the same number of each kind. Data is compared with `numpy.allclose`, using `rtol` and `atol`. +- **Data:** the lines, `axhline` and `axvline` lines, scatter plots and bar plots, which must have the same number of each kind. Data is compared with `numpy.allclose`, using `rtol` and `atol`. The series can be in any order, and a line also matches when its points are in reverse order, since it looks the same. The points in a scatter plot and the bars in a bar plot can be in any order. Keyword arguments change what's checked: -| Argument | Default | Checks | -| -------------- | ------- | ----------------------------------------------------------------------------------------------------------- | -| `check_scales` | `True` | The x and y axis scales. | -| `check_data` | `True` | The data in lines, scatter plots and bar plots. | -| `ordered` | `False` | Requires lines, scatter plots, bar plots and legend labels to be in the reference's order. | -| `allow_extra` | `False` | Allows more lines, scatter plots or bar plots than the reference, such as a student's extra reference line. | -| `check_labels` | `False` | The x and y axis labels, exactly. | -| `check_titles` | `False` | The title of each subplot and the figure, exactly. | -| `check_legend` | `False` | The legend's labels and title, or that neither plot has a legend. | -| `check_limits` | `False` | The x and y axis limits, within `rtol` and `atol`. | -| `check_styles` | `False` | Line colors, styles and markers, scatter point colors, and bar colors. | +| Argument | Default | Checks | +| -------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `check_scales` | `True` | The x and y axis scales. | +| `check_data` | `True` | The data in lines, `axhline` and `axvline` lines, scatter plots and bar plots. | +| `ordered` | `False` | Requires lines, scatter plots, bar plots and legend labels to be in the reference's order. | +| `allow_extra` | `False` | Allows more lines, scatter plots, bar plots, text or annotations than the reference, such as a student's extra reference line. | +| `check_labels` | `False` | The x and y axis labels, exactly. | +| `check_titles` | `False` | The title of each subplot and the figure, exactly. | +| `check_legend` | `False` | The legend's labels and title, or that neither plot has a legend. | +| `check_limits` | `False` | The x and y axis limits, within `rtol` and `atol`. | +| `check_styles` | `False` | Line colors, styles and markers, scatter point colors, and bar colors. | +| `check_texts` | `False` | The strings of `text` and `annotate` labels, and the point each annotation points at. The position of plain text isn't compared. | A failure describes the first difference found, such as `Expected a log scale on the y-axis of your plot, but found a linear scale.` or `No line in your plot matches the line labeled 'exp'.` Lines diff --git a/src/pytest_prairielearn_grader/assertions.py b/src/pytest_prairielearn_grader/assertions.py index 6b98aae..ea442da 100644 --- a/src/pytest_prairielearn_grader/assertions.py +++ b/src/pytest_prairielearn_grader/assertions.py @@ -204,6 +204,7 @@ def assert_plot_equal( check_legend: bool = False, check_limits: bool = False, check_styles: bool = False, + check_texts: bool = False, msg: str | None = None, ) -> None: """Assert that a student's matplotlib plot matches a reference plot. @@ -213,8 +214,8 @@ def assert_plot_equal( test itself. Subplots are compared in order. By default this checks the number of subplots, the axis scales, and the - data in each line, scatter plot and bar plot, which must have the same - number of each kind. Series can be in any order, and a line also matches + data in each line, ``axhline`` and ``axvline`` line, scatter plot and bar + plot, which must have the same number of each kind. Series can be in any order, and a line also matches when its points are in reverse order. Scatter points and bars can be in any order. Numbers are compared with ``numpy.allclose``. @@ -237,8 +238,8 @@ def assert_plot_equal( Require lines, scatter plots, bar plots and legend labels to be in the same order as in the reference. allow_extra: - Allow the student's plot to have more lines, scatter plots or bar - plots than the reference. + Allow the student's plot to have more lines, scatter plots, bar + plots, text or annotations than the reference. check_labels: Compare the x and y axis labels exactly. check_titles: @@ -251,6 +252,10 @@ def assert_plot_equal( check_styles: Compare line colors, line styles and markers, scatter point colors, and bar colors and orientation. + check_texts: + Compare the strings of ``text`` and ``annotate`` labels, and the point + each annotation points at. The position of plain text isn't compared. + Extra text is allowed with *allow_extra*. msg: Optional message to show instead of the description of the difference. @@ -280,6 +285,7 @@ def assert_plot_equal( check_legend=check_legend, check_limits=check_limits, check_styles=check_styles, + check_texts=check_texts, ) if mismatch is not None: raise AssertionError(msg or mismatch) diff --git a/src/pytest_prairielearn_grader/matplotlib_json.py b/src/pytest_prairielearn_grader/matplotlib_json.py index 624c2a5..067b530 100644 --- a/src/pytest_prairielearn_grader/matplotlib_json.py +++ b/src/pytest_prairielearn_grader/matplotlib_json.py @@ -22,6 +22,8 @@ from matplotlib.figure import Figure from matplotlib.lines import Line2D from matplotlib.patches import Rectangle +from matplotlib.text import Annotation +from matplotlib.text import Text MATPLOTLIB_FIGURE_TYPE = "matplotlib_figure" @@ -30,6 +32,13 @@ _DRAWSTYLES = frozenset({"default", "steps", "steps-pre", "steps-mid", "steps-post"}) _ORIENTATIONS = frozenset({"vertical", "horizontal"}) _AXIS_UNITS = frozenset({"number", "date", "category"}) +_TEXT_COORDINATES = frozenset({"data", "axes"}) +_ANNOTATION_TEXT_COORDINATES = frozenset({"data", "axes", "offset points"}) +_ORIENTATIONS_FOR_LINES = frozenset({"horizontal", "vertical"}) +# matplotlib names for the coordinate systems that annotations can use. +_ANNOTATION_COORDINATE_NAMES = {"data": "data", "axes fraction": "axes", "offset points": "offset points"} +_HORIZONTAL_ALIGNMENTS = frozenset({"left", "center", "right"}) +_VERTICAL_ALIGNMENTS = frozenset({"top", "bottom", "center", "baseline", "center_baseline"}) _MAX_FIGURE_SIZE_INCHES = 1000.0 @@ -86,8 +95,13 @@ def _check_axes(axes: Any) -> None: raise UnsupportedPlotError("inset axes aren't supported") if axes.images: raise UnsupportedPlotError("images aren't supported") - if axes.texts: - raise UnsupportedPlotError("text and annotations aren't supported") + for text in axes.texts: + if type(text) is Annotation: + _annotation_coordinates(text) + elif type(text) is Text: + _text_coordinates(axes, text) + else: + raise UnsupportedPlotError(f"{type(text).__name__} text isn't supported") if axes.tables: raise UnsupportedPlotError("tables aren't supported") if axes.artists: @@ -123,8 +137,8 @@ def _check_axes(axes: Any) -> None: for line in axes.lines: if type(line) is not Line2D: raise UnsupportedPlotError(f"{type(line).__name__} lines aren't supported") - if line.get_transform() is not axes.transData: - raise UnsupportedPlotError("lines that don't use data coordinates, such as axhline and axvline, aren't supported") + if line.get_transform() is not axes.transData and reference_line_orientation(axes, line) is None: + raise UnsupportedPlotError("lines that don't use data coordinates, other than axhline and axvline, aren't supported") if not isinstance(line.get_marker(), str | None): raise UnsupportedPlotError("custom line markers aren't supported") @@ -150,6 +164,43 @@ def _axis_to_json(axis: Any) -> dict[str, Any]: return {"scale": scale, "label": axis.get_label_text(), "units": units, "categories": categories} +def _text_coordinates(axes: Any, text: Any) -> str: + # Compare by identity, as for scatter offsets. + transform = text.get_transform() + if transform is axes.transData: + return "data" + if transform is axes.transAxes: + return "axes" + raise UnsupportedPlotError("text that doesn't use data or axes coordinates isn't supported") + + +def _annotation_coordinates(annotation: Any) -> tuple[str, str]: + point = _ANNOTATION_COORDINATE_NAMES.get(annotation.xycoords) if isinstance(annotation.xycoords, str) else None + text = _ANNOTATION_COORDINATE_NAMES.get(annotation.anncoords) if isinstance(annotation.anncoords, str) else None + if point not in _TEXT_COORDINATES or text is None: + raise UnsupportedPlotError( + "annotations are supported only with xycoords 'data' or 'axes fraction' and textcoords 'data', 'axes fraction' or 'offset points'" + ) + return point, text + + +def reference_line_orientation(axes: Any, line: Any) -> str | None: + """Return "horizontal" or "vertical" for a line drawn by axhline or axvline, or None.""" + # Compare by identity: these are the transforms that axhline and axvline use. + transform = line.get_transform() + if transform is axes.get_yaxis_transform(which="grid"): + values = np.asarray(line.get_ydata(orig=False), dtype=float) + orientation = "horizontal" + elif transform is axes.get_xaxis_transform(which="grid"): + values = np.asarray(line.get_xdata(orig=False), dtype=float) + orientation = "vertical" + else: + return None + if values.shape != (2,) or values[0] != values[1]: + return None + return orientation + + def _floats(values: Any) -> list[float]: return np.asarray(values, dtype=float).ravel().tolist() @@ -167,6 +218,23 @@ def _line_to_json(line: Any) -> dict[str, Any]: } +def _reference_line_to_json(axes: Any, line: Any) -> dict[str, Any]: + orientation = reference_line_orientation(axes, line) + # The value is in data coordinates, and the extent along the line is a fraction of the axes. + value, extent = ( + (line.get_ydata(orig=False), line.get_xdata()) if orientation == "horizontal" else (line.get_xdata(orig=False), line.get_ydata()) + ) + return { + "orientation": orientation, + "value": float(_floats(value)[0]), + "extent": _floats(extent), + "label": str(line.get_label()), + "color": to_hex(line.get_color(), keep_alpha=True), + "linewidth": float(line.get_linewidth()), + "linestyle": line.get_linestyle(), + } + + def _scatter_to_json(collection: Any) -> dict[str, Any]: offsets = collection.get_offsets() return { @@ -190,6 +258,38 @@ def _bars_to_json(container: Any) -> dict[str, Any]: } +def _data_point(axes: Any, point: Any, coordinates: str) -> list[float]: + if coordinates != "data": + return _floats(point) + # Convert category and date positions to numbers. + return [float(axes.xaxis.convert_units(point[0])), float(axes.yaxis.convert_units(point[1]))] + + +def _text_to_json(axes: Any, text: Any) -> dict[str, Any]: + result: dict[str, Any] = { + "text": text.get_text(), + "horizontal_alignment": text.get_horizontalalignment(), + "vertical_alignment": text.get_verticalalignment(), + "rotation": float(text.get_rotation()), + } + if type(text) is Annotation: + # Typed as Any because the matplotlib stubs don't include an annotation's xy and arrow_patch. + annotation: Any = text + point_coordinates, text_coordinates = _annotation_coordinates(annotation) + result |= { + "kind": "annotation", + "point": _data_point(axes, annotation.xy, point_coordinates), + "coordinates": point_coordinates, + "text_position": _data_point(axes, annotation.xyann, text_coordinates), + "text_coordinates": text_coordinates, + "arrow": annotation.arrow_patch is not None, + } + else: + coordinates = _text_coordinates(axes, text) + result |= {"kind": "text", "point": _data_point(axes, text.get_position(), coordinates), "coordinates": coordinates} + return result + + def _axes_to_json(axes: Any) -> dict[str, Any]: _check_axes(axes) legend = axes.get_legend() @@ -200,9 +300,11 @@ def _axes_to_json(axes: Any) -> dict[str, Any]: "title": {location: axes.get_title(loc=location) for location in ("left", "center", "right")}, "x_limits": [float(value) for value in axes.get_xlim()], "y_limits": [float(value) for value in axes.get_ylim()], - "lines": [_line_to_json(line) for line in axes.lines], + "lines": [_line_to_json(line) for line in axes.lines if line.get_transform() is axes.transData], + "reference_lines": [_reference_line_to_json(axes, line) for line in axes.lines if line.get_transform() is not axes.transData], "scatters": [_scatter_to_json(collection) for collection in axes.collections], "bars": [_bars_to_json(container) for container in axes.containers], + "texts": [_text_to_json(axes, text) for text in axes.texts], "legend": None if legend is None else {"title": legend.get_title().get_text(), "labels": [text.get_text() for text in legend.get_texts()]}, @@ -331,6 +433,56 @@ def _add_line(axes: Any, data: Any, x_categories: list[str], y_categories: list[ ) +_MATPLOTLIB_ANNOTATION_COORDINATES = {name: matplotlib_name for matplotlib_name, name in _ANNOTATION_COORDINATE_NAMES.items()} + + +def _add_text(axes: Any, data: Any) -> None: + text = _as_dict(data, "text") + kind = _as_choice(text["kind"], frozenset({"text", "annotation"}), "text kind") + coordinates = _as_choice(text["coordinates"], _TEXT_COORDINATES, "text coordinates") + point = _as_finite_numbers(text["point"], 2, "text position") + options = { + "horizontalalignment": _as_choice(text["horizontal_alignment"], _HORIZONTAL_ALIGNMENTS, "text alignment"), + "verticalalignment": _as_choice(text["vertical_alignment"], _VERTICAL_ALIGNMENTS, "text alignment"), + "rotation": _as_finite_number(text["rotation"], "text rotation"), + } + content = _as_str(text["text"], "text") + if kind == "text": + axes.text(*point, content, transform=axes.transData if coordinates == "data" else axes.transAxes, **options) + return + text_coordinates = _as_choice(text["text_coordinates"], _ANNOTATION_TEXT_COORDINATES, "annotation text coordinates") + if type(text["arrow"]) is not bool: + raise ValueError("annotation arrow should be true or false") + axes.annotate( + content, + xy=point, + xycoords=_MATPLOTLIB_ANNOTATION_COORDINATES[coordinates], + xytext=_as_finite_numbers(text["text_position"], 2, "annotation text position"), + textcoords=_MATPLOTLIB_ANNOTATION_COORDINATES[text_coordinates], + arrowprops={"arrowstyle": "->"} if text["arrow"] else None, + **options, + ) + + +def _add_reference_line(axes: Any, data: Any) -> None: + line = _as_dict(data, "reference line") + start, end = _as_finite_numbers(line["extent"], 2, "reference line extent") + add = ( + axes.axhline + if _as_choice(line["orientation"], _ORIENTATIONS_FOR_LINES, "reference line orientation") == "horizontal" + else axes.axvline + ) + add( + _as_finite_number(line["value"], "reference line value"), + start, + end, + label=_as_str(line["label"], "reference line label"), + color=_as_str(line["color"], "reference line color"), + linewidth=_as_finite_number(line["linewidth"], "reference line width"), + linestyle=_as_choice(line["linestyle"], _LINESTYLES, "reference line style"), + ) + + def _add_axes(figure: Any, data: Any) -> None: axes_data = _as_dict(data, "axes") axes = figure.add_axes(_as_finite_numbers(axes_data["position"], 4, "axes position")) @@ -355,6 +507,10 @@ def _add_axes(figure: Any, data: Any) -> None: _apply_axis_units(axes.yaxis, y_axis, y_categories) for line in _as_list(axes_data["lines"], "lines"): _add_line(axes, line, x_categories, y_categories) + for line in _as_list(axes_data["reference_lines"], "reference lines"): + _add_reference_line(axes, line) + for text in _as_list(axes_data["texts"], "texts"): + _add_text(axes, text) axes.set_xlabel(x_axis["label"]) axes.set_ylabel(y_axis["label"]) diff --git a/src/pytest_prairielearn_grader/plot_checks.py b/src/pytest_prairielearn_grader/plot_checks.py index 17b4962..29b7fe1 100644 --- a/src/pytest_prairielearn_grader/plot_checks.py +++ b/src/pytest_prairielearn_grader/plot_checks.py @@ -16,6 +16,10 @@ from matplotlib.container import BarContainer from matplotlib.figure import Figure from matplotlib.lines import Line2D +from matplotlib.text import Annotation +from matplotlib.text import Text + +from .matplotlib_json import reference_line_orientation @dataclass(frozen=True, slots=True) @@ -47,6 +51,7 @@ class Options: check_legend: bool check_limits: bool check_styles: bool + check_texts: bool class Mismatch(Exception): @@ -106,9 +111,18 @@ def _compare_axes(actual: Any, reference: Any, where: str, opts: Options) -> Non raise Mismatch(f"Expected a {expected_scale} scale on the {axis}-axis of {where}, but found a {actual_scale} scale.") if opts.check_data or opts.check_styles: - for kind, extract in (("line", _lines), ("scatter plot", _scatters), ("bar plot", _bars)): + for kind, extract in ( + ("line", _lines), + ("horizontal line", _horizontal_lines), + ("vertical line", _vertical_lines), + ("scatter plot", _scatters), + ("bar plot", _bars), + ): _compare_series(extract(actual), extract(reference), kind, where, opts) + if opts.check_texts: + _compare_texts(_texts(actual), _texts(reference), where, opts) + if opts.check_limits: for axis in ("x", "y"): actual_limits = getattr(actual, f"get_{axis}lim")() @@ -185,10 +199,104 @@ def _lines(axes: Any) -> list[Series]: }, ) for line in axes.lines - if type(line) is Line2D + if type(line) is Line2D and line.get_transform() is axes.transData ] +def _reference_lines(axes: Any, orientation: str) -> list[Series]: + series = [] + for line in axes.lines: + if type(line) is not Line2D or reference_line_orientation(axes, line) != orientation: + continue + # The value is in data coordinates, and the extent along the line is a fraction of the axes. + if orientation == "horizontal": + value, extent = line.get_ydata(orig=False), line.get_xdata() + else: + value, extent = line.get_xdata(orig=False), line.get_ydata() + series.append( + Series( + kind=f"{orientation} line", + label=str(line.get_label()), + data=[_floats(value)[:1], _floats(extent)], + reversible=False, + styles={"color": _hex(line.get_color()), "line style": line.get_linestyle()}, + ) + ) + return series + + +def _horizontal_lines(axes: Any) -> list[Series]: + return _reference_lines(axes, "horizontal") + + +def _vertical_lines(axes: Any) -> list[Series]: + return _reference_lines(axes, "vertical") + + +@dataclass(frozen=True, slots=True) +class PlotText: + kind: str + text: str + # The annotated point for an annotation. Plain text is matched by its string alone. + point: np.ndarray | None + coordinates: str + + +def _texts(axes: Any) -> list[PlotText]: + texts = [] + for text in axes.texts: + if type(text) is Annotation: + # Typed as Any because the matplotlib stubs don't include an annotation's xy. + annotation: Any = text + coordinates = "data" if annotation.xycoords == "data" else "axes" + point = annotation.xy + if coordinates == "data": + point = (axes.xaxis.convert_units(point[0]), axes.yaxis.convert_units(point[1])) + texts.append(PlotText("annotation", text.get_text(), _floats(point), coordinates)) + elif type(text) is Text: + texts.append(PlotText("text", text.get_text(), None, "data")) + return texts + + +def _format_point(point: np.ndarray, coordinates: str) -> str: + position = f"({point[0]:g}, {point[1]:g})" + return position if coordinates == "data" else f"{position} in axes coordinates" + + +def _compare_texts(actual: list[PlotText], expected: list[PlotText], where: str, opts: Options) -> None: + for kind in ("text", "annotation"): + actual_count = sum(text.kind == kind for text in actual) + expected_count = sum(text.kind == kind for text in expected) + if opts.allow_extra: + if actual_count < expected_count: + raise Mismatch(f"Expected at least {_plural(expected_count, kind)} in {where}, but found {actual_count}.") + elif actual_count != expected_count: + raise Mismatch(f"Expected {_plural(expected_count, kind)} in {where}, but found {actual_count}.") + + def matches(candidate: PlotText, reference: PlotText) -> bool: + if candidate.kind != reference.kind or candidate.text != reference.text: + return False + if reference.point is None or candidate.point is None: + return True + return candidate.coordinates == reference.coordinates and np.allclose( + candidate.point, reference.point, rtol=opts.rtol, atol=opts.atol + ) + + remaining = list(actual) + for reference in expected: + position = next((i for i, candidate in enumerate(remaining) if matches(candidate, reference)), None) + if position is not None: + remaining.pop(position) + continue + same_text = next((text for text in remaining if text.kind == reference.kind and text.text == reference.text), None) + if reference.point is not None and same_text is not None and same_text.point is not None: + raise Mismatch( + f"Expected the annotation {reference.text!r} in {where} to point at {_format_point(reference.point, reference.coordinates)}, " + f"but it points at {_format_point(same_text.point, same_text.coordinates)}." + ) + raise Mismatch(f"Expected the {reference.kind} {reference.text!r} in {where}, but didn't find it.") + + def _sorted_points(columns: list[np.ndarray], colors: list[str]) -> tuple[list[np.ndarray], list[str]]: """Sort points so that the order they were plotted in doesn't matter.""" order = np.lexsort(columns[::-1]) if len(columns[0]) else np.arange(0) diff --git a/tests/test_matplotlib_json.py b/tests/test_matplotlib_json.py index 48f2979..054dd87 100644 --- a/tests/test_matplotlib_json.py +++ b/tests/test_matplotlib_json.py @@ -13,6 +13,8 @@ from matplotcheck.base import PlotTester from matplotlib.colors import to_hex from matplotlib.figure import Figure +from matplotlib.text import Annotation +from matplotlib.text import Text from pytest_prairielearn_grader.json_utils import from_json from pytest_prairielearn_grader.json_utils import to_json @@ -198,6 +200,60 @@ def test_legend_labels_kept() -> None: assert legend.get_title().get_text() == "Key" +def test_text_and_annotations_kept() -> None: + _, axes = plt.subplots() + axes.plot([1, 2, 3], [1, 4, 9]) + axes.text(1, 5, "data", horizontalalignment="center", rotation=45) + axes.text(0.5, 0.9, "axes", transform=axes.transAxes) + axes.annotate("max", xy=(3, 9), xytext=(0, -20), textcoords="offset points", arrowprops={"facecolor": "black"}) + axes.annotate("corner", xy=(0.1, 0.1), xycoords="axes fraction") + + rebuilt: Any = round_trip(axes).axes[0] + + texts = rebuilt.texts + assert [(type(text), text.get_text()) for text in texts] == [ + (Text, "data"), + (Text, "axes"), + (Annotation, "max"), + (Annotation, "corner"), + ] + assert texts[0].get_position() == (1, 5) + assert texts[0].get_transform() is rebuilt.transData + assert (texts[0].get_horizontalalignment(), texts[0].get_rotation()) == ("center", 45) + assert texts[1].get_transform() is rebuilt.transAxes + assert (texts[2].xy, texts[2].xyann, texts[2].anncoords) == ((3, 9), (0, -20), "offset points") + assert texts[2].arrow_patch is not None + assert (texts[3].xy, texts[3].xycoords, texts[3].arrow_patch) == ((0.1, 0.1), "axes fraction", None) + + +def test_text_on_category_axis() -> None: + _, axes = plt.subplots() + axes.bar(["a", "b"], [1, 2]) + # The matplotlib stubs don't allow category positions. + axes.text("b", 1.5, "tall") # type: ignore[arg-type] + + rebuilt: Any = round_trip(axes).axes[0] + + assert rebuilt.texts[0].get_position() == (1, 1.5) + + +def test_axhline_and_axvline_kept() -> None: + _, axes = plt.subplots() + axes.plot([1, 2], [3, 4]) + axes.axhline(3.5, 0.1, 0.9, color="red", linestyle="--", label="threshold") + axes.axvline(1.5) + + rebuilt: Any = round_trip(axes).axes[0] + + data_line, horizontal, vertical = rebuilt.lines + assert data_line.get_transform() is rebuilt.transData + assert horizontal.get_transform() is rebuilt.get_yaxis_transform(which="grid") + assert (list(horizontal.get_xdata()), list(horizontal.get_ydata())) == ([0.1, 0.9], [3.5, 3.5]) + assert (to_hex(horizontal.get_color()), horizontal.get_linestyle(), horizontal.get_label()) == (to_hex("red"), "--", "threshold") + assert vertical.get_transform() is rebuilt.get_xaxis_transform(which="grid") + assert list(vertical.get_xdata()) == [1.5, 1.5] + + def test_rebuilt_figure_can_be_drawn() -> None: _, axes = plt.subplots() axes.plot([1, 2], [3, 4], label="data") @@ -249,18 +305,16 @@ def _image(axes: Any) -> None: axes.imshow([[1, 2], [3, 4]]) -def _text(axes: Any) -> None: - axes.plot([1, 2], [3, 4]) - axes.text(1, 3, "note") +def _figure_text(axes: Any) -> None: + axes.text(0.5, 0.5, "note", transform=axes.figure.transFigure) -def _annotation(axes: Any) -> None: - axes.annotate("note", (1, 3)) +def _figure_annotation(axes: Any) -> None: + axes.annotate("note", (0.5, 0.5), xycoords="figure fraction") -def _axhline(axes: Any) -> None: - axes.plot([1, 2], [3, 4]) - axes.axhline(3) +def _annotation_text_in_pixels(axes: Any) -> None: + axes.annotate("note", (1, 3), xytext=(5, 5), textcoords="offset pixels") def _axline(axes: Any) -> None: @@ -326,9 +380,9 @@ def _function_scale(axes: Any) -> None: (_bar_with_errors, "errorbar"), (_stem, "stem"), (_image, "images"), - (_text, "text"), - (_annotation, "text"), - (_axhline, "axhline"), + (_figure_text, "data or axes coordinates"), + (_figure_annotation, "xycoords"), + (_annotation_text_in_pixels, "textcoords"), (_axline, "AxLine"), (_axes_coordinates, "data coordinates"), (_blended_coordinates, "data coordinates"), @@ -395,6 +449,9 @@ def valid_data() -> dict[str, Any]: axes.plot([1, 2], [3, 4], label="line") axes.scatter([1], [2]) axes.bar([1], [2]) + axes.axhline(1) + axes.text(1, 2, "text") + axes.annotate("annotation", (1, 2), arrowprops={}) axes.legend() return json.loads(json.dumps(to_json(axes)))["_value"] @@ -455,6 +512,17 @@ def change(data: dict[str, Any]) -> None: _set(["axes", 0, "scatters", 0, "x"], []), _set(["axes", 0, "legend", "labels"], "line"), _set(["axes", 0, "title"], "title"), + _delete(["axes", 0, "texts"]), + _set(["axes", 0, "texts", 0, "kind"], "arrow"), + _set(["axes", 0, "texts", 0, "coordinates"], "figure"), + _set(["axes", 0, "texts", 0, "point"], [1]), + _set(["axes", 0, "texts", 0, "text"], 5), + _set(["axes", 0, "texts", 0, "horizontal_alignment"], "middle"), + _set(["axes", 0, "texts", 1, "text_coordinates"], "offset pixels"), + _set(["axes", 0, "texts", 1, "arrow"], "yes"), + _set(["axes", 0, "reference_lines", 0, "orientation"], "diagonal"), + _set(["axes", 0, "reference_lines", 0, "extent"], [0]), + _set(["axes", 0, "reference_lines", 0, "value"], float("inf")), ], ) def test_malformed_data_raises_value_error(change: Callable[[dict[str, Any]], None]) -> None: diff --git a/tests/test_plot_assertions.py b/tests/test_plot_assertions.py index 146a034..09e2fa3 100644 --- a/tests/test_plot_assertions.py +++ b/tests/test_plot_assertions.py @@ -41,6 +41,10 @@ def full_plot(axes: Any) -> None: axes.set_ylabel("Output") axes.set_title("Results") axes.set_title("left", loc="left") + axes.axhline(50, color="gray", linestyle=":", label="threshold") + axes.axvline(5) + axes.text(2, 80, "note") + axes.annotate("peak", xy=(10, 100), xytext=(0, -20), textcoords="offset points", arrowprops={"arrowstyle": "->"}) axes.set_xlim(-1, 12) axes.legend(title="Key") @@ -64,10 +68,36 @@ def full_plot(axes: Any) -> None: pytest.param(lambda a: (semilogy(a), a.plot(X, X)), semilogy, {"allow_extra": True}, id="allow-extra"), pytest.param(lambda a: (a.semilogy(X, np.exp(X), "r:"), a.set_xlabel("other")), semilogy, {}, id="labels-and-styles-unchecked"), pytest.param(lambda a: a.plot(X, X), semilogy, {"check_scales": False, "check_data": False}, id="nothing-checked"), + pytest.param(lambda a: (semilogy(a), a.text(3, 10, "e^x")), semilogy, {}, id="extra-text-unchecked"), + pytest.param( + lambda a: (semilogy(a), a.text(1, 1, "note"), a.annotate("max", xy=(10, np.exp(10)), xytext=(1, 1))), + lambda a: (semilogy(a), a.text(5, 5, "note"), a.annotate("max", xy=(10, np.exp(10)))), + {"check_texts": True}, + id="text-position-free", + ), + pytest.param( + lambda a: (semilogy(a), a.text(1, 1, "extra")), + semilogy, + {"check_texts": True, "allow_extra": True}, + id="extra-text-allowed", + ), + pytest.param( + lambda a: (a.axvline(2), a.axhline(1, color="red"), semilogy(a)), + lambda a: (semilogy(a), a.axhline(1), a.axvline(2)), + {}, + id="reference-lines", + ), pytest.param( full_plot, full_plot, - {"check_labels": True, "check_titles": True, "check_legend": True, "check_limits": True, "check_styles": True}, + { + "check_labels": True, + "check_titles": True, + "check_legend": True, + "check_limits": True, + "check_styles": True, + "check_texts": True, + }, id="everything", ), ], @@ -95,6 +125,7 @@ def test_plot_sent_from_the_sandbox_matches() -> None: check_legend=True, check_limits=True, check_styles=True, + check_texts=True, ) @@ -208,6 +239,41 @@ def test_plot_sent_from_the_sandbox_matches() -> None: "Expected the expected bar plot in your plot to have colors #1f77b4ff, but found #ff0000ff.", id="bar-color", ), + pytest.param( + semilogy, + lambda a: (semilogy(a), a.axhline(100)), + {}, + "Expected 1 horizontal line in your plot, but found 0.", + id="missing-axhline", + ), + pytest.param( + lambda a: (semilogy(a), a.axvline(4)), + lambda a: (semilogy(a), a.axvline(5)), + {}, + "No vertical line in your plot matches the expected vertical line.", + id="wrong-axvline", + ), + pytest.param( + lambda a: (semilogy(a), a.text(1, 1, "growth")), + lambda a: (semilogy(a), a.text(1, 1, "Growth")), + {"check_texts": True}, + "Expected the text 'Growth' in your plot, but didn't find it.", + id="text-string", + ), + pytest.param( + lambda a: (semilogy(a), a.text(1, 1, "a"), a.text(1, 1, "b")), + lambda a: (semilogy(a), a.text(1, 1, "a")), + {"check_texts": True}, + "Expected 1 text in your plot, but found 2.", + id="text-count", + ), + pytest.param( + lambda a: (semilogy(a), a.annotate("max", xy=(9, np.exp(9)))), + lambda a: (semilogy(a), a.annotate("max", xy=(10, np.exp(10)))), + {"check_texts": True}, + "Expected the annotation 'max' in your plot to point at (10, 22026.5), but it points at (9, 8103.08).", + id="annotation-point", + ), pytest.param( lambda a: a.plot(X, X), semilogy, From 436c224a57d81f5d179fc31ec21042c6cd7036ca Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Fri, 13 Nov 2026 20:34:28 -0500 Subject: [PATCH 2/2] Report the sandbox's own initialization timeout instead of racing it The parent read the start response with the same timeout the sandbox used for the student code, so a slow submission got either message at random. Wait a little longer so the sandbox's timeout response arrives first. Co-Authored-By: Claude Opus 5.5 --- src/pytest_prairielearn_grader/fixture.py | 11 ++++++++--- .../test_initialization_timeout/expected_outcome.json | 2 +- .../scenario_root/test_timeout/expected_outcome.json | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 921d4bb..dd2a519 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -36,6 +36,9 @@ SCRIPT_PATH = str(files("pytest_prairielearn_grader").joinpath("_student_code_runner.py")) DEFAULT_TIMEOUT = 1.0 +# Extra time to wait for the start response, so that the sandbox's own +# initialization timeout is reported instead of racing against the socket timeout. +START_RESPONSE_GRACE = 0.5 # Modes for the data files folder (see _make_data_files_dir). Everyone can read the files and list the # folders, and nobody can change them; only the owner (root, in the Docker image) can make them writable again. @@ -305,8 +308,10 @@ def _connect_to_server(self, timeout: float) -> None: self.student_socket.settimeout(timeout) self.student_socket.connect((host, int(port))) - def _read_start_response(self) -> ProcessStartResponse: + def _read_start_response(self, initialization_timeout: float) -> ProcessStartResponse: """Read and return the ProcessStartResponse sent after the start message.""" + assert self.student_socket is not None + self.student_socket.settimeout(initialization_timeout + START_RESPONSE_GRACE) try: data = self._read_from_socket().decode() res: ProcessStartResponse = json.loads(data) @@ -488,7 +493,7 @@ def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_T helper_modules=self.helper_modules, ) self._send_json_object(json_message) - return self._read_start_response() + return self._read_start_response(initialization_timeout) def query_setup_raw(self, var_to_query: str) -> SetupQueryResponse: self._assert_process_running() @@ -581,7 +586,7 @@ def start_workspace_server(self, *, initialization_timeout: float = DEFAULT_TIME helper_modules=self.helper_modules, ) self._send_json_object(json_message) - return self._read_start_response() + return self._read_start_response(initialization_timeout) def __repr__(self) -> str: return f"WorkspaceFixture(workspace_dir={self.workspace_dir})" diff --git a/tests/scenario_root/test_initialization_timeout/expected_outcome.json b/tests/scenario_root/test_initialization_timeout/expected_outcome.json index 79d80ad..4ab7b03 100644 --- a/tests/scenario_root/test_initialization_timeout/expected_outcome.json +++ b/tests/scenario_root/test_initialization_timeout/expected_outcome.json @@ -13,7 +13,7 @@ }, { "max_points": 2, - "message": "No response from initialization with timeout 0.05", + "message": "Student code initialization timed out", "name": "marker_overrides_module", "points_frac": 0.0, "points": 0.0, diff --git a/tests/scenario_root/test_timeout/expected_outcome.json b/tests/scenario_root/test_timeout/expected_outcome.json index 484f995..eb8f064 100644 --- a/tests/scenario_root/test_timeout/expected_outcome.json +++ b/tests/scenario_root/test_timeout/expected_outcome.json @@ -4,7 +4,7 @@ "tests": [ { "max_points": 2, - "message_old": "No response from initialization with timeout 0.05", + "message_old": "Student code initialization timed out", "name": "initialization_timeout", "points_frac": 0.0, "points": 0.0, @@ -22,7 +22,7 @@ }, { "max_points": 2, - "message": "No response from initialization with timeout 0.08", + "message": "Student code initialization timed out", "name": "marker_overrides_module", "points_frac": 0.0, "points": 0.0,