diff --git a/quick_start.md b/quick_start.md index bb14f43..c3df74a 100644 --- a/quick_start.md +++ b/quick_start.md @@ -627,14 +627,17 @@ An `Axes` value brings its whole figure, so subplots are never lost: use `figure What's sent to the test: - **Lines**, with their data, label, color, width, style, marker and draw style. This includes `plot`, `semilogy`, `loglog` and `step`. +- **Horizontal and vertical lines** from `axhline` and `axvline`, with their position, extent, label, color, width and style. +- **Text** from `text`, in data or axes coordinates, with its position, alignment and rotation. +- **Annotations** from `annotate`, with the annotated point in data or axes coordinates, the text position in data or axes coordinates or offset points, and whether it has an arrow. - **Scatter points**, with their positions, sizes and colors. - **Bars and histograms**, with their positions, sizes, colors and orientation. - **Axes details:** scales, labels, titles, limits, position and legend labels. Twin axes come back as separate axes in `figure.axes`. - **Figure details:** size and overall title. -Other plot content makes the query fail with a message saying what isn't supported, rather than losing data. This includes `errorbar`, `stem`, `fill_between`, `pie`, `imshow`, `contour`, `vlines` and `hlines`, `axhline` and `axvline`, `axvspan`, text and annotations, colormapped scatter plots, polar axes, inset axes and subfigures. +Other plot content makes the query fail with a message saying what isn't supported, rather than losing data. This includes `errorbar`, `stem`, `fill_between`, `pie`, `imshow`, `contour`, `vlines` and `hlines`, `axline`, `axvspan`, text in figure coordinates, colormapped scatter plots, polar axes, inset axes and subfigures. -Some details aren't sent, so don't grade them: grid lines, tick positions and labels, line and marker transparency set separately from color, scatter marker shapes, the base of a log scale, and `symlog` settings. Values are sent as decimal numbers, and dates come back as matplotlib date numbers (use `matplotlib.dates.num2date` to convert them). Categories on a line come back as strings. +Some details aren't sent, so don't grade them: grid lines, tick positions and labels, line and marker transparency set separately from color, scatter marker shapes, the base of a log scale, `symlog` settings, text fonts and colors, and arrow styles, since every annotation arrow comes back as a plain arrow. Values are sent as decimal numbers, and dates come back as matplotlib date numbers (use `matplotlib.dates.num2date` to convert them). Categories on a line come back as strings. Importing matplotlib can take most of the default 1-second sandbox timeout, so set a longer one in plot questions, such as `sandbox_timeout = 15` at the top of the test file. @@ -672,24 +675,25 @@ By default, it checks: - **Subplots:** the number of axes. - **Scales:** the x and y axis scales, such as linear or log. -- **Data:** the lines, scatter plots and bar plots, which must have the same number of each kind. Data is compared with `numpy.allclose`, using `rtol` and `atol`. +- **Data:** the lines, `axhline` and `axvline` lines, scatter plots and bar plots, which must have the same number of each kind. Data is compared with `numpy.allclose`, using `rtol` and `atol`. The series can be in any order, and a line also matches when its points are in reverse order, since it looks the same. The points in a scatter plot and the bars in a bar plot can be in any order. Keyword arguments change what's checked: -| Argument | Default | Checks | -| -------------- | ------- | ----------------------------------------------------------------------------------------------------------- | -| `check_scales` | `True` | The x and y axis scales. | -| `check_data` | `True` | The data in lines, scatter plots and bar plots. | -| `ordered` | `False` | Requires lines, scatter plots, bar plots and legend labels to be in the reference's order. | -| `allow_extra` | `False` | Allows more lines, scatter plots or bar plots than the reference, such as a student's extra reference line. | -| `check_labels` | `False` | The x and y axis labels, exactly. | -| `check_titles` | `False` | The title of each subplot and the figure, exactly. | -| `check_legend` | `False` | The legend's labels and title, or that neither plot has a legend. | -| `check_limits` | `False` | The x and y axis limits, within `rtol` and `atol`. | -| `check_styles` | `False` | Line colors, styles and markers, scatter point colors, and bar colors. | +| Argument | Default | Checks | +| -------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `check_scales` | `True` | The x and y axis scales. | +| `check_data` | `True` | The data in lines, `axhline` and `axvline` lines, scatter plots and bar plots. | +| `ordered` | `False` | Requires lines, scatter plots, bar plots and legend labels to be in the reference's order. | +| `allow_extra` | `False` | Allows more lines, scatter plots, bar plots, text or annotations than the reference, such as a student's extra reference line. | +| `check_labels` | `False` | The x and y axis labels, exactly. | +| `check_titles` | `False` | The title of each subplot and the figure, exactly. | +| `check_legend` | `False` | The legend's labels and title, or that neither plot has a legend. | +| `check_limits` | `False` | The x and y axis limits, within `rtol` and `atol`. | +| `check_styles` | `False` | Line colors, styles and markers, scatter point colors, and bar colors. | +| `check_texts` | `False` | The strings of `text` and `annotate` labels, and the point each annotation points at. The position of plain text isn't compared. | A failure describes the first difference found, such as `Expected a log scale on the y-axis of your plot, but found a linear scale.` or `No line in your plot matches the line labeled 'exp'.` Lines diff --git a/src/pytest_prairielearn_grader/assertions.py b/src/pytest_prairielearn_grader/assertions.py index 6b98aae..ea442da 100644 --- a/src/pytest_prairielearn_grader/assertions.py +++ b/src/pytest_prairielearn_grader/assertions.py @@ -204,6 +204,7 @@ def assert_plot_equal( check_legend: bool = False, check_limits: bool = False, check_styles: bool = False, + check_texts: bool = False, msg: str | None = None, ) -> None: """Assert that a student's matplotlib plot matches a reference plot. @@ -213,8 +214,8 @@ def assert_plot_equal( test itself. Subplots are compared in order. By default this checks the number of subplots, the axis scales, and the - data in each line, scatter plot and bar plot, which must have the same - number of each kind. Series can be in any order, and a line also matches + data in each line, ``axhline`` and ``axvline`` line, scatter plot and bar + plot, which must have the same number of each kind. Series can be in any order, and a line also matches when its points are in reverse order. Scatter points and bars can be in any order. Numbers are compared with ``numpy.allclose``. @@ -237,8 +238,8 @@ def assert_plot_equal( Require lines, scatter plots, bar plots and legend labels to be in the same order as in the reference. allow_extra: - Allow the student's plot to have more lines, scatter plots or bar - plots than the reference. + Allow the student's plot to have more lines, scatter plots, bar + plots, text or annotations than the reference. check_labels: Compare the x and y axis labels exactly. check_titles: @@ -251,6 +252,10 @@ def assert_plot_equal( check_styles: Compare line colors, line styles and markers, scatter point colors, and bar colors and orientation. + check_texts: + Compare the strings of ``text`` and ``annotate`` labels, and the point + each annotation points at. The position of plain text isn't compared. + Extra text is allowed with *allow_extra*. msg: Optional message to show instead of the description of the difference. @@ -280,6 +285,7 @@ def assert_plot_equal( check_legend=check_legend, check_limits=check_limits, check_styles=check_styles, + check_texts=check_texts, ) if mismatch is not None: raise AssertionError(msg or mismatch) diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index 921d4bb..dd2a519 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -36,6 +36,9 @@ SCRIPT_PATH = str(files("pytest_prairielearn_grader").joinpath("_student_code_runner.py")) DEFAULT_TIMEOUT = 1.0 +# Extra time to wait for the start response, so that the sandbox's own +# initialization timeout is reported instead of racing against the socket timeout. +START_RESPONSE_GRACE = 0.5 # Modes for the data files folder (see _make_data_files_dir). Everyone can read the files and list the # folders, and nobody can change them; only the owner (root, in the Docker image) can make them writable again. @@ -305,8 +308,10 @@ def _connect_to_server(self, timeout: float) -> None: self.student_socket.settimeout(timeout) self.student_socket.connect((host, int(port))) - def _read_start_response(self) -> ProcessStartResponse: + def _read_start_response(self, initialization_timeout: float) -> ProcessStartResponse: """Read and return the ProcessStartResponse sent after the start message.""" + assert self.student_socket is not None + self.student_socket.settimeout(initialization_timeout + START_RESPONSE_GRACE) try: data = self._read_from_socket().decode() res: ProcessStartResponse = json.loads(data) @@ -488,7 +493,7 @@ def start_student_code_server(self, *, initialization_timeout: float = DEFAULT_T helper_modules=self.helper_modules, ) self._send_json_object(json_message) - return self._read_start_response() + return self._read_start_response(initialization_timeout) def query_setup_raw(self, var_to_query: str) -> SetupQueryResponse: self._assert_process_running() @@ -581,7 +586,7 @@ def start_workspace_server(self, *, initialization_timeout: float = DEFAULT_TIME helper_modules=self.helper_modules, ) self._send_json_object(json_message) - return self._read_start_response() + return self._read_start_response(initialization_timeout) def __repr__(self) -> str: return f"WorkspaceFixture(workspace_dir={self.workspace_dir})" diff --git a/src/pytest_prairielearn_grader/matplotlib_json.py b/src/pytest_prairielearn_grader/matplotlib_json.py index 624c2a5..067b530 100644 --- a/src/pytest_prairielearn_grader/matplotlib_json.py +++ b/src/pytest_prairielearn_grader/matplotlib_json.py @@ -22,6 +22,8 @@ from matplotlib.figure import Figure from matplotlib.lines import Line2D from matplotlib.patches import Rectangle +from matplotlib.text import Annotation +from matplotlib.text import Text MATPLOTLIB_FIGURE_TYPE = "matplotlib_figure" @@ -30,6 +32,13 @@ _DRAWSTYLES = frozenset({"default", "steps", "steps-pre", "steps-mid", "steps-post"}) _ORIENTATIONS = frozenset({"vertical", "horizontal"}) _AXIS_UNITS = frozenset({"number", "date", "category"}) +_TEXT_COORDINATES = frozenset({"data", "axes"}) +_ANNOTATION_TEXT_COORDINATES = frozenset({"data", "axes", "offset points"}) +_ORIENTATIONS_FOR_LINES = frozenset({"horizontal", "vertical"}) +# matplotlib names for the coordinate systems that annotations can use. +_ANNOTATION_COORDINATE_NAMES = {"data": "data", "axes fraction": "axes", "offset points": "offset points"} +_HORIZONTAL_ALIGNMENTS = frozenset({"left", "center", "right"}) +_VERTICAL_ALIGNMENTS = frozenset({"top", "bottom", "center", "baseline", "center_baseline"}) _MAX_FIGURE_SIZE_INCHES = 1000.0 @@ -86,8 +95,13 @@ def _check_axes(axes: Any) -> None: raise UnsupportedPlotError("inset axes aren't supported") if axes.images: raise UnsupportedPlotError("images aren't supported") - if axes.texts: - raise UnsupportedPlotError("text and annotations aren't supported") + for text in axes.texts: + if type(text) is Annotation: + _annotation_coordinates(text) + elif type(text) is Text: + _text_coordinates(axes, text) + else: + raise UnsupportedPlotError(f"{type(text).__name__} text isn't supported") if axes.tables: raise UnsupportedPlotError("tables aren't supported") if axes.artists: @@ -123,8 +137,8 @@ def _check_axes(axes: Any) -> None: for line in axes.lines: if type(line) is not Line2D: raise UnsupportedPlotError(f"{type(line).__name__} lines aren't supported") - if line.get_transform() is not axes.transData: - raise UnsupportedPlotError("lines that don't use data coordinates, such as axhline and axvline, aren't supported") + if line.get_transform() is not axes.transData and reference_line_orientation(axes, line) is None: + raise UnsupportedPlotError("lines that don't use data coordinates, other than axhline and axvline, aren't supported") if not isinstance(line.get_marker(), str | None): raise UnsupportedPlotError("custom line markers aren't supported") @@ -150,6 +164,43 @@ def _axis_to_json(axis: Any) -> dict[str, Any]: return {"scale": scale, "label": axis.get_label_text(), "units": units, "categories": categories} +def _text_coordinates(axes: Any, text: Any) -> str: + # Compare by identity, as for scatter offsets. + transform = text.get_transform() + if transform is axes.transData: + return "data" + if transform is axes.transAxes: + return "axes" + raise UnsupportedPlotError("text that doesn't use data or axes coordinates isn't supported") + + +def _annotation_coordinates(annotation: Any) -> tuple[str, str]: + point = _ANNOTATION_COORDINATE_NAMES.get(annotation.xycoords) if isinstance(annotation.xycoords, str) else None + text = _ANNOTATION_COORDINATE_NAMES.get(annotation.anncoords) if isinstance(annotation.anncoords, str) else None + if point not in _TEXT_COORDINATES or text is None: + raise UnsupportedPlotError( + "annotations are supported only with xycoords 'data' or 'axes fraction' and textcoords 'data', 'axes fraction' or 'offset points'" + ) + return point, text + + +def reference_line_orientation(axes: Any, line: Any) -> str | None: + """Return "horizontal" or "vertical" for a line drawn by axhline or axvline, or None.""" + # Compare by identity: these are the transforms that axhline and axvline use. + transform = line.get_transform() + if transform is axes.get_yaxis_transform(which="grid"): + values = np.asarray(line.get_ydata(orig=False), dtype=float) + orientation = "horizontal" + elif transform is axes.get_xaxis_transform(which="grid"): + values = np.asarray(line.get_xdata(orig=False), dtype=float) + orientation = "vertical" + else: + return None + if values.shape != (2,) or values[0] != values[1]: + return None + return orientation + + def _floats(values: Any) -> list[float]: return np.asarray(values, dtype=float).ravel().tolist() @@ -167,6 +218,23 @@ def _line_to_json(line: Any) -> dict[str, Any]: } +def _reference_line_to_json(axes: Any, line: Any) -> dict[str, Any]: + orientation = reference_line_orientation(axes, line) + # The value is in data coordinates, and the extent along the line is a fraction of the axes. + value, extent = ( + (line.get_ydata(orig=False), line.get_xdata()) if orientation == "horizontal" else (line.get_xdata(orig=False), line.get_ydata()) + ) + return { + "orientation": orientation, + "value": float(_floats(value)[0]), + "extent": _floats(extent), + "label": str(line.get_label()), + "color": to_hex(line.get_color(), keep_alpha=True), + "linewidth": float(line.get_linewidth()), + "linestyle": line.get_linestyle(), + } + + def _scatter_to_json(collection: Any) -> dict[str, Any]: offsets = collection.get_offsets() return { @@ -190,6 +258,38 @@ def _bars_to_json(container: Any) -> dict[str, Any]: } +def _data_point(axes: Any, point: Any, coordinates: str) -> list[float]: + if coordinates != "data": + return _floats(point) + # Convert category and date positions to numbers. + return [float(axes.xaxis.convert_units(point[0])), float(axes.yaxis.convert_units(point[1]))] + + +def _text_to_json(axes: Any, text: Any) -> dict[str, Any]: + result: dict[str, Any] = { + "text": text.get_text(), + "horizontal_alignment": text.get_horizontalalignment(), + "vertical_alignment": text.get_verticalalignment(), + "rotation": float(text.get_rotation()), + } + if type(text) is Annotation: + # Typed as Any because the matplotlib stubs don't include an annotation's xy and arrow_patch. + annotation: Any = text + point_coordinates, text_coordinates = _annotation_coordinates(annotation) + result |= { + "kind": "annotation", + "point": _data_point(axes, annotation.xy, point_coordinates), + "coordinates": point_coordinates, + "text_position": _data_point(axes, annotation.xyann, text_coordinates), + "text_coordinates": text_coordinates, + "arrow": annotation.arrow_patch is not None, + } + else: + coordinates = _text_coordinates(axes, text) + result |= {"kind": "text", "point": _data_point(axes, text.get_position(), coordinates), "coordinates": coordinates} + return result + + def _axes_to_json(axes: Any) -> dict[str, Any]: _check_axes(axes) legend = axes.get_legend() @@ -200,9 +300,11 @@ def _axes_to_json(axes: Any) -> dict[str, Any]: "title": {location: axes.get_title(loc=location) for location in ("left", "center", "right")}, "x_limits": [float(value) for value in axes.get_xlim()], "y_limits": [float(value) for value in axes.get_ylim()], - "lines": [_line_to_json(line) for line in axes.lines], + "lines": [_line_to_json(line) for line in axes.lines if line.get_transform() is axes.transData], + "reference_lines": [_reference_line_to_json(axes, line) for line in axes.lines if line.get_transform() is not axes.transData], "scatters": [_scatter_to_json(collection) for collection in axes.collections], "bars": [_bars_to_json(container) for container in axes.containers], + "texts": [_text_to_json(axes, text) for text in axes.texts], "legend": None if legend is None else {"title": legend.get_title().get_text(), "labels": [text.get_text() for text in legend.get_texts()]}, @@ -331,6 +433,56 @@ def _add_line(axes: Any, data: Any, x_categories: list[str], y_categories: list[ ) +_MATPLOTLIB_ANNOTATION_COORDINATES = {name: matplotlib_name for matplotlib_name, name in _ANNOTATION_COORDINATE_NAMES.items()} + + +def _add_text(axes: Any, data: Any) -> None: + text = _as_dict(data, "text") + kind = _as_choice(text["kind"], frozenset({"text", "annotation"}), "text kind") + coordinates = _as_choice(text["coordinates"], _TEXT_COORDINATES, "text coordinates") + point = _as_finite_numbers(text["point"], 2, "text position") + options = { + "horizontalalignment": _as_choice(text["horizontal_alignment"], _HORIZONTAL_ALIGNMENTS, "text alignment"), + "verticalalignment": _as_choice(text["vertical_alignment"], _VERTICAL_ALIGNMENTS, "text alignment"), + "rotation": _as_finite_number(text["rotation"], "text rotation"), + } + content = _as_str(text["text"], "text") + if kind == "text": + axes.text(*point, content, transform=axes.transData if coordinates == "data" else axes.transAxes, **options) + return + text_coordinates = _as_choice(text["text_coordinates"], _ANNOTATION_TEXT_COORDINATES, "annotation text coordinates") + if type(text["arrow"]) is not bool: + raise ValueError("annotation arrow should be true or false") + axes.annotate( + content, + xy=point, + xycoords=_MATPLOTLIB_ANNOTATION_COORDINATES[coordinates], + xytext=_as_finite_numbers(text["text_position"], 2, "annotation text position"), + textcoords=_MATPLOTLIB_ANNOTATION_COORDINATES[text_coordinates], + arrowprops={"arrowstyle": "->"} if text["arrow"] else None, + **options, + ) + + +def _add_reference_line(axes: Any, data: Any) -> None: + line = _as_dict(data, "reference line") + start, end = _as_finite_numbers(line["extent"], 2, "reference line extent") + add = ( + axes.axhline + if _as_choice(line["orientation"], _ORIENTATIONS_FOR_LINES, "reference line orientation") == "horizontal" + else axes.axvline + ) + add( + _as_finite_number(line["value"], "reference line value"), + start, + end, + label=_as_str(line["label"], "reference line label"), + color=_as_str(line["color"], "reference line color"), + linewidth=_as_finite_number(line["linewidth"], "reference line width"), + linestyle=_as_choice(line["linestyle"], _LINESTYLES, "reference line style"), + ) + + def _add_axes(figure: Any, data: Any) -> None: axes_data = _as_dict(data, "axes") axes = figure.add_axes(_as_finite_numbers(axes_data["position"], 4, "axes position")) @@ -355,6 +507,10 @@ def _add_axes(figure: Any, data: Any) -> None: _apply_axis_units(axes.yaxis, y_axis, y_categories) for line in _as_list(axes_data["lines"], "lines"): _add_line(axes, line, x_categories, y_categories) + for line in _as_list(axes_data["reference_lines"], "reference lines"): + _add_reference_line(axes, line) + for text in _as_list(axes_data["texts"], "texts"): + _add_text(axes, text) axes.set_xlabel(x_axis["label"]) axes.set_ylabel(y_axis["label"]) diff --git a/src/pytest_prairielearn_grader/plot_checks.py b/src/pytest_prairielearn_grader/plot_checks.py index 17b4962..29b7fe1 100644 --- a/src/pytest_prairielearn_grader/plot_checks.py +++ b/src/pytest_prairielearn_grader/plot_checks.py @@ -16,6 +16,10 @@ from matplotlib.container import BarContainer from matplotlib.figure import Figure from matplotlib.lines import Line2D +from matplotlib.text import Annotation +from matplotlib.text import Text + +from .matplotlib_json import reference_line_orientation @dataclass(frozen=True, slots=True) @@ -47,6 +51,7 @@ class Options: check_legend: bool check_limits: bool check_styles: bool + check_texts: bool class Mismatch(Exception): @@ -106,9 +111,18 @@ def _compare_axes(actual: Any, reference: Any, where: str, opts: Options) -> Non raise Mismatch(f"Expected a {expected_scale} scale on the {axis}-axis of {where}, but found a {actual_scale} scale.") if opts.check_data or opts.check_styles: - for kind, extract in (("line", _lines), ("scatter plot", _scatters), ("bar plot", _bars)): + for kind, extract in ( + ("line", _lines), + ("horizontal line", _horizontal_lines), + ("vertical line", _vertical_lines), + ("scatter plot", _scatters), + ("bar plot", _bars), + ): _compare_series(extract(actual), extract(reference), kind, where, opts) + if opts.check_texts: + _compare_texts(_texts(actual), _texts(reference), where, opts) + if opts.check_limits: for axis in ("x", "y"): actual_limits = getattr(actual, f"get_{axis}lim")() @@ -185,10 +199,104 @@ def _lines(axes: Any) -> list[Series]: }, ) for line in axes.lines - if type(line) is Line2D + if type(line) is Line2D and line.get_transform() is axes.transData ] +def _reference_lines(axes: Any, orientation: str) -> list[Series]: + series = [] + for line in axes.lines: + if type(line) is not Line2D or reference_line_orientation(axes, line) != orientation: + continue + # The value is in data coordinates, and the extent along the line is a fraction of the axes. + if orientation == "horizontal": + value, extent = line.get_ydata(orig=False), line.get_xdata() + else: + value, extent = line.get_xdata(orig=False), line.get_ydata() + series.append( + Series( + kind=f"{orientation} line", + label=str(line.get_label()), + data=[_floats(value)[:1], _floats(extent)], + reversible=False, + styles={"color": _hex(line.get_color()), "line style": line.get_linestyle()}, + ) + ) + return series + + +def _horizontal_lines(axes: Any) -> list[Series]: + return _reference_lines(axes, "horizontal") + + +def _vertical_lines(axes: Any) -> list[Series]: + return _reference_lines(axes, "vertical") + + +@dataclass(frozen=True, slots=True) +class PlotText: + kind: str + text: str + # The annotated point for an annotation. Plain text is matched by its string alone. + point: np.ndarray | None + coordinates: str + + +def _texts(axes: Any) -> list[PlotText]: + texts = [] + for text in axes.texts: + if type(text) is Annotation: + # Typed as Any because the matplotlib stubs don't include an annotation's xy. + annotation: Any = text + coordinates = "data" if annotation.xycoords == "data" else "axes" + point = annotation.xy + if coordinates == "data": + point = (axes.xaxis.convert_units(point[0]), axes.yaxis.convert_units(point[1])) + texts.append(PlotText("annotation", text.get_text(), _floats(point), coordinates)) + elif type(text) is Text: + texts.append(PlotText("text", text.get_text(), None, "data")) + return texts + + +def _format_point(point: np.ndarray, coordinates: str) -> str: + position = f"({point[0]:g}, {point[1]:g})" + return position if coordinates == "data" else f"{position} in axes coordinates" + + +def _compare_texts(actual: list[PlotText], expected: list[PlotText], where: str, opts: Options) -> None: + for kind in ("text", "annotation"): + actual_count = sum(text.kind == kind for text in actual) + expected_count = sum(text.kind == kind for text in expected) + if opts.allow_extra: + if actual_count < expected_count: + raise Mismatch(f"Expected at least {_plural(expected_count, kind)} in {where}, but found {actual_count}.") + elif actual_count != expected_count: + raise Mismatch(f"Expected {_plural(expected_count, kind)} in {where}, but found {actual_count}.") + + def matches(candidate: PlotText, reference: PlotText) -> bool: + if candidate.kind != reference.kind or candidate.text != reference.text: + return False + if reference.point is None or candidate.point is None: + return True + return candidate.coordinates == reference.coordinates and np.allclose( + candidate.point, reference.point, rtol=opts.rtol, atol=opts.atol + ) + + remaining = list(actual) + for reference in expected: + position = next((i for i, candidate in enumerate(remaining) if matches(candidate, reference)), None) + if position is not None: + remaining.pop(position) + continue + same_text = next((text for text in remaining if text.kind == reference.kind and text.text == reference.text), None) + if reference.point is not None and same_text is not None and same_text.point is not None: + raise Mismatch( + f"Expected the annotation {reference.text!r} in {where} to point at {_format_point(reference.point, reference.coordinates)}, " + f"but it points at {_format_point(same_text.point, same_text.coordinates)}." + ) + raise Mismatch(f"Expected the {reference.kind} {reference.text!r} in {where}, but didn't find it.") + + def _sorted_points(columns: list[np.ndarray], colors: list[str]) -> tuple[list[np.ndarray], list[str]]: """Sort points so that the order they were plotted in doesn't matter.""" order = np.lexsort(columns[::-1]) if len(columns[0]) else np.arange(0) diff --git a/tests/scenario_root/test_initialization_timeout/expected_outcome.json b/tests/scenario_root/test_initialization_timeout/expected_outcome.json index 79d80ad..4ab7b03 100644 --- a/tests/scenario_root/test_initialization_timeout/expected_outcome.json +++ b/tests/scenario_root/test_initialization_timeout/expected_outcome.json @@ -13,7 +13,7 @@ }, { "max_points": 2, - "message": "No response from initialization with timeout 0.05", + "message": "Student code initialization timed out", "name": "marker_overrides_module", "points_frac": 0.0, "points": 0.0, diff --git a/tests/scenario_root/test_timeout/expected_outcome.json b/tests/scenario_root/test_timeout/expected_outcome.json index 484f995..eb8f064 100644 --- a/tests/scenario_root/test_timeout/expected_outcome.json +++ b/tests/scenario_root/test_timeout/expected_outcome.json @@ -4,7 +4,7 @@ "tests": [ { "max_points": 2, - "message_old": "No response from initialization with timeout 0.05", + "message_old": "Student code initialization timed out", "name": "initialization_timeout", "points_frac": 0.0, "points": 0.0, @@ -22,7 +22,7 @@ }, { "max_points": 2, - "message": "No response from initialization with timeout 0.08", + "message": "Student code initialization timed out", "name": "marker_overrides_module", "points_frac": 0.0, "points": 0.0, diff --git a/tests/test_matplotlib_json.py b/tests/test_matplotlib_json.py index 48f2979..054dd87 100644 --- a/tests/test_matplotlib_json.py +++ b/tests/test_matplotlib_json.py @@ -13,6 +13,8 @@ from matplotcheck.base import PlotTester from matplotlib.colors import to_hex from matplotlib.figure import Figure +from matplotlib.text import Annotation +from matplotlib.text import Text from pytest_prairielearn_grader.json_utils import from_json from pytest_prairielearn_grader.json_utils import to_json @@ -198,6 +200,60 @@ def test_legend_labels_kept() -> None: assert legend.get_title().get_text() == "Key" +def test_text_and_annotations_kept() -> None: + _, axes = plt.subplots() + axes.plot([1, 2, 3], [1, 4, 9]) + axes.text(1, 5, "data", horizontalalignment="center", rotation=45) + axes.text(0.5, 0.9, "axes", transform=axes.transAxes) + axes.annotate("max", xy=(3, 9), xytext=(0, -20), textcoords="offset points", arrowprops={"facecolor": "black"}) + axes.annotate("corner", xy=(0.1, 0.1), xycoords="axes fraction") + + rebuilt: Any = round_trip(axes).axes[0] + + texts = rebuilt.texts + assert [(type(text), text.get_text()) for text in texts] == [ + (Text, "data"), + (Text, "axes"), + (Annotation, "max"), + (Annotation, "corner"), + ] + assert texts[0].get_position() == (1, 5) + assert texts[0].get_transform() is rebuilt.transData + assert (texts[0].get_horizontalalignment(), texts[0].get_rotation()) == ("center", 45) + assert texts[1].get_transform() is rebuilt.transAxes + assert (texts[2].xy, texts[2].xyann, texts[2].anncoords) == ((3, 9), (0, -20), "offset points") + assert texts[2].arrow_patch is not None + assert (texts[3].xy, texts[3].xycoords, texts[3].arrow_patch) == ((0.1, 0.1), "axes fraction", None) + + +def test_text_on_category_axis() -> None: + _, axes = plt.subplots() + axes.bar(["a", "b"], [1, 2]) + # The matplotlib stubs don't allow category positions. + axes.text("b", 1.5, "tall") # type: ignore[arg-type] + + rebuilt: Any = round_trip(axes).axes[0] + + assert rebuilt.texts[0].get_position() == (1, 1.5) + + +def test_axhline_and_axvline_kept() -> None: + _, axes = plt.subplots() + axes.plot([1, 2], [3, 4]) + axes.axhline(3.5, 0.1, 0.9, color="red", linestyle="--", label="threshold") + axes.axvline(1.5) + + rebuilt: Any = round_trip(axes).axes[0] + + data_line, horizontal, vertical = rebuilt.lines + assert data_line.get_transform() is rebuilt.transData + assert horizontal.get_transform() is rebuilt.get_yaxis_transform(which="grid") + assert (list(horizontal.get_xdata()), list(horizontal.get_ydata())) == ([0.1, 0.9], [3.5, 3.5]) + assert (to_hex(horizontal.get_color()), horizontal.get_linestyle(), horizontal.get_label()) == (to_hex("red"), "--", "threshold") + assert vertical.get_transform() is rebuilt.get_xaxis_transform(which="grid") + assert list(vertical.get_xdata()) == [1.5, 1.5] + + def test_rebuilt_figure_can_be_drawn() -> None: _, axes = plt.subplots() axes.plot([1, 2], [3, 4], label="data") @@ -249,18 +305,16 @@ def _image(axes: Any) -> None: axes.imshow([[1, 2], [3, 4]]) -def _text(axes: Any) -> None: - axes.plot([1, 2], [3, 4]) - axes.text(1, 3, "note") +def _figure_text(axes: Any) -> None: + axes.text(0.5, 0.5, "note", transform=axes.figure.transFigure) -def _annotation(axes: Any) -> None: - axes.annotate("note", (1, 3)) +def _figure_annotation(axes: Any) -> None: + axes.annotate("note", (0.5, 0.5), xycoords="figure fraction") -def _axhline(axes: Any) -> None: - axes.plot([1, 2], [3, 4]) - axes.axhline(3) +def _annotation_text_in_pixels(axes: Any) -> None: + axes.annotate("note", (1, 3), xytext=(5, 5), textcoords="offset pixels") def _axline(axes: Any) -> None: @@ -326,9 +380,9 @@ def _function_scale(axes: Any) -> None: (_bar_with_errors, "errorbar"), (_stem, "stem"), (_image, "images"), - (_text, "text"), - (_annotation, "text"), - (_axhline, "axhline"), + (_figure_text, "data or axes coordinates"), + (_figure_annotation, "xycoords"), + (_annotation_text_in_pixels, "textcoords"), (_axline, "AxLine"), (_axes_coordinates, "data coordinates"), (_blended_coordinates, "data coordinates"), @@ -395,6 +449,9 @@ def valid_data() -> dict[str, Any]: axes.plot([1, 2], [3, 4], label="line") axes.scatter([1], [2]) axes.bar([1], [2]) + axes.axhline(1) + axes.text(1, 2, "text") + axes.annotate("annotation", (1, 2), arrowprops={}) axes.legend() return json.loads(json.dumps(to_json(axes)))["_value"] @@ -455,6 +512,17 @@ def change(data: dict[str, Any]) -> None: _set(["axes", 0, "scatters", 0, "x"], []), _set(["axes", 0, "legend", "labels"], "line"), _set(["axes", 0, "title"], "title"), + _delete(["axes", 0, "texts"]), + _set(["axes", 0, "texts", 0, "kind"], "arrow"), + _set(["axes", 0, "texts", 0, "coordinates"], "figure"), + _set(["axes", 0, "texts", 0, "point"], [1]), + _set(["axes", 0, "texts", 0, "text"], 5), + _set(["axes", 0, "texts", 0, "horizontal_alignment"], "middle"), + _set(["axes", 0, "texts", 1, "text_coordinates"], "offset pixels"), + _set(["axes", 0, "texts", 1, "arrow"], "yes"), + _set(["axes", 0, "reference_lines", 0, "orientation"], "diagonal"), + _set(["axes", 0, "reference_lines", 0, "extent"], [0]), + _set(["axes", 0, "reference_lines", 0, "value"], float("inf")), ], ) def test_malformed_data_raises_value_error(change: Callable[[dict[str, Any]], None]) -> None: diff --git a/tests/test_plot_assertions.py b/tests/test_plot_assertions.py index 146a034..09e2fa3 100644 --- a/tests/test_plot_assertions.py +++ b/tests/test_plot_assertions.py @@ -41,6 +41,10 @@ def full_plot(axes: Any) -> None: axes.set_ylabel("Output") axes.set_title("Results") axes.set_title("left", loc="left") + axes.axhline(50, color="gray", linestyle=":", label="threshold") + axes.axvline(5) + axes.text(2, 80, "note") + axes.annotate("peak", xy=(10, 100), xytext=(0, -20), textcoords="offset points", arrowprops={"arrowstyle": "->"}) axes.set_xlim(-1, 12) axes.legend(title="Key") @@ -64,10 +68,36 @@ def full_plot(axes: Any) -> None: pytest.param(lambda a: (semilogy(a), a.plot(X, X)), semilogy, {"allow_extra": True}, id="allow-extra"), pytest.param(lambda a: (a.semilogy(X, np.exp(X), "r:"), a.set_xlabel("other")), semilogy, {}, id="labels-and-styles-unchecked"), pytest.param(lambda a: a.plot(X, X), semilogy, {"check_scales": False, "check_data": False}, id="nothing-checked"), + pytest.param(lambda a: (semilogy(a), a.text(3, 10, "e^x")), semilogy, {}, id="extra-text-unchecked"), + pytest.param( + lambda a: (semilogy(a), a.text(1, 1, "note"), a.annotate("max", xy=(10, np.exp(10)), xytext=(1, 1))), + lambda a: (semilogy(a), a.text(5, 5, "note"), a.annotate("max", xy=(10, np.exp(10)))), + {"check_texts": True}, + id="text-position-free", + ), + pytest.param( + lambda a: (semilogy(a), a.text(1, 1, "extra")), + semilogy, + {"check_texts": True, "allow_extra": True}, + id="extra-text-allowed", + ), + pytest.param( + lambda a: (a.axvline(2), a.axhline(1, color="red"), semilogy(a)), + lambda a: (semilogy(a), a.axhline(1), a.axvline(2)), + {}, + id="reference-lines", + ), pytest.param( full_plot, full_plot, - {"check_labels": True, "check_titles": True, "check_legend": True, "check_limits": True, "check_styles": True}, + { + "check_labels": True, + "check_titles": True, + "check_legend": True, + "check_limits": True, + "check_styles": True, + "check_texts": True, + }, id="everything", ), ], @@ -95,6 +125,7 @@ def test_plot_sent_from_the_sandbox_matches() -> None: check_legend=True, check_limits=True, check_styles=True, + check_texts=True, ) @@ -208,6 +239,41 @@ def test_plot_sent_from_the_sandbox_matches() -> None: "Expected the expected bar plot in your plot to have colors #1f77b4ff, but found #ff0000ff.", id="bar-color", ), + pytest.param( + semilogy, + lambda a: (semilogy(a), a.axhline(100)), + {}, + "Expected 1 horizontal line in your plot, but found 0.", + id="missing-axhline", + ), + pytest.param( + lambda a: (semilogy(a), a.axvline(4)), + lambda a: (semilogy(a), a.axvline(5)), + {}, + "No vertical line in your plot matches the expected vertical line.", + id="wrong-axvline", + ), + pytest.param( + lambda a: (semilogy(a), a.text(1, 1, "growth")), + lambda a: (semilogy(a), a.text(1, 1, "Growth")), + {"check_texts": True}, + "Expected the text 'Growth' in your plot, but didn't find it.", + id="text-string", + ), + pytest.param( + lambda a: (semilogy(a), a.text(1, 1, "a"), a.text(1, 1, "b")), + lambda a: (semilogy(a), a.text(1, 1, "a")), + {"check_texts": True}, + "Expected 1 text in your plot, but found 2.", + id="text-count", + ), + pytest.param( + lambda a: (semilogy(a), a.annotate("max", xy=(9, np.exp(9)))), + lambda a: (semilogy(a), a.annotate("max", xy=(10, np.exp(10)))), + {"check_texts": True}, + "Expected the annotation 'max' in your plot to point at (10, 22026.5), but it points at (9, 8103.08).", + id="annotation-point", + ), pytest.param( lambda a: a.plot(X, X), semilogy,