diff --git a/quick_start.md b/quick_start.md index 5c694c3..7491d70 100644 --- a/quick_start.md +++ b/quick_start.md @@ -168,7 +168,9 @@ tests/ └── data.json (if the tests need params) ``` -Then run `pytest tests/test_student.py`. +Then run `pytest tests/test_student.py`. To run the tests of several questions at once, pass their +folder, such as `pytest questions/`. The grader uses pytest's `importlib` import mode, so each +question's `test_student.py` is imported separately, unless you pass a different `--import-mode`. ## Where Code Belongs diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 07c0c13..6352b15 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -959,6 +959,12 @@ def pytest_runtest_setup(item): raise ValueError(f"benchmark mark can't have {name!r} keyword argument.") +def _import_mode_was_chosen(config: Config) -> bool: + """Whether the user passed --import-mode, on the command line, in addopts or in PYTEST_ADDOPTS.""" + args = [*config.invocation_params.args, *config.getini("addopts"), *os.environ.get("PYTEST_ADDOPTS", "").split()] + return any(arg == "--import-mode" or arg.startswith("--import-mode=") for arg in args) + + @pytest.hookimpl(trylast=True) # force the other plugins to initialise, fixes issue with capture not being properly initialized def pytest_configure(config: Config) -> None: # config.addinivalue_line("markers", "benchmark: mark a test with custom benchmark settings.") @@ -970,6 +976,12 @@ def pytest_configure(config: Config) -> None: config.addinivalue_line("markers", "sandbox_timeout(timeout_value): sets a timeout for initialization of the sandbox fixture") config.addinivalue_line("markers", "output(level): controls the detail level of grading failure output") + # Every question's tests are in a file named test_student.py. pytest's default import mode + # imports each one as the module `test_student`, so a run over several questions stops with an + # "import file mismatch" error. The importlib mode names each module by its path instead. + if not _import_mode_was_chosen(config): + config.option.importmode = "importlib" + # Only register our plugin if it hasn't been already (e.g., in case of multiple conftests) if not hasattr(config, "result_collector_plugin"): config.result_collector_plugin = ResultCollectorPlugin() # type: ignore[attr-defined] diff --git a/tests/test_import_mode.py b/tests/test_import_mode.py new file mode 100644 index 0000000..30e0a11 --- /dev/null +++ b/tests/test_import_mode.py @@ -0,0 +1,33 @@ +import pytest + +pytest_plugins = ("pytester",) + +TESTS = """ +import pytest + + +@pytest.mark.grading_data(name="double", points=1) +def test_double(sandbox) -> None: + assert sandbox.query_function("double", 2) == 4 +""" + + +def make_questions(pytester: pytest.Pytester) -> None: + """Make two questions whose tests are both in a file named test_student.py.""" + for question in ["first", "second"]: + tests_dir = pytester.mkdir(question) + tests_dir.joinpath("test_student.py").write_text(TESTS) + tests_dir.joinpath("test_student").mkdir() + tests_dir.joinpath("test_student", "student_code.py").write_text("def double(x):\n return 2 * x\n") + + +def test_questions_with_the_same_test_file_name_run_together(pytester: pytest.Pytester) -> None: + make_questions(pytester) + pytester.runpytest().assert_outcomes(passed=2) + + +def test_chosen_import_mode_is_kept(pytester: pytest.Pytester) -> None: + make_questions(pytester) + result = pytester.runpytest("--import-mode=prepend") + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines(["*import file mismatch*"])