diff --git a/.github/workflows/nightly-v2.yml b/.github/workflows/nightly-v2.yml index a612593abe..f4f9e38cbd 100644 --- a/.github/workflows/nightly-v2.yml +++ b/.github/workflows/nightly-v2.yml @@ -759,6 +759,36 @@ jobs: if-no-files-found: ignore retention-days: 14 + windows: + name: "t3:windows | fast unit tests (windows-latest, py3.12)" + runs-on: windows-latest + timeout-minutes: 50 + steps: + - name: Checkout code + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 + + - name: Install dependencies + run: uv sync --group dev + + - name: Run fast tests + run: uv run -- python -m pytest tests -m "fast and not (slow or stress or resource_heavy or live_integration)" --tb=short --timeout=120 --junitxml=junit-fast-windows.xml + + - name: Upload fast test report + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: t3-windows + path: junit-fast-windows.xml + if-no-files-found: ignore + retention-days: 14 durations-refresh: name: "t3:durations | duration refresh" runs-on: ubuntu-latest @@ -855,6 +885,7 @@ jobs: - quarantine - linkcheck - liveness + - windows - durations-refresh if: ${{ always() && !cancelled() }} runs-on: ubuntu-latest @@ -872,7 +903,7 @@ jobs: shell: bash run: | set -euo pipefail - for job in docker cloud perf matrix browser extension install drift quarantine linkcheck liveness durations-refresh; do + for job in docker cloud perf matrix browser extension install drift quarantine linkcheck liveness windows durations-refresh; do case "$job" in durations-refresh) domain=durations ;; *) domain="$job" ;; diff --git a/docs/internal/nightly-t3.md b/docs/internal/nightly-t3.md index 41328bd47f..074e12e1fb 100644 --- a/docs/internal/nightly-t3.md +++ b/docs/internal/nightly-t3.md @@ -24,6 +24,7 @@ red domain does not hide the others, and each domain uploads its own artifacts. | `t3:quarantine` | `quarantine` | Runs tests marked `quarantine`. It passes with a notice until the marker is registered in `pytest.ini`. | | `t3:linkcheck` | `linkcheck` | Checks external documentation links with Sphinx and uploads the linkcheck report. | | `t3:liveness` | `liveness` | Fails when a workflow with a `schedule:` trigger has no scheduled run inside its cadence window, and prints the newest ten scheduled runs of any workflow it flags. Runs `scripts/scheduled_workflow_liveness.py`. | +| `t3:windows` | `windows` | The fast unit tier on `windows-latest` with Python 3.12 (no coverage gate). The matrix domain only builds and smoke-tests on Windows; this is the Windows unit suite. | | `t3:durations` | `durations-refresh` | Records pytest `--durations` and JUnit reports for the fast and slow tiers in the `t3-durations` artifact. It also regenerates the duration file there when the tooling is present. Failed sampled tests retain their artifacts and fail the domain; missing verdicts fail closed. | ## Live cloud tests are opt-in diff --git a/tests/unit/workflows/test_nightly_t3_workflow.py b/tests/unit/workflows/test_nightly_t3_workflow.py index 417ddbccdd..08e802b615 100644 --- a/tests/unit/workflows/test_nightly_t3_workflow.py +++ b/tests/unit/workflows/test_nightly_t3_workflow.py @@ -4,6 +4,7 @@ import json import os import re +import shlex import shutil import subprocess import sys @@ -33,6 +34,7 @@ "quarantine": "t3:quarantine", "linkcheck": "t3:linkcheck", "liveness": "t3:liveness", + "windows": "t3:windows", "durations-refresh": "t3:durations", } @@ -446,14 +448,15 @@ def test_perf_comparison_uses_only_the_current_job_result(tmp_path: Path, result setup["run"], tmp_path, {"RUNNER_TEMP": str(runner_temp), "GITHUB_ENV": str(github_env)} ) assert setup_result.returncode == 0, setup_result.stdout + setup_result.stderr - key, _, configured_root = github_env.read_text().strip().partition("=") + key, _, configured_root = github_env.read_text(encoding="utf-8").strip().partition("=") assert key == "BENCHBOX_OUTPUT_DIR" and Path(configured_root) == output_root step = next(step for step in _steps(job) if step.get("id") == "current") github_output = tmp_path / "github-output" result = _run_workflow_script(step["run"], tmp_path, {key: configured_root, "GITHUB_OUTPUT": str(github_output)}) assert (result.returncode == 0) == (result_count == 1), result.stdout + result.stderr if result_count == 1: - assert github_output.read_text().strip() == f"path={results / 'tpch_sf001_duckdb_sql_0.json'}" + published = github_output.read_text(encoding="utf-8").strip().removeprefix("path=") + assert Path(published) == results / "tpch_sf001_duckdb_sql_0.json" @pytest.mark.parametrize("tier", ["fast", "slow"]) @@ -483,9 +486,11 @@ def test_duration_measurement_keeps_assertion_reports_and_rejects_runner_failure ) result = _run_workflow_script(step["run"], tmp_path) assert result.returncode == expected, result.stdout + result.stderr - assert (tmp_path / "t3-durations" / f"junit-{tier}.xml").read_text() == '' + assert (tmp_path / "t3-durations" / f"junit-{tier}.xml").read_text(encoding="utf-8") == '' if tee_exit == 0: - assert (tmp_path / "t3-durations" / f"pytest-{tier}.exit").read_text().strip() == str(pytest_exit) + assert (tmp_path / "t3-durations" / f"pytest-{tier}.exit").read_text(encoding="utf-8").strip() == str( + pytest_exit + ) @pytest.mark.parametrize("compare_exit", [0, 1]) @@ -501,7 +506,7 @@ def test_perf_comparison_preserves_threshold_and_failure(tmp_path: Path, compare script = 'uv() { printf "%s\\n" "$@" > "$ARGUMENTS"; return "$COMPARE_EXIT"; }\n' + script result = _run_workflow_script(script, tmp_path, {"ARGUMENTS": str(arguments), "COMPARE_EXIT": str(compare_exit)}) assert result.returncode == compare_exit, result.stdout + result.stderr - assert arguments.read_text().splitlines() == [ + assert arguments.read_text(encoding="utf-8").splitlines() == [ "run", "benchbox", "compare", @@ -563,14 +568,14 @@ def test_failed_duration_samples_keep_both_reports_and_regenerated_artifact( env = {"FAST_EXIT": str(fast_exit), "SLOW_EXIT": str(slow_exit)} for name in ["Measure fast tier durations", "Measure slow tier durations", "Regenerate duration file"]: step = next(step for step in steps if step.get("name") == name) - script = step["run"].replace("uv run -- python", f"uv run -- {sys.executable}") + script = step["run"].replace("uv run -- python", f"uv run -- {shlex.quote(Path(sys.executable).as_posix())}") result = _run_workflow_script(script, tmp_path, env) assert result.returncode == 0, result.stdout + result.stderr output = tmp_path / "t3-durations" for tier in ["fast", "slow"]: assert (output / f"junit-{tier}.xml").is_file() - assert (output / f"durations-{tier}.txt").read_text().strip() == f"measurement {tier}" - artifact = json.loads((output / "test_durations.json").read_text()) + assert (output / f"durations-{tier}.txt").read_text(encoding="utf-8").strip() == f"measurement {tier}" + artifact = json.loads((output / "test_durations.json").read_text(encoding="utf-8")) assert len(artifact["tests"]) == 2 final = next(step for step in steps if step.get("name") == "Require successful sampled tests") result = _run_workflow_script(final["run"], tmp_path) @@ -714,3 +719,20 @@ def test_liveness_domain_checks_out_the_default_branch_that_scheduled_runs_come_ ) assert checkout["with"]["ref"] == "develop" + + +def test_windows_domain_runs_the_fast_tier_without_continue_on_error() -> None: + job = _load()["jobs"]["windows"] + text = _run_text(job) + + assert job["runs-on"] == "windows-latest" + assert "continue-on-error" not in job + assert "fast and not (slow or stress or resource_heavy or live_integration)" in text + assert "python-version" in str([step.get("with") for step in _steps(job)]) + assert isinstance(job["timeout-minutes"], int) and job["timeout-minutes"] <= 60 + + +def test_windows_domain_timeout_leaves_margin_over_the_measured_run() -> None: + measured_run_minutes = 38.13 + + assert _load()["jobs"]["windows"]["timeout-minutes"] >= measured_run_minutes * 1.25