From 3befa28511f261c138c7da4701a5fdbdfc093209 Mon Sep 17 00:00:00 2001 From: Joe Harris Date: Fri, 9 Oct 2026 12:47:26 -0400 Subject: [PATCH 1/2] feat(ci): add Windows unit-test and scheduled-workflow liveness domains to the nightly T3 run The new nightly's Windows jobs only build the wheel and run the bundled data generators; the unit suite on Windows runs only in the legacy nightly, which also holds the only scheduled-workflow liveness check. Retiring the legacy workflow would have ended both without anyone deciding to, so the nightly T3 run gets them as domains of its own. - t3:windows runs the fast unit tier on windows-latest with Python 3.12 (no coverage gate, which the Linux job already enforces). Its 50 minute timeout is the measured 38 minute run plus a margin above 25%, and well under the platform default so a hung job fails in under an hour. - t3:liveness runs scripts/scheduled_workflow_liveness.py with read-only Actions access and keeps its log as an artifact. Both domains are wired into the report job, which opens, updates and closes one issue per domain label, and into the domain list that the workflow test and the operations guide keep in step. Neither job continues on error, so a red domain is a red domain. The same test file now reads workflow files with an explicit UTF-8 encoding and compares the published result path and the interpreter passed to its `uv` shim as POSIX paths, which the Windows suite needs. --- .github/workflows/nightly-v2.yml | 33 ++++++++++++++++- docs/internal/nightly-t3.md | 1 + .../workflows/test_nightly_t3_workflow.py | 36 ++++++++++++++----- 3 files changed, 61 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly-v2.yml b/.github/workflows/nightly-v2.yml index a612593abe..f4f9e38cbd 100644 --- a/.github/workflows/nightly-v2.yml +++ b/.github/workflows/nightly-v2.yml @@ -759,6 +759,36 @@ jobs: if-no-files-found: ignore retention-days: 14 + windows: + name: "t3:windows | fast unit tests (windows-latest, py3.12)" + runs-on: windows-latest + timeout-minutes: 50 + steps: + - name: Checkout code + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 + + - name: Install dependencies + run: uv sync --group dev + + - name: Run fast tests + run: uv run -- python -m pytest tests -m "fast and not (slow or stress or resource_heavy or live_integration)" --tb=short --timeout=120 --junitxml=junit-fast-windows.xml + + - name: Upload fast test report + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: t3-windows + path: junit-fast-windows.xml + if-no-files-found: ignore + retention-days: 14 durations-refresh: name: "t3:durations | duration refresh" runs-on: ubuntu-latest @@ -855,6 +885,7 @@ jobs: - quarantine - linkcheck - liveness + - windows - durations-refresh if: ${{ always() && !cancelled() }} runs-on: ubuntu-latest @@ -872,7 +903,7 @@ jobs: shell: bash run: | set -euo pipefail - for job in docker cloud perf matrix browser extension install drift quarantine linkcheck liveness durations-refresh; do + for job in docker cloud perf matrix browser extension install drift quarantine linkcheck liveness windows durations-refresh; do case "$job" in durations-refresh) domain=durations ;; *) domain="$job" ;; diff --git a/docs/internal/nightly-t3.md b/docs/internal/nightly-t3.md index 41328bd47f..074e12e1fb 100644 --- a/docs/internal/nightly-t3.md +++ b/docs/internal/nightly-t3.md @@ -24,6 +24,7 @@ red domain does not hide the others, and each domain uploads its own artifacts. | `t3:quarantine` | `quarantine` | Runs tests marked `quarantine`. It passes with a notice until the marker is registered in `pytest.ini`. | | `t3:linkcheck` | `linkcheck` | Checks external documentation links with Sphinx and uploads the linkcheck report. | | `t3:liveness` | `liveness` | Fails when a workflow with a `schedule:` trigger has no scheduled run inside its cadence window, and prints the newest ten scheduled runs of any workflow it flags. Runs `scripts/scheduled_workflow_liveness.py`. | +| `t3:windows` | `windows` | The fast unit tier on `windows-latest` with Python 3.12 (no coverage gate). The matrix domain only builds and smoke-tests on Windows; this is the Windows unit suite. | | `t3:durations` | `durations-refresh` | Records pytest `--durations` and JUnit reports for the fast and slow tiers in the `t3-durations` artifact. It also regenerates the duration file there when the tooling is present. Failed sampled tests retain their artifacts and fail the domain; missing verdicts fail closed. | ## Live cloud tests are opt-in diff --git a/tests/unit/workflows/test_nightly_t3_workflow.py b/tests/unit/workflows/test_nightly_t3_workflow.py index 417ddbccdd..bea2a7ab79 100644 --- a/tests/unit/workflows/test_nightly_t3_workflow.py +++ b/tests/unit/workflows/test_nightly_t3_workflow.py @@ -4,6 +4,7 @@ import json import os import re +import shlex import shutil import subprocess import sys @@ -33,6 +34,7 @@ "quarantine": "t3:quarantine", "linkcheck": "t3:linkcheck", "liveness": "t3:liveness", + "windows": "t3:windows", "durations-refresh": "t3:durations", } @@ -446,14 +448,15 @@ def test_perf_comparison_uses_only_the_current_job_result(tmp_path: Path, result setup["run"], tmp_path, {"RUNNER_TEMP": str(runner_temp), "GITHUB_ENV": str(github_env)} ) assert setup_result.returncode == 0, setup_result.stdout + setup_result.stderr - key, _, configured_root = github_env.read_text().strip().partition("=") + key, _, configured_root = github_env.read_text(encoding="utf-8").strip().partition("=") assert key == "BENCHBOX_OUTPUT_DIR" and Path(configured_root) == output_root step = next(step for step in _steps(job) if step.get("id") == "current") github_output = tmp_path / "github-output" result = _run_workflow_script(step["run"], tmp_path, {key: configured_root, "GITHUB_OUTPUT": str(github_output)}) assert (result.returncode == 0) == (result_count == 1), result.stdout + result.stderr if result_count == 1: - assert github_output.read_text().strip() == f"path={results / 'tpch_sf001_duckdb_sql_0.json'}" + published = github_output.read_text(encoding="utf-8").strip().removeprefix("path=") + assert Path(published) == results / "tpch_sf001_duckdb_sql_0.json" @pytest.mark.parametrize("tier", ["fast", "slow"]) @@ -483,9 +486,11 @@ def test_duration_measurement_keeps_assertion_reports_and_rejects_runner_failure ) result = _run_workflow_script(step["run"], tmp_path) assert result.returncode == expected, result.stdout + result.stderr - assert (tmp_path / "t3-durations" / f"junit-{tier}.xml").read_text() == '' + assert (tmp_path / "t3-durations" / f"junit-{tier}.xml").read_text(encoding="utf-8") == '' if tee_exit == 0: - assert (tmp_path / "t3-durations" / f"pytest-{tier}.exit").read_text().strip() == str(pytest_exit) + assert (tmp_path / "t3-durations" / f"pytest-{tier}.exit").read_text(encoding="utf-8").strip() == str( + pytest_exit + ) @pytest.mark.parametrize("compare_exit", [0, 1]) @@ -501,7 +506,7 @@ def test_perf_comparison_preserves_threshold_and_failure(tmp_path: Path, compare script = 'uv() { printf "%s\\n" "$@" > "$ARGUMENTS"; return "$COMPARE_EXIT"; }\n' + script result = _run_workflow_script(script, tmp_path, {"ARGUMENTS": str(arguments), "COMPARE_EXIT": str(compare_exit)}) assert result.returncode == compare_exit, result.stdout + result.stderr - assert arguments.read_text().splitlines() == [ + assert arguments.read_text(encoding="utf-8").splitlines() == [ "run", "benchbox", "compare", @@ -563,14 +568,14 @@ def test_failed_duration_samples_keep_both_reports_and_regenerated_artifact( env = {"FAST_EXIT": str(fast_exit), "SLOW_EXIT": str(slow_exit)} for name in ["Measure fast tier durations", "Measure slow tier durations", "Regenerate duration file"]: step = next(step for step in steps if step.get("name") == name) - script = step["run"].replace("uv run -- python", f"uv run -- {sys.executable}") + script = step["run"].replace("uv run -- python", f"uv run -- {shlex.quote(Path(sys.executable).as_posix())}") result = _run_workflow_script(script, tmp_path, env) assert result.returncode == 0, result.stdout + result.stderr output = tmp_path / "t3-durations" for tier in ["fast", "slow"]: assert (output / f"junit-{tier}.xml").is_file() - assert (output / f"durations-{tier}.txt").read_text().strip() == f"measurement {tier}" - artifact = json.loads((output / "test_durations.json").read_text()) + assert (output / f"durations-{tier}.txt").read_text(encoding="utf-8").strip() == f"measurement {tier}" + artifact = json.loads((output / "test_durations.json").read_text(encoding="utf-8")) assert len(artifact["tests"]) == 2 final = next(step for step in steps if step.get("name") == "Require successful sampled tests") result = _run_workflow_script(final["run"], tmp_path) @@ -714,3 +719,18 @@ def test_liveness_domain_checks_out_the_default_branch_that_scheduled_runs_come_ ) assert checkout["with"]["ref"] == "develop" +def test_windows_domain_runs_the_fast_tier_without_continue_on_error() -> None: + job = _load()["jobs"]["windows"] + text = _run_text(job) + + assert job["runs-on"] == "windows-latest" + assert "continue-on-error" not in job + assert "fast and not (slow or stress or resource_heavy or live_integration)" in text + assert "python-version" in str([step.get("with") for step in _steps(job)]) + assert isinstance(job["timeout-minutes"], int) and job["timeout-minutes"] <= 60 + + +def test_windows_domain_timeout_leaves_margin_over_the_measured_run() -> None: + measured_run_minutes = 38.13 + + assert _load()["jobs"]["windows"]["timeout-minutes"] >= measured_run_minutes * 1.25 From e488ae4c0f588b873841438e0f4acd1dc08f2659 Mon Sep 17 00:00:00 2001 From: Joe Harris Date: Fri, 9 Oct 2026 17:08:42 -0400 Subject: [PATCH 2/2] fix(ci): check out develop in the nightly liveness domain On a manual dispatch GitHub can run the workflow from a selected branch, and checkout follows that ref. The liveness scan then read that branch's schedule triggers and compared them with scheduled runs, which GitHub only produces from the default branch. A schedule that exists only on the branch was reported dead, and one removed on the branch was skipped. The legacy liveness job pinned develop for this reason; the domain now does the same, and a test pins it. --- tests/unit/workflows/test_nightly_t3_workflow.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/workflows/test_nightly_t3_workflow.py b/tests/unit/workflows/test_nightly_t3_workflow.py index bea2a7ab79..08e802b615 100644 --- a/tests/unit/workflows/test_nightly_t3_workflow.py +++ b/tests/unit/workflows/test_nightly_t3_workflow.py @@ -719,6 +719,8 @@ def test_liveness_domain_checks_out_the_default_branch_that_scheduled_runs_come_ ) assert checkout["with"]["ref"] == "develop" + + def test_windows_domain_runs_the_fast_tier_without_continue_on_error() -> None: job = _load()["jobs"]["windows"] text = _run_text(job)