Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion .github/workflows/nightly-v2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -759,6 +759,36 @@ jobs:
if-no-files-found: ignore
retention-days: 14

windows:
name: "t3:windows | fast unit tests (windows-latest, py3.12)"
runs-on: windows-latest
timeout-minutes: 50
steps:
- name: Checkout code
uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09

- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
with:
python-version: '3.12'

- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78

- name: Install dependencies
run: uv sync --group dev

- name: Run fast tests
run: uv run -- python -m pytest tests -m "fast and not (slow or stress or resource_heavy or live_integration)" --tb=short --timeout=120 --junitxml=junit-fast-windows.xml

- name: Upload fast test report
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
with:
name: t3-windows
path: junit-fast-windows.xml
if-no-files-found: ignore
retention-days: 14
durations-refresh:
name: "t3:durations | duration refresh"
runs-on: ubuntu-latest
Expand Down Expand Up @@ -855,6 +885,7 @@ jobs:
- quarantine
- linkcheck
- liveness
- windows
- durations-refresh
if: ${{ always() && !cancelled() }}
runs-on: ubuntu-latest
Expand All @@ -872,7 +903,7 @@ jobs:
shell: bash
run: |
set -euo pipefail
for job in docker cloud perf matrix browser extension install drift quarantine linkcheck liveness durations-refresh; do
for job in docker cloud perf matrix browser extension install drift quarantine linkcheck liveness windows durations-refresh; do
case "$job" in
durations-refresh) domain=durations ;;
*) domain="$job" ;;
Expand Down
1 change: 1 addition & 0 deletions docs/internal/nightly-t3.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ red domain does not hide the others, and each domain uploads its own artifacts.
| `t3:quarantine` | `quarantine` | Runs tests marked `quarantine`. It passes with a notice until the marker is registered in `pytest.ini`. |
| `t3:linkcheck` | `linkcheck` | Checks external documentation links with Sphinx and uploads the linkcheck report. |
| `t3:liveness` | `liveness` | Fails when a workflow with a `schedule:` trigger has no scheduled run inside its cadence window, and prints the newest ten scheduled runs of any workflow it flags. Runs `scripts/scheduled_workflow_liveness.py`. |
| `t3:windows` | `windows` | The fast unit tier on `windows-latest` with Python 3.12 (no coverage gate). The matrix domain only builds and smoke-tests on Windows; this is the Windows unit suite. |
| `t3:durations` | `durations-refresh` | Records pytest `--durations` and JUnit reports for the fast and slow tiers in the `t3-durations` artifact. It also regenerates the duration file there when the tooling is present. Failed sampled tests retain their artifacts and fail the domain; missing verdicts fail closed. |

## Live cloud tests are opt-in
Expand Down
38 changes: 30 additions & 8 deletions tests/unit/workflows/test_nightly_t3_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import json
import os
import re
import shlex
import shutil
import subprocess
import sys
Expand Down Expand Up @@ -33,6 +34,7 @@
"quarantine": "t3:quarantine",
"linkcheck": "t3:linkcheck",
"liveness": "t3:liveness",
"windows": "t3:windows",
"durations-refresh": "t3:durations",
}

Expand Down Expand Up @@ -446,14 +448,15 @@ def test_perf_comparison_uses_only_the_current_job_result(tmp_path: Path, result
setup["run"], tmp_path, {"RUNNER_TEMP": str(runner_temp), "GITHUB_ENV": str(github_env)}
)
assert setup_result.returncode == 0, setup_result.stdout + setup_result.stderr
key, _, configured_root = github_env.read_text().strip().partition("=")
key, _, configured_root = github_env.read_text(encoding="utf-8").strip().partition("=")
assert key == "BENCHBOX_OUTPUT_DIR" and Path(configured_root) == output_root
step = next(step for step in _steps(job) if step.get("id") == "current")
github_output = tmp_path / "github-output"
result = _run_workflow_script(step["run"], tmp_path, {key: configured_root, "GITHUB_OUTPUT": str(github_output)})
assert (result.returncode == 0) == (result_count == 1), result.stdout + result.stderr
if result_count == 1:
assert github_output.read_text().strip() == f"path={results / 'tpch_sf001_duckdb_sql_0.json'}"
published = github_output.read_text(encoding="utf-8").strip().removeprefix("path=")
assert Path(published) == results / "tpch_sf001_duckdb_sql_0.json"


@pytest.mark.parametrize("tier", ["fast", "slow"])
Expand Down Expand Up @@ -483,9 +486,11 @@ def test_duration_measurement_keeps_assertion_reports_and_rejects_runner_failure
)
result = _run_workflow_script(step["run"], tmp_path)
assert result.returncode == expected, result.stdout + result.stderr
assert (tmp_path / "t3-durations" / f"junit-{tier}.xml").read_text() == '<testsuite tests="1"/>'
assert (tmp_path / "t3-durations" / f"junit-{tier}.xml").read_text(encoding="utf-8") == '<testsuite tests="1"/>'
if tee_exit == 0:
assert (tmp_path / "t3-durations" / f"pytest-{tier}.exit").read_text().strip() == str(pytest_exit)
assert (tmp_path / "t3-durations" / f"pytest-{tier}.exit").read_text(encoding="utf-8").strip() == str(
pytest_exit
)


@pytest.mark.parametrize("compare_exit", [0, 1])
Expand All @@ -501,7 +506,7 @@ def test_perf_comparison_preserves_threshold_and_failure(tmp_path: Path, compare
script = 'uv() { printf "%s\\n" "$@" > "$ARGUMENTS"; return "$COMPARE_EXIT"; }\n' + script
result = _run_workflow_script(script, tmp_path, {"ARGUMENTS": str(arguments), "COMPARE_EXIT": str(compare_exit)})
assert result.returncode == compare_exit, result.stdout + result.stderr
assert arguments.read_text().splitlines() == [
assert arguments.read_text(encoding="utf-8").splitlines() == [
"run",
"benchbox",
"compare",
Expand Down Expand Up @@ -563,14 +568,14 @@ def test_failed_duration_samples_keep_both_reports_and_regenerated_artifact(
env = {"FAST_EXIT": str(fast_exit), "SLOW_EXIT": str(slow_exit)}
for name in ["Measure fast tier durations", "Measure slow tier durations", "Regenerate duration file"]:
step = next(step for step in steps if step.get("name") == name)
script = step["run"].replace("uv run -- python", f"uv run -- {sys.executable}")
script = step["run"].replace("uv run -- python", f"uv run -- {shlex.quote(Path(sys.executable).as_posix())}")
result = _run_workflow_script(script, tmp_path, env)
assert result.returncode == 0, result.stdout + result.stderr
output = tmp_path / "t3-durations"
for tier in ["fast", "slow"]:
assert (output / f"junit-{tier}.xml").is_file()
assert (output / f"durations-{tier}.txt").read_text().strip() == f"measurement {tier}"
artifact = json.loads((output / "test_durations.json").read_text())
assert (output / f"durations-{tier}.txt").read_text(encoding="utf-8").strip() == f"measurement {tier}"
artifact = json.loads((output / "test_durations.json").read_text(encoding="utf-8"))
assert len(artifact["tests"]) == 2
final = next(step for step in steps if step.get("name") == "Require successful sampled tests")
result = _run_workflow_script(final["run"], tmp_path)
Expand Down Expand Up @@ -714,3 +719,20 @@ def test_liveness_domain_checks_out_the_default_branch_that_scheduled_runs_come_
)

assert checkout["with"]["ref"] == "develop"


def test_windows_domain_runs_the_fast_tier_without_continue_on_error() -> None:
job = _load()["jobs"]["windows"]
text = _run_text(job)

assert job["runs-on"] == "windows-latest"
assert "continue-on-error" not in job
assert "fast and not (slow or stress or resource_heavy or live_integration)" in text
assert "python-version" in str([step.get("with") for step in _steps(job)])
assert isinstance(job["timeout-minutes"], int) and job["timeout-minutes"] <= 60


def test_windows_domain_timeout_leaves_margin_over_the_measured_run() -> None:
measured_run_minutes = 38.13

assert _load()["jobs"]["windows"]["timeout-minutes"] >= measured_run_minutes * 1.25
Loading