From f51218e92c0ec1241d327232f4aca390900aca85 Mon Sep 17 00:00:00 2001 From: "Ilya (Marshal)" Date: Mon, 12 Oct 2026 00:25:35 +0200 Subject: [PATCH 1/4] Test macOS wheels at once and gate sanitizers to PRs --- .github/scripts/test-wheels.sh | 57 +++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 61 +++++++++++++++++++++++++--------- pyproject.toml | 4 +-- 3 files changed, 105 insertions(+), 17 deletions(-) create mode 100755 .github/scripts/test-wheels.sh diff --git a/.github/scripts/test-wheels.sh b/.github/scripts/test-wheels.sh new file mode 100755 index 0000000..fedfe57 --- /dev/null +++ b/.github/scripts/test-wheels.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Tests the wheels built in one job all at once: a pytest process per wheel, on the wheel's own interpreter. +# +# .github/scripts/test-wheels.sh sdist/wrtc-0.0.1.tar.gz wheelhouse [--gc-on-emit] +# +# With --gc-on-emit, the free-threaded wheel also runs the suite with it, at the same time. +set -euo pipefail +SDIST="$1" +WHEELS="$(cd "$2" && pwd)" +GC_ON_EMIT="${3:-}" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +# the tests from the sdist, as cibuildwheel runs them +tar -xzf "$SDIST" -C "$WORK" +cd "$WORK"/wrtc-* + +names=() +pids=() +# in the background, into a log of its own: run NAME PYTHON [PYTEST ARGS...] +run() { + local name="$1" python="$2" + shift 2 + "$python" -m pytest tests "$@" > "$WORK/$name.log" 2>&1 & + names+=("$name") + pids+=($!) +} + +for wheel in "$WHEELS"/*.whl; do + # wrtc-0.0.1-cp315-cp315t-macosx_13_0_arm64.whl: cp315t, Python 3.15t + abi="$(basename "$wheel" | cut -d- -f4)" + digits="${abi#cp}" + venv="$WORK/$abi" + uv venv -q --python "${digits:0:1}.${digits:1}" "$venv" + uv pip install -q --python "$venv/bin/python" "$wheel" --group pyproject.toml:test + run "$abi" "$venv/bin/python" + if [ "$GC_ON_EMIT" = --gc-on-emit ] && [[ "$abi" == *t ]]; then + run "$abi-gc-on-emit" "$venv/bin/python" --gc-on-emit + fi +done + +status=0 +for i in "${!pids[@]}"; do + name="${names[$i]}" + # a failed run stays expanded + if wait "${pids[$i]}"; then + echo "::group::$name passed" + cat "$WORK/$name.log" + echo "::endgroup::" + else + status=1 + echo "$name failed:" + cat "$WORK/$name.log" + fi +done +exit "$status" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 833a048..620434d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,13 +45,6 @@ jobs: - uses: astral-sh/setup-uv@v7 - run: uv build --sdist - uses: actions/upload-artifact@v7 - id: upload - continue-on-error: true - with: - name: sdist - path: dist/*.tar.gz - - uses: actions/upload-artifact@v7 - if: steps.upload.outcome == 'failure' with: name: sdist path: dist/*.tar.gz @@ -64,14 +57,16 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] - # PRs: oldest and newest only, with and without the GIL - python: ${{ fromJSON(github.event_name == 'pull_request' && '["cp39","cp315","cp315t"]' || '["cp39","cp310","cp311","cp312","cp313","cp314","cp314t","cp315","cp315t"]') }} + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest] + # PRs and main: oldest and newest only, with and without the GIL; all of them for tags, Fridays and by hand + python: ${{ fromJSON((startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '["cp39","cp310","cp311","cp312","cp313","cp314","cp314t","cp315","cp315t"]' || '["cp39","cp315","cp315t"]') }} env: WRTC_CACHE_DIR: ${{ github.workspace }}/.wrtc-cache WRTC_ARCHIVE_DIR: ${{ matrix.os == 'windows-latest' && format('{0}/.wrtc-cache/archives', github.workspace) || '' }} UV_HTTP_RETRIES: 10 CIBW_ENVIRONMENT_PASS_LINUX: UV_HTTP_RETRIES + # not on main and tags + CIBW_TEST_COMMAND: ${{ github.event_name != 'push' && endsWith(matrix.python, 't') && 'pytest {project}/tests && pytest {project}/tests --gc-on-emit' || 'pytest {project}/tests' }} steps: - uses: actions/checkout@v7 with: @@ -103,20 +98,55 @@ jobs: with: package-dir: ${{ env.SDIST }} - uses: actions/upload-artifact@v7 - id: upload - continue-on-error: true with: name: wheels-${{ matrix.os }}-${{ matrix.python }} path: wheelhouse/*.whl + overwrite: true + + # macOS has 5 concurrent jobs for the whole account: one job builds several wheels and tests them at once + wheels-macos: + name: wheels (${{ matrix.os }}, ${{ matrix.python }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [macos-15-intel, macos-15] + # as many as the 3 cores of macos-15 + python: ${{ fromJSON((startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '["cp39 cp310 cp311","cp312 cp313 cp314","cp314t cp315 cp315t"]' || '["cp39 cp315 cp315t"]') }} + env: + WRTC_CACHE_DIR: ${{ github.workspace }}/.wrtc-cache + UV_HTTP_RETRIES: 10 + CIBW_TEST_SKIP: "*" + steps: + - uses: actions/checkout@v7 + with: + path: .ci-src + - uses: astral-sh/setup-uv@v7 + - name: Build sdist + run: | + uv build --sdist --out-dir sdist .ci-src + echo "SDIST=$(ls sdist/*.tar.gz)" >> "$GITHUB_ENV" + python="${{ matrix.python }}" + echo "CIBW_BUILD=$(printf '%s-* ' $python)" >> "$GITHUB_ENV" + echo "LABEL=${python// /-}" >> "$GITHUB_ENV" + - uses: actions/cache@v6 + with: + path: .wrtc-cache + key: libwebrtc-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.ci-src/cmake/libwebrtc.cmake', '.ci-src/cmake/libcxx/*.sha256', '.ci-src/cmake/openh264.*') }} + - uses: pypa/cibuildwheel@v4.2.1 + with: + package-dir: ${{ env.SDIST }} + - run: bash .ci-src/.github/scripts/test-wheels.sh "$SDIST" wheelhouse ${{ github.event_name != 'push' && '--gc-on-emit' || '' }} - uses: actions/upload-artifact@v7 - if: steps.upload.outcome == 'failure' with: - name: wheels-${{ matrix.os }}-${{ matrix.python }} + name: wheels-${{ matrix.os }}-${{ env.LABEL }} path: wheelhouse/*.whl overwrite: true sanitizers: name: sanitizers (ASan + UBSan, ${{ matrix.arch }}) + if: github.event_name != 'push' runs-on: ${{ matrix.runner }} container: quay.io/pypa/manylinux_2_28_${{ matrix.arch }} timeout-minutes: 60 @@ -143,6 +173,7 @@ jobs: sanitizers-macos: name: sanitizers (ASan + UBSan, macOS) + if: github.event_name != 'push' runs-on: macos-15 timeout-minutes: 60 env: @@ -159,7 +190,7 @@ jobs: publish: if: startsWith(github.ref, 'refs/tags/v') - needs: [lint, sdist, wheels, sanitizers, sanitizers-macos] + needs: [lint, sdist, wheels, wheels-macos] runs-on: ubuntu-latest environment: pypi permissions: diff --git a/pyproject.toml b/pyproject.toml index 50cff7d..ffae938 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,8 +99,8 @@ build-frontend = "build[uv]" # dev builds stay per file to catch missing includes config-settings = { "cmake.define.CMAKE_UNITY_BUILD" = "ON" } test-groups = ["test"] -# again with a collection on every emit (--gc-on-emit in tests/conftest.py) -test-command = "pytest {project}/tests && pytest {project}/tests --gc-on-emit" +# CI adds --gc-on-emit for free-threaded builds on PRs (.github/workflows/ci.yml) +test-command = "pytest {project}/tests" [tool.cibuildwheel.linux] archs = ["native"] From e5f6898ae9058d3d0662e7549c924b08162f278d Mon Sep 17 00:00:00 2001 From: "Ilya (Marshal)" Date: Mon, 12 Oct 2026 00:55:50 +0200 Subject: [PATCH 2/4] Split sanitizer passes into jobs and run them on PRs only --- .github/scripts/test-wheels.sh | 57 ---------------------------- .github/workflows/ci.yml | 69 +++++++++------------------------- 2 files changed, 17 insertions(+), 109 deletions(-) delete mode 100755 .github/scripts/test-wheels.sh diff --git a/.github/scripts/test-wheels.sh b/.github/scripts/test-wheels.sh deleted file mode 100755 index fedfe57..0000000 --- a/.github/scripts/test-wheels.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# -# Tests the wheels built in one job all at once: a pytest process per wheel, on the wheel's own interpreter. -# -# .github/scripts/test-wheels.sh sdist/wrtc-0.0.1.tar.gz wheelhouse [--gc-on-emit] -# -# With --gc-on-emit, the free-threaded wheel also runs the suite with it, at the same time. -set -euo pipefail -SDIST="$1" -WHEELS="$(cd "$2" && pwd)" -GC_ON_EMIT="${3:-}" -WORK="$(mktemp -d)" -trap 'rm -rf "$WORK"' EXIT - -# the tests from the sdist, as cibuildwheel runs them -tar -xzf "$SDIST" -C "$WORK" -cd "$WORK"/wrtc-* - -names=() -pids=() -# in the background, into a log of its own: run NAME PYTHON [PYTEST ARGS...] -run() { - local name="$1" python="$2" - shift 2 - "$python" -m pytest tests "$@" > "$WORK/$name.log" 2>&1 & - names+=("$name") - pids+=($!) -} - -for wheel in "$WHEELS"/*.whl; do - # wrtc-0.0.1-cp315-cp315t-macosx_13_0_arm64.whl: cp315t, Python 3.15t - abi="$(basename "$wheel" | cut -d- -f4)" - digits="${abi#cp}" - venv="$WORK/$abi" - uv venv -q --python "${digits:0:1}.${digits:1}" "$venv" - uv pip install -q --python "$venv/bin/python" "$wheel" --group pyproject.toml:test - run "$abi" "$venv/bin/python" - if [ "$GC_ON_EMIT" = --gc-on-emit ] && [[ "$abi" == *t ]]; then - run "$abi-gc-on-emit" "$venv/bin/python" --gc-on-emit - fi -done - -status=0 -for i in "${!pids[@]}"; do - name="${names[$i]}" - # a failed run stays expanded - if wait "${pids[$i]}"; then - echo "::group::$name passed" - cat "$WORK/$name.log" - echo "::endgroup::" - else - status=1 - echo "$name failed:" - cat "$WORK/$name.log" - fi -done -exit "$status" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 620434d..2ce5386 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] # PRs and main: oldest and newest only, with and without the GIL; all of them for tags, Fridays and by hand python: ${{ fromJSON((startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '["cp39","cp310","cp311","cp312","cp313","cp314","cp314t","cp315","cp315t"]' || '["cp39","cp315","cp315t"]') }} env: @@ -103,49 +103,8 @@ jobs: path: wheelhouse/*.whl overwrite: true - # macOS has 5 concurrent jobs for the whole account: one job builds several wheels and tests them at once - wheels-macos: - name: wheels (${{ matrix.os }}, ${{ matrix.python }}) - runs-on: ${{ matrix.os }} - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - os: [macos-15-intel, macos-15] - # as many as the 3 cores of macos-15 - python: ${{ fromJSON((startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '["cp39 cp310 cp311","cp312 cp313 cp314","cp314t cp315 cp315t"]' || '["cp39 cp315 cp315t"]') }} - env: - WRTC_CACHE_DIR: ${{ github.workspace }}/.wrtc-cache - UV_HTTP_RETRIES: 10 - CIBW_TEST_SKIP: "*" - steps: - - uses: actions/checkout@v7 - with: - path: .ci-src - - uses: astral-sh/setup-uv@v7 - - name: Build sdist - run: | - uv build --sdist --out-dir sdist .ci-src - echo "SDIST=$(ls sdist/*.tar.gz)" >> "$GITHUB_ENV" - python="${{ matrix.python }}" - echo "CIBW_BUILD=$(printf '%s-* ' $python)" >> "$GITHUB_ENV" - echo "LABEL=${python// /-}" >> "$GITHUB_ENV" - - uses: actions/cache@v6 - with: - path: .wrtc-cache - key: libwebrtc-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.ci-src/cmake/libwebrtc.cmake', '.ci-src/cmake/libcxx/*.sha256', '.ci-src/cmake/openh264.*') }} - - uses: pypa/cibuildwheel@v4.2.1 - with: - package-dir: ${{ env.SDIST }} - - run: bash .ci-src/.github/scripts/test-wheels.sh "$SDIST" wheelhouse ${{ github.event_name != 'push' && '--gc-on-emit' || '' }} - - uses: actions/upload-artifact@v7 - with: - name: wheels-${{ matrix.os }}-${{ env.LABEL }} - path: wheelhouse/*.whl - overwrite: true - sanitizers: - name: sanitizers (ASan + UBSan, ${{ matrix.arch }}) + name: sanitizers (ASan + UBSan, ${{ matrix.arch }}, ${{ matrix.pass }}) if: github.event_name != 'push' runs-on: ${{ matrix.runner }} container: quay.io/pypa/manylinux_2_28_${{ matrix.arch }} @@ -153,10 +112,15 @@ jobs: strategy: fail-fast: false matrix: + arch: [x86_64, aarch64] + pass: [plain, gc-on-emit, stress] include: - { arch: x86_64, runner: ubuntu-latest } # also covers the libc++ built from source for arm64 - { arch: aarch64, runner: ubuntu-24.04-arm } + - { pass: plain, args: "" } + - { pass: gc-on-emit, args: --gc-on-emit } + - { pass: stress, args: --stress -k "stress or chaos" } steps: - uses: actions/checkout@v7 # github.workspace is the host's path @@ -165,17 +129,19 @@ jobs: with: path: .wrtc-cache key: libwebrtc-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('cmake/libwebrtc.cmake', 'cmake/libcxx/*.sha256', 'cmake/openh264.*') }} - - run: .github/scripts/sanitizers.sh - # the collector running on libwebrtc threads, whenever they emit (see tests/conftest.py) - - run: .github/scripts/sanitizers.sh --gc-on-emit - # long random sequences of calls (tests/chaos.py) - - run: .github/scripts/sanitizers.sh --stress -k "stress or chaos" + - run: .github/scripts/sanitizers.sh ${{ matrix.args }} sanitizers-macos: - name: sanitizers (ASan + UBSan, macOS) + name: sanitizers (ASan + UBSan, macOS, ${{ matrix.pass }}) if: github.event_name != 'push' runs-on: macos-15 timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - { pass: plain, args: "" } + - { pass: gc-on-emit, args: --gc-on-emit } env: WRTC_CACHE_DIR: ${{ github.workspace }}/.wrtc-cache steps: @@ -185,12 +151,11 @@ jobs: path: .wrtc-cache key: libwebrtc-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('cmake/libwebrtc.cmake', 'cmake/libcxx/*.sha256', 'cmake/openh264.*') }} - uses: astral-sh/setup-uv@v7 - - run: .github/scripts/sanitizers-macos.sh - - run: .github/scripts/sanitizers-macos.sh --gc-on-emit + - run: .github/scripts/sanitizers-macos.sh ${{ matrix.args }} publish: if: startsWith(github.ref, 'refs/tags/v') - needs: [lint, sdist, wheels, wheels-macos] + needs: [lint, sdist, wheels] runs-on: ubuntu-latest environment: pypi permissions: From e6732f528bc29a0192ef345cce7481d8ca02acdf Mon Sep 17 00:00:00 2001 From: "Ilya (Marshal)" Date: Mon, 12 Oct 2026 01:17:34 +0200 Subject: [PATCH 3/4] Exclude intel macos from PRs, fiz flacky test --- .github/workflows/ci.yml | 3 +++ tests/test_stats.py | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ce5386..391f01d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,9 @@ jobs: os: [ubuntu-latest, ubuntu-24.04-arm, macos-15-intel, macos-15, windows-latest] # PRs and main: oldest and newest only, with and without the GIL; all of them for tags, Fridays and by hand python: ${{ fromJSON((startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '["cp39","cp310","cp311","cp312","cp313","cp314","cp314t","cp315","cp315t"]' || '["cp39","cp315","cp315t"]') }} + # Intel macOS with the full matrix only + exclude: + - os: ${{ !(startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && 'macos-15-intel' || '' }} env: WRTC_CACHE_DIR: ${{ github.workspace }}/.wrtc-cache WRTC_ARCHIVE_DIR: ${{ matrix.os == 'windows-latest' && format('{0}/.wrtc-cache/archives', github.workspace) || '' }} diff --git a/tests/test_stats.py b/tests/test_stats.py index 9d70d65..62d88b8 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -19,6 +19,8 @@ from tests.helpers import connect, stats_of_type, wait_for_event, wait_until, wait_until_unmuted from webrtc.models import rtc_stats +ICE_TYPES = {'candidate-pair', 'local-candidate', 'remote-candidate'} + async def send_audio( caller: webrtc.RTCPeerConnection, callee: webrtc.RTCPeerConnection, stream: webrtc.MediaStream @@ -34,6 +36,11 @@ async def send_audio( return remote +def ice_free_ids(report: webrtc.RTCStatsReport) -> set[str]: + """The ids of the stats in a report, but of its candidate pair and candidates.""" + return {stats.id for stats in report.values() if stats.type not in ICE_TYPES} + + @pytest.mark.asyncio async def test_connection_stats( caller: webrtc.RTCPeerConnection, callee: webrtc.RTCPeerConnection, audio_stream: webrtc.MediaStream @@ -74,7 +81,10 @@ async def has_remote_inbound() -> bool: assert len(stats_of_type(sender_report, 'outbound-rtp')) > 0 assert len(stats_of_type(sender_report, 'inbound-rtp')) == 0 track_report = await caller.get_stats(audio_stream.get_tracks()[0]) - assert set(track_report.keys()) == set(sender_report.keys()) + assert ice_free_ids(track_report) == ice_free_ids(sender_report) + # ICE may still switch the selected pair between the two reports + assert len(stats_of_type(track_report, 'candidate-pair')) == 1 + assert len(stats_of_type(sender_report, 'candidate-pair')) == 1 @pytest.mark.asyncio From 2bd013da1a474dbb695aaffbce57bb97c9c3450e Mon Sep 17 00:00:00 2001 From: "Ilya (Marshal)" Date: Mon, 12 Oct 2026 01:31:48 +0200 Subject: [PATCH 4/4] Run --gc-on-emit for every interpreter on PRs again --- .github/workflows/ci.yml | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 391f01d..5fa2f7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,8 +68,8 @@ jobs: WRTC_ARCHIVE_DIR: ${{ matrix.os == 'windows-latest' && format('{0}/.wrtc-cache/archives', github.workspace) || '' }} UV_HTTP_RETRIES: 10 CIBW_ENVIRONMENT_PASS_LINUX: UV_HTTP_RETRIES - # not on main and tags - CIBW_TEST_COMMAND: ${{ github.event_name != 'push' && endsWith(matrix.python, 't') && 'pytest {project}/tests && pytest {project}/tests --gc-on-emit' || 'pytest {project}/tests' }} + # --gc-on-emit not on main and tags + CIBW_TEST_COMMAND: ${{ github.event_name != 'push' && 'pytest {project}/tests && pytest {project}/tests --gc-on-emit' || 'pytest {project}/tests' }} steps: - uses: actions/checkout@v7 with: diff --git a/pyproject.toml b/pyproject.toml index ffae938..e63e236 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,7 +99,7 @@ build-frontend = "build[uv]" # dev builds stay per file to catch missing includes config-settings = { "cmake.define.CMAKE_UNITY_BUILD" = "ON" } test-groups = ["test"] -# CI adds --gc-on-emit for free-threaded builds on PRs (.github/workflows/ci.yml) +# CI adds --gc-on-emit on PRs (.github/workflows/ci.yml) test-command = "pytest {project}/tests" [tool.cibuildwheel.linux]