From 548d284c6a00aab428e3c3e08b44fec42338b805 Mon Sep 17 00:00:00 2001 From: Crash0v3rrid3 Date: Mon, 3 Aug 2026 18:22:57 +0530 Subject: [PATCH 1/6] ci: add PR smoke-test workflow for the a11y-scan SPM plugin Runs an end-to-end accessibility scan on every PR: builds the plugin and executes a real scan against the tests/spm harness (sample SwiftUI sources with intentional a11y issues), reusing the repo's own gated integration test (testA11yScanPluginRuns) so the invocation stays in one place. The scan downloads the BrowserStack CLI and makes authenticated calls, so it is gated to same-repo PRs (secrets are never exposed to fork PRs) and manual dispatch. Without the BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY secrets configured the e2e test XCTSkips and the job still passes. actions/checkout pinned by SHA to match existing workflows (DEVA11Y-476). Co-Authored-By: Claude Opus 4.8 --- .github/workflows/spm-smoke-test.yml | 52 ++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/spm-smoke-test.yml diff --git a/.github/workflows/spm-smoke-test.yml b/.github/workflows/spm-smoke-test.yml new file mode 100644 index 0000000..923c5d6 --- /dev/null +++ b/.github/workflows/spm-smoke-test.yml @@ -0,0 +1,52 @@ +# Smoke-tests the `a11y-scan` SwiftPM command plugin end-to-end on every PR: +# builds the plugin and runs a real accessibility scan against the tests/spm +# harness (sample SwiftUI sources with intentional a11y issues). It reuses the +# repository's own gated integration test (testA11yScanPluginRuns) so the scan +# invocation stays defined in exactly one place. +# +# The scan downloads the BrowserStack CLI and makes authenticated network calls, +# so it needs BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY repo secrets. Those +# secrets are never exposed to fork PRs, so this job is gated to same-repo PRs +# (and manual dispatch); fork PRs skip it. If the secrets are not configured yet, +# the end-to-end test XCTSkips and the job still passes (build + unit test only). +name: SPM plugin smoke test + +on: + pull_request: + branches: [main, master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: spm-smoke-${{ github.ref }} + cancel-in-progress: true + +jobs: + spm-smoke: + name: a11y-scan end-to-end (SwiftPM) + runs-on: macos-14 + timeout-minutes: 25 + # Secrets are unavailable to fork PRs, so the authenticated scan can only run + # on same-repo PRs or a manual dispatch. Fork PRs skip this job. + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.fork == false + env: + BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }} + BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} + # Un-gates tests/spm/Tests/A11yDemoLibTests/testA11yScanPluginRuns, which is + # skipped unless RUN_A11Y_SCAN=1 and BrowserStack credentials are present. + RUN_A11Y_SCAN: "1" + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Swift toolchain + run: swift --version + + - name: Build plugin package + run: swift build + + - name: End-to-end scan smoke (tests/spm) + working-directory: tests/spm + run: swift test From 79daf7fa7eb8eabc5aa2114a43a8fe2c01035799 Mon Sep 17 00:00:00 2001 From: Crash0v3rrid3 Date: Mon, 3 Aug 2026 18:25:35 +0530 Subject: [PATCH 2/6] ci: build/test tests/spm instead of plugin-only root The repo root is a plugin-only package with no buildable target, so `swift build` there fails ("does not contain a buildable target"). Building the tests/spm harness compiles the a11y-scan command plugin via the path dependency plus the sample sources, so use that as the build step. Verified locally on Swift 6.2: tests/spm `swift build` compiles the plugin, and `swift test` passes with the e2e scan test skipping when RUN_A11Y_SCAN is unset. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/spm-smoke-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/spm-smoke-test.yml b/.github/workflows/spm-smoke-test.yml index 923c5d6..e8b93a0 100644 --- a/.github/workflows/spm-smoke-test.yml +++ b/.github/workflows/spm-smoke-test.yml @@ -44,7 +44,11 @@ jobs: - name: Swift toolchain run: swift --version - - name: Build plugin package + # The repo root is a plugin-only package (no buildable target), so it is + # not built directly. Building the tests/spm harness compiles both the + # a11y-scan command plugin (via the path dependency) and the sample sources. + - name: Build harness (compiles the a11y-scan plugin) + working-directory: tests/spm run: swift build - name: End-to-end scan smoke (tests/spm) From 15d5875b0f4323fab8221a4ea0d810ba5852c9ec Mon Sep 17 00:00:00 2001 From: Crash0v3rrid3 Date: Tue, 4 Aug 2026 10:48:49 +0530 Subject: [PATCH 3/6] fix(tests): use valid SwiftPM network-permission flag in scan scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run-a11y-scan.sh passed `--allow-network-connections 'all(ports: [])'` — that is PackageDescription API syntax, not a valid CLI value, and the empty port list did not satisfy the a11y-scan plugin's declared need for ports 80/443. SwiftPM therefore refused the scan: error: Plugin 'a11y-scan' wants permission to allow all network connections on ports: 80, 443. Use `--allow-network-connections all:80,443` to allow this. Surfaced by the new PR smoke-test job, which is the first thing to run the scan in CI. Fix per SwiftPM's own guidance: `all:80,443`. Applied to the SwiftPM and Xcode harness scripts and the tests/spm README (same bug in all three). Verified locally: the CLI now clears the permission gate and runs. Co-Authored-By: Claude Opus 4.8 --- tests/spm/README.md | 2 +- tests/spm/scripts/run-a11y-scan.sh | 2 +- tests/xcode-app/scripts/run-a11y-scan.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/spm/README.md b/tests/spm/README.md index 3ee07fd..af737e6 100644 --- a/tests/spm/README.md +++ b/tests/spm/README.md @@ -34,7 +34,7 @@ The script runs: swift package plugin \ --allow-writing-to-directory ~/.cache \ --allow-writing-to-package-directory \ - --allow-network-connections 'all(ports: [])' \ + --allow-network-connections all:80,443 \ scan --include "**/*.swift" --include "**/*.xib" --include "**/*.storyboard" ``` diff --git a/tests/spm/scripts/run-a11y-scan.sh b/tests/spm/scripts/run-a11y-scan.sh index 8bbb3f0..7ccc2e3 100755 --- a/tests/spm/scripts/run-a11y-scan.sh +++ b/tests/spm/scripts/run-a11y-scan.sh @@ -16,7 +16,7 @@ cd "$(dirname "$0")/.." swift package plugin \ --allow-writing-to-directory "$HOME/.cache" \ --allow-writing-to-package-directory \ - --allow-network-connections 'all(ports: [])' \ + --allow-network-connections all:80,443 \ scan \ --include "**/*.swift" \ --include "**/*.xib" \ diff --git a/tests/xcode-app/scripts/run-a11y-scan.sh b/tests/xcode-app/scripts/run-a11y-scan.sh index 0cfc296..c494b04 100755 --- a/tests/xcode-app/scripts/run-a11y-scan.sh +++ b/tests/xcode-app/scripts/run-a11y-scan.sh @@ -20,7 +20,7 @@ fi swift package plugin \ --allow-writing-to-directory "$HOME/.cache" \ --allow-writing-to-package-directory \ - --allow-network-connections 'all(ports: [])' \ + --allow-network-connections all:80,443 \ scan \ --include "**/*.swift" \ --include "**/*.xib" \ From 69d5925be573e941585c6298436892eb98b1e08d Mon Sep 17 00:00:00 2001 From: Crash0v3rrid3 Date: Tue, 4 Aug 2026 15:56:51 +0530 Subject: [PATCH 4/6] =?UTF-8?q?ci:=20add=20scripts-lint=20job=20=E2=80=94?= =?UTF-8?q?=20bash=20-n=20on=20every=20launcher=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a second, secret-free job that syntax-checks all six launcher scripts under scripts/ (bash/zsh/fish x cli/spm) with `bash -n`. They are all bash scripts (the zsh/fish variants only differ in which login shell they source credentials from), so a single bash syntax gate covers them. Runs on all PRs including forks; scripts are not executed (they self-update, register git hooks and need credentials). Complements verify-selfupdate-checksums.yml, which covers checksum integrity but not syntax. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/spm-smoke-test.yml | 40 +++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/.github/workflows/spm-smoke-test.yml b/.github/workflows/spm-smoke-test.yml index e8b93a0..097fd4a 100644 --- a/.github/workflows/spm-smoke-test.yml +++ b/.github/workflows/spm-smoke-test.yml @@ -6,9 +6,12 @@ # # The scan downloads the BrowserStack CLI and makes authenticated network calls, # so it needs BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY repo secrets. Those -# secrets are never exposed to fork PRs, so this job is gated to same-repo PRs +# secrets are never exposed to fork PRs, so that job is gated to same-repo PRs # (and manual dispatch); fork PRs skip it. If the secrets are not configured yet, # the end-to-end test XCTSkips and the job still passes (build + unit test only). +# +# A second job (scripts-lint) syntax-checks every launcher script under scripts/. +# It needs no secrets, so it runs on all PRs including forks. name: SPM plugin smoke test on: @@ -54,3 +57,38 @@ jobs: - name: End-to-end scan smoke (tests/spm) working-directory: tests/spm run: swift test + + scripts-lint: + name: Launcher scripts (bash syntax) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + # Every script under scripts/ — the bash, zsh and fish variants alike — is + # a bash script (`#!/usr/bin/env bash`); the variants differ only in which + # login shell they source BrowserStack creds from. So all of them are + # syntax-checked with `bash -n`. The scripts self-update, register git + # hooks and need credentials, so they are not executed here — this is a + # static syntax gate. (Checksum integrity is covered separately by + # verify-selfupdate-checksums.yml.) + - name: Syntax-check all launcher scripts (bash -n) + run: | + set -uo pipefail + shopt -s globstar nullglob + scripts=(scripts/**/*.sh) + if [ ${#scripts[@]} -eq 0 ]; then + echo "::error::No .sh scripts found under scripts/ — checkout or glob is wrong." + exit 1 + fi + status=0 + for script in "${scripts[@]}"; do + if bash -n "$script"; then + echo "::notice file=${script}::bash -n OK" + else + echo "::error file=${script}::bash -n failed — fix the syntax error above." + status=1 + fi + done + exit "$status" From dd87f278cfdc5d5b4c55e3ca47064cfebb44debf Mon Sep 17 00:00:00 2001 From: Crash0v3rrid3 Date: Tue, 4 Aug 2026 16:36:57 +0530 Subject: [PATCH 5/6] fix(review): make no-secrets path truly skip, harden scripts-lint Code-review follow-ups on the smoke workflow: - Graceful degradation was only claimed, not real. GitHub exposes an unset secret as an empty string (present, not nil), and the reused test skips only on `env[...] != nil`, so a repo without the secrets would run the scan with empty creds and fail (script's `:?` under set -euo pipefail), not skip. Guard the scan step on the secrets being non-empty so it is skipped when absent and the job stays green on the build step. Fix the header comment to match. - Correct the scripts-lint comment: the launchers' shebang is `#!/usr/bin/env bash -il`, not `#!/usr/bin/env bash`. - Drop the fork-controllable filename from the `::notice/::error file=` workflow commands (workflow-command injection vector on fork PRs); log plain lines instead. bash -n still prints the real error location. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/spm-smoke-test.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/spm-smoke-test.yml b/.github/workflows/spm-smoke-test.yml index 097fd4a..bbd44ef 100644 --- a/.github/workflows/spm-smoke-test.yml +++ b/.github/workflows/spm-smoke-test.yml @@ -7,8 +7,9 @@ # The scan downloads the BrowserStack CLI and makes authenticated network calls, # so it needs BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY repo secrets. Those # secrets are never exposed to fork PRs, so that job is gated to same-repo PRs -# (and manual dispatch); fork PRs skip it. If the secrets are not configured yet, -# the end-to-end test XCTSkips and the job still passes (build + unit test only). +# (and manual dispatch); fork PRs skip it. The scan step is itself guarded on the +# secrets being present, so if they are not configured the scan is skipped and the +# job still passes on the build step alone. # # A second job (scripts-lint) syntax-checks every launcher script under scripts/. # It needs no secrets, so it runs on all PRs including forks. @@ -54,7 +55,12 @@ jobs: working-directory: tests/spm run: swift build + # Guarded on the secrets actually being set: GitHub exposes an unset secret + # as an empty string (present, not nil), so without this guard the scan would + # run with empty credentials and fail. When the secrets are absent this step + # is skipped and the job stays green on the build step alone. - name: End-to-end scan smoke (tests/spm) + if: env.BROWSERSTACK_USERNAME != '' && env.BROWSERSTACK_ACCESS_KEY != '' working-directory: tests/spm run: swift test @@ -67,7 +73,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # Every script under scripts/ — the bash, zsh and fish variants alike — is - # a bash script (`#!/usr/bin/env bash`); the variants differ only in which + # a bash script (`#!/usr/bin/env bash -il`); the variants differ only in which # login shell they source BrowserStack creds from. So all of them are # syntax-checked with `bash -n`. The scripts self-update, register git # hooks and need credentials, so they are not executed here — this is a @@ -84,10 +90,13 @@ jobs: fi status=0 for script in "${scripts[@]}"; do + # Plain log lines, not ::notice file=/::error file= workflow commands: + # scripts/ filenames are attacker-controllable on fork PRs, and + # interpolating them into a workflow command is an injection vector. if bash -n "$script"; then - echo "::notice file=${script}::bash -n OK" + echo "OK $script" else - echo "::error file=${script}::bash -n failed — fix the syntax error above." + echo "FAILED $script (bash -n syntax error above)" status=1 fi done From c24c577455bde056c2b82905e490388c7c0b3da3 Mon Sep 17 00:00:00 2001 From: Crash0v3rrid3 Date: Tue, 4 Aug 2026 16:47:39 +0530 Subject: [PATCH 6/6] fix(review): assert scan detects issues + retry transient scan failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the two open code-review findings on the smoke workflow: - Fidelity (#2): the e2e test asserted only that --non-strict exits 0, so a scan that authenticated but found nothing (silent no-op) would pass green. Now run the scan twice and use the tool's own exit-code contract: strict mode must exit non-zero, proving the intentional issues in SampleViews.swift were actually detected — not just that the plugin ran. Uses exit codes, not brittle output matching. Also drains output for diagnostics and treats an empty credential value as absent (skip) to match the workflow guard. - Flakiness (#3): the scan hits BrowserStack (network + auth + CLI download) on every same-repo PR. Wrapped it in a bounded retry (3 attempts, 20s backoff) so a transient upstream hiccup doesn't red-block a PR; a consistent failure still fails the gate. swift test reuses the first build, so retries only re-run the scan. Verified locally: tests compile; no-creds path still skips cleanly. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/spm-smoke-test.yml | 23 +++++++++- .../A11yDemoLibTests/A11yDemoLibTests.swift | 42 ++++++++++++++++--- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/.github/workflows/spm-smoke-test.yml b/.github/workflows/spm-smoke-test.yml index bbd44ef..572468e 100644 --- a/.github/workflows/spm-smoke-test.yml +++ b/.github/workflows/spm-smoke-test.yml @@ -59,10 +59,31 @@ jobs: # as an empty string (present, not nil), so without this guard the scan would # run with empty credentials and fail. When the secrets are absent this step # is skipped and the job stays green on the build step alone. + # + # The scan hits BrowserStack (network + auth + CLI download), so a transient + # upstream hiccup should not red-block a PR. Retry the scan up to 3 times with + # backoff; a consistent failure still fails the gate. `swift test` reuses the + # first attempt's build, so retries only re-run the scan. - name: End-to-end scan smoke (tests/spm) if: env.BROWSERSTACK_USERNAME != '' && env.BROWSERSTACK_ACCESS_KEY != '' working-directory: tests/spm - run: swift test + run: | + set -uo pipefail + attempts=3 + for i in $(seq 1 "$attempts"); do + echo "::group::a11y-scan smoke attempt $i/$attempts" + if swift test; then + echo "::endgroup::" + exit 0 + fi + echo "::endgroup::" + if [ "$i" -lt "$attempts" ]; then + echo "Attempt $i failed; retrying in 20s (occasional transient upstream failures are expected)." + sleep 20 + fi + done + echo "::error::a11y-scan smoke failed after $attempts attempts." + exit 1 scripts-lint: name: Launcher scripts (bash syntax) diff --git a/tests/spm/Tests/A11yDemoLibTests/A11yDemoLibTests.swift b/tests/spm/Tests/A11yDemoLibTests/A11yDemoLibTests.swift index ba847f4..9d8a9e9 100644 --- a/tests/spm/Tests/A11yDemoLibTests/A11yDemoLibTests.swift +++ b/tests/spm/Tests/A11yDemoLibTests/A11yDemoLibTests.swift @@ -19,7 +19,11 @@ final class A11yDemoLibTests: XCTestCase { guard env["RUN_A11Y_SCAN"] == "1" else { throw XCTSkip("Set RUN_A11Y_SCAN=1 (with BrowserStack creds) to run the plugin end-to-end.") } - guard env["BROWSERSTACK_USERNAME"] != nil, env["BROWSERSTACK_ACCESS_KEY"] != nil else { + // Treat an empty value as absent: CI exposes an unset secret as "" (present, + // not nil), and running the scan with empty credentials fails at auth rather + // than skipping. + guard env["BROWSERSTACK_USERNAME"]?.isEmpty == false, + env["BROWSERSTACK_ACCESS_KEY"]?.isEmpty == false else { throw XCTSkip("BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY are required for the scan.") } @@ -30,15 +34,43 @@ final class A11yDemoLibTests: XCTestCase { .deletingLastPathComponent() let script = packageDir.appendingPathComponent("scripts/run-a11y-scan.sh") + // Run the scan twice and use the tool's own exit-code contract to prove it + // not only ran but actually detected the intentional issues in + // SampleViews.swift: + // * --non-strict -> exit 0 (CLI downloaded, authenticated, ran cleanly; + // issues do not fail the run) + // * strict -> exit != 0 (issues were found; strict fails on issues) + // Asserting only the non-strict exit 0 would also pass if the scan + // authenticated but found nothing -- a silent no-op. Requiring the strict + // run to fail closes that gap. + let clean = try runScan(script: script, packageDir: packageDir, strict: false) + XCTAssertEqual( + clean.status, 0, + "a11y-scan did not run cleanly in --non-strict mode (exit \(clean.status)).\n\(clean.output)") + + let strict = try runScan(script: script, packageDir: packageDir, strict: true) + XCTAssertNotEqual( + strict.status, 0, + "a11y-scan ran but reported no issues against SampleViews.swift (strict exit 0) -- possible silent no-op or engine regression.\n\(strict.output)") + } + + /// Runs `scripts/run-a11y-scan.sh` (optionally strict) and returns its exit + /// status plus combined stdout/stderr. Draining to EOF before `waitUntilExit` + /// avoids a full-pipe-buffer deadlock without a background reader. + private func runScan(script: URL, packageDir: URL, strict: Bool) throws -> (status: Int32, output: String) { let process = Process() process.executableURL = URL(fileURLWithPath: "/bin/bash") - process.arguments = [script.path, "--non-strict"] + process.arguments = strict ? [script.path] : [script.path, "--non-strict"] process.currentDirectoryURL = packageDir + + let pipe = Pipe() + process.standardOutput = pipe + process.standardError = pipe + try process.run() + let collected = pipe.fileHandleForReading.readDataToEndOfFile() process.waitUntilExit() - // --non-strict makes the scan exit 0 even when issues are found, so a - // clean exit means the plugin downloaded, authenticated, and ran. - XCTAssertEqual(process.terminationStatus, 0, "a11y-scan plugin failed to run") + return (process.terminationStatus, String(data: collected, encoding: .utf8) ?? "") } }