From 58d6f8d92bc5819e525d18f1028ce5704ab5cf9d Mon Sep 17 00:00:00 2001 From: Jason Murray Date: Sat, 31 Oct 2026 05:48:33 -0700 Subject: [PATCH 1/6] Report action.yml line coverage to GitHub Code Quality Replay action.yml's bash steps under kcov across the default, JUnit, line-count and exclude-glob scenarios, then upload the merged Cobertura report with actions/upload-code-coverage. kcov v43 is built from source (ubuntu-latest no longer packages it) and cached per runner image. --- .github/workflows/coverage.yaml | 214 ++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 .github/workflows/coverage.yaml diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml new file mode 100644 index 0000000..836e912 --- /dev/null +++ b/.github/workflows/coverage.yaml @@ -0,0 +1,214 @@ +name: Coverage + +# Reports line coverage of the bash in action.yml to GitHub Code Quality. +# +# The smoke tests run the action through the Actions runner (`uses: ./`), which +# nothing can trace, so this job extracts the same `run:` blocks from action.yml +# and replays them under kcov across the same scenarios. + +on: + push: + branches: + - main + pull_request: + types: + - opened + - synchronize + - reopened + +permissions: + contents: read + code-quality: write + +jobs: + coverage: + runs-on: ubuntu-latest + name: action.yml Coverage + env: + KCOV_VERSION: v43 + STEPS: coverage/steps + OUT: coverage/kcov + GLOB: examples/spec/**/*_spec.rb + JUNIT: coverage/rspec.xml + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - uses: actions/checkout@v4 + with: + repository: eliotsykes/rspec-rails-examples + path: examples/ + + # kcov is built from source because Ubuntu 24.04 (ubuntu-latest) no + # longer packages it, and the v42 release binary targets Ubuntu 22.04. + # The install prefix is cached per kcov version and runner image OS. + - name: Cache kcov + id: kcov-cache + uses: actions/cache@v4 + with: + path: ~/kcov + key: kcov-${{ env.KCOV_VERSION }}-${{ runner.os }}-${{ runner.arch }}-${{ env.ImageOS }} + + - name: Install kcov dependencies + # Needed at runtime too, so this runs even on a cache hit. + run: | + set -euo pipefail + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends \ + build-essential cmake git ca-certificates pkg-config python3 \ + binutils-dev libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev \ + libssl-dev zlib1g-dev + + - name: Build kcov + if: steps.kcov-cache.outputs.cache-hit != 'true' + run: | + set -euo pipefail + src=$(mktemp -d) + git clone -q --depth 1 --branch "$KCOV_VERSION" https://github.com/SimonKagstrom/kcov "$src" + cmake -S "$src" -B "$src/build" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$HOME/kcov" + cmake --build "$src/build" -j"$(nproc)" + cmake --install "$src/build" + + - name: Add kcov to PATH + run: | + echo "$HOME/kcov/bin" >> "$GITHUB_PATH" + "$HOME/kcov/bin/kcov" --version + + - name: Extract action.yml steps + # Writes coverage/steps/step_N.sh (one per `run:` block) and step_N.env + # (literal `env:` values). Each script is padded with blank lines so its + # line numbers match action.yml. + run: | + set -euo pipefail + python3 -I - action.yml "$STEPS" <<'PY' + import pathlib + import sys + + import yaml + + src, out = sys.argv[1], pathlib.Path(sys.argv[2]) + out.mkdir(parents=True, exist_ok=True) + + root = yaml.compose(open(src)) + + + def get(node, key): + for k, v in node.value: + if k.value == key: + return v + return None + + + for i, step in enumerate(get(get(root, "runs"), "steps").value): + run = get(step, "run") + if run is None: + continue + # The block scalar's indicator ("|") is on start_mark.line + # (0-indexed), so its content starts on the next line. + script = out / f"step_{i}.sh" + script.write_text("#!/usr/bin/env bash" + "\n" * (run.start_mark.line + 1) + run.value) + script.chmod(0o755) + + env = get(step, "env") + lines = [] + if env is not None: + for k, v in env.value: + if "${{" not in v.value: + lines.append(f"{k.value}={v.value}") + (out / f"step_{i}.env").write_text("\n".join(lines) + "\n") + PY + + - name: Cover Install split_tests + run: | + set -euo pipefail + mkdir -p "$OUT" + set -a + . "$STEPS/step_0.env" + set +a + kcov --include-pattern=step_ "$OUT/install" "$STEPS/step_0.sh" + + - name: Cover Split Tests + run: | + set -euo pipefail + + # Minimal JUnit report built from the fixture's spec files. + python3 -I - "$JUNIT" <<'PY' + import glob, sys + files = sorted(glob.glob("examples/spec/**/*_spec.rb", recursive=True)) + cases = "".join( + f'' + for i, f in enumerate(files) + ) + open(sys.argv[1], "w").write( + f'{cases}' + ) + PY + + # scenario NAME [VAR=value ...]: run the split step with these inputs. + scenario() { + local name=$1 + shift + env SPLIT_TOTAL=5 SPLIT_INDEX=0 GLOB="$GLOB" \ + JUNIT_PATH= LINE_COUNT= EXCLUDE_GLOB_INPUT= \ + GITHUB_OUTPUT="$(mktemp)" "$@" \ + kcov --include-pattern=step_ "$OUT/$name" "$STEPS/step_1.sh" + } + + scenario default + scenario junit JUNIT_PATH="$JUNIT" + scenario line-count LINE_COUNT=true + scenario exclude-glob EXCLUDE_GLOB_INPUT='examples/spec/api/**/t*_spec.rb' + + - name: Build Cobertura report for action.yml + # Merges the scenarios, then maps the step scripts onto action.yml as a + # single file so Code Quality can attribute lines to it. + run: | + set -euo pipefail + kcov --merge coverage/merged "$OUT"/*/ + python3 -I - "$(find coverage/merged -name cobertura.xml -path '*kcov-merged*' | head -n1)" coverage/cobertura.xml action.yml <<'PY' + import sys + import xml.etree.ElementTree as ET + + src, dst, target = sys.argv[1:4] + tree = ET.parse(src) + root = tree.getroot() + + lines = {} + for line in root.iter("line"): + n = int(line.get("number")) + # A line in several step scripts is hit if any of them hit it. + prev = lines.get(n) + if prev is None or int(line.get("hits", "0")) > int(prev.get("hits", "0")): + lines[n] = line + + for sources in root.findall("sources"): + root.remove(sources) + sources = ET.Element("sources") + ET.SubElement(sources, "source").text = "." + root.insert(0, sources) + + packages = root.find("packages") + for child in list(packages): + packages.remove(child) + package = ET.SubElement(packages, "package", name=".") + cls = ET.SubElement(ET.SubElement(package, "classes"), "class", name=target, filename=target) + ET.SubElement(cls, "methods") + cls_lines = ET.SubElement(cls, "lines") + for n in sorted(lines): + cls_lines.append(lines[n]) + + covered = sum(1 for l in lines.values() if int(l.get("hits", "0")) > 0) + rate = f"{covered / len(lines):.4f}" if lines else "0" + for el in (root, package, cls): + el.set("line-rate", rate) + tree.write(dst, xml_declaration=True, encoding="utf-8") + print(f"{covered}/{len(lines)} lines covered") + PY + + - name: Upload coverage report + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: actions/upload-code-coverage@v1 + with: + file: coverage/cobertura.xml + language: bash + label: code-coverage/kcov From 1afc83a1c1fadcf8af6246480647599162198880 Mon Sep 17 00:00:00 2001 From: Jason Murray Date: Sat, 31 Oct 2026 10:58:53 -0700 Subject: [PATCH 2/6] Upload coverage to Codecov instead of GitHub Code Quality Code Quality is only available on Team and Enterprise plans, so it is not available for this repository. --- .github/workflows/coverage.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 836e912..0005a1b 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -1,6 +1,6 @@ name: Coverage -# Reports line coverage of the bash in action.yml to GitHub Code Quality. +# Reports line coverage of the bash in action.yml to Codecov. # # The smoke tests run the action through the Actions runner (`uses: ./`), which # nothing can trace, so this job extracts the same `run:` blocks from action.yml @@ -18,7 +18,6 @@ on: permissions: contents: read - code-quality: write jobs: coverage: @@ -206,9 +205,10 @@ jobs: PY - name: Upload coverage report + # Fork PRs have no access to secrets, so only upload from this repo. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - uses: actions/upload-code-coverage@v1 + uses: codecov/codecov-action@v5 with: - file: coverage/cobertura.xml - language: bash - label: code-coverage/kcov + files: coverage/cobertura.xml + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true From ac15ac3b2767a0e9d982ba0c8a4589a785678d17 Mon Sep 17 00:00:00 2001 From: Jason Murray Date: Sat, 31 Oct 2026 10:59:21 -0700 Subject: [PATCH 3/6] Fix coverage workflow rejected as invalid GitHub parsed the literal expression opener inside the embedded Python as an expression. Test for the closing-brace pair instead. --- .github/workflows/coverage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 0005a1b..5df0bfe 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -112,7 +112,7 @@ jobs: lines = [] if env is not None: for k, v in env.value: - if "${{" not in v.value: + if "{{" not in v.value: lines.append(f"{k.value}={v.value}") (out / f"step_{i}.env").write_text("\n".join(lines) + "\n") PY From 843aaecf5ac70d28fd911bf53581d4ff3cad1ea6 Mon Sep 17 00:00:00 2001 From: Jason Murray Date: Sat, 31 Oct 2026 11:01:02 -0700 Subject: [PATCH 4/6] Upload only the merged report to Codecov Disable Codecov's file search so it does not also pick up kcov's per-scenario reports under coverage/kcov. --- .github/workflows/coverage.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 5df0bfe..c48c2f6 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -210,5 +210,6 @@ jobs: uses: codecov/codecov-action@v5 with: files: coverage/cobertura.xml + disable_search: true token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true From c12da274b398e6694bea2c70213794e37637b4cc Mon Sep 17 00:00:00 2001 From: Jason Murray Date: Sat, 31 Oct 2026 12:39:36 -0700 Subject: [PATCH 5/6] Fix kcov cache key: ImageOS is not in the env context The key ended in a trailing dash with no OS component. Read the release from /etc/os-release in a step instead. --- .github/workflows/coverage.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index c48c2f6..4e830dc 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -40,13 +40,17 @@ jobs: # kcov is built from source because Ubuntu 24.04 (ubuntu-latest) no # longer packages it, and the v42 release binary targets Ubuntu 22.04. - # The install prefix is cached per kcov version and runner image OS. + # The install prefix is cached per kcov version and runner OS release. + - name: Detect runner OS release + id: os-release + run: echo "id=$(. /etc/os-release && echo "$ID-$VERSION_ID")" >> "$GITHUB_OUTPUT" + - name: Cache kcov id: kcov-cache uses: actions/cache@v4 with: path: ~/kcov - key: kcov-${{ env.KCOV_VERSION }}-${{ runner.os }}-${{ runner.arch }}-${{ env.ImageOS }} + key: kcov-${{ env.KCOV_VERSION }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.os-release.outputs.id }} - name: Install kcov dependencies # Needed at runtime too, so this runs even on a cache hit. From 0d443fcae89365520660eeac331f1fc50f293dc8 Mon Sep 17 00:00:00 2001 From: Jason Murray Date: Sat, 31 Oct 2026 12:47:28 -0700 Subject: [PATCH 6/6] Cover the working-directory input in the coverage replay main added a working-directory input, which the replay did not set, so the step failed on an unbound variable. Set it for every scenario and add relative, absolute and nonexistent cases to mirror the smoke tests. --- .github/workflows/coverage.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 4e830dc..640949a 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -152,7 +152,7 @@ jobs: local name=$1 shift env SPLIT_TOTAL=5 SPLIT_INDEX=0 GLOB="$GLOB" \ - JUNIT_PATH= LINE_COUNT= EXCLUDE_GLOB_INPUT= \ + JUNIT_PATH= LINE_COUNT= EXCLUDE_GLOB_INPUT= WORKING_DIRECTORY= \ GITHUB_OUTPUT="$(mktemp)" "$@" \ kcov --include-pattern=step_ "$OUT/$name" "$STEPS/step_1.sh" } @@ -161,6 +161,14 @@ jobs: scenario junit JUNIT_PATH="$JUNIT" scenario line-count LINE_COUNT=true scenario exclude-glob EXCLUDE_GLOB_INPUT='examples/spec/api/**/t*_spec.rb' + scenario working-directory-relative WORKING_DIRECTORY=examples GLOB='spec/**/*_spec.rb' + scenario working-directory-absolute WORKING_DIRECTORY="$GITHUB_WORKSPACE/examples" GLOB='spec/**/*_spec.rb' + + # The step must reject a nonexistent working-directory. + if scenario working-directory-missing WORKING_DIRECTORY=examples/does-not-exist GLOB='spec/**/*_spec.rb'; then + echo "::error::A nonexistent working-directory was accepted" + exit 1 + fi - name: Build Cobertura report for action.yml # Merges the scenarios, then maps the step scripts onto action.yml as a