From c6e42adf86e5f71f7d9ff3bb2f506fd399a85ec6 Mon Sep 17 00:00:00 2001 From: femi Date: Thu, 12 Nov 2026 09:04:25 +0000 Subject: [PATCH] feat(stats): report testrun stats to the tcache The tcache has a /stats endpoint, but nothing calls it, so no test run records how many tests it ran or how long it took. scripts/stats_json.py builds a small JSON document from the Allure results: the counts, grouped by historyId the same way as count_test_results.py, plus the run identity, the wall-clock duration, the node and CLI versions and the CLI coverage numbers. runner/report_stats.sh uploads it with one PUT. regression.sh sends one row per run and node_upgrade_pytest.sh one row per step. A run is identified by project, testrun name, run id, step and origin. Each workflow's runner/env_* file names its run through TCACHE_TESTRUN_NAME; a typed CI_TESTRUN_NAME wins and marks the run as a release. The run id is GITHUB_RUN_ID on CI. A local upgrade run gets one id from node_upgrade.sh, exported as TCACHE_RUN_ID, so its steps are stored as one run. A run under MARKEXPR or DESELECT_FROM_FILE is marked filtered. The upload is off unless ENABLE_TCACHE_STATS is truthy. It reuses TCACHE_BASIC_AUTH and TCACHE_URL, every call is "|| :" and curl has a timeout, so it can neither change the exit code nor hold up a run. The reusable workflows pass the tcache secrets and the flag to the test step; upgrade_reusable.yaml did not declare the secrets at all. The nightly, nightly-cli, nightly-dbsync and regression-leios workflows still do not pass them (#3624). --- .github/workflows/nightly_upgrade.yaml | 2 + .github/workflows/regression_reusable.yaml | 3 + .github/workflows/upgrade.yaml | 3 + .github/workflows/upgrade_reusable.yaml | 7 + .pre-commit-config.yaml | 2 +- README.md | 29 +- runner/env_leios | 1 + runner/env_nightly | 1 + runner/env_nightly_cli | 1 + runner/env_nightly_dbsync | 1 + runner/env_nightly_upgrade | 1 + runner/env_regression | 1 + runner/env_regression_dbsync | 1 + runner/env_upgrade | 1 + runner/node_upgrade.sh | 5 + runner/node_upgrade_pytest.sh | 10 + runner/regression.sh | 3 + runner/report_stats.sh | 72 ++++ scripts/stats_json.py | 366 +++++++++++++++++++++ 19 files changed, 495 insertions(+), 15 deletions(-) create mode 100755 runner/report_stats.sh create mode 100755 scripts/stats_json.py diff --git a/.github/workflows/nightly_upgrade.yaml b/.github/workflows/nightly_upgrade.yaml index 27fe7d32a..2f268dbc9 100644 --- a/.github/workflows/nightly_upgrade.yaml +++ b/.github/workflows/nightly_upgrade.yaml @@ -20,3 +20,5 @@ jobs: GMAIL_USERNAME: ${{ secrets.GMAIL_USERNAME }} GMAIL_PASSWORD: ${{ secrets.GMAIL_PASSWORD }} COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + TCACHE_BASIC_AUTH: ${{ secrets.TCACHE_BASIC_AUTH }} + TCACHE_URL: ${{ secrets.TCACHE_URL }} diff --git a/.github/workflows/regression_reusable.yaml b/.github/workflows/regression_reusable.yaml index 0704019ba..f15bbe69c 100644 --- a/.github/workflows/regression_reusable.yaml +++ b/.github/workflows/regression_reusable.yaml @@ -147,6 +147,9 @@ jobs: id: testing-step env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TCACHE_BASIC_AUTH: ${{ secrets.TCACHE_BASIC_AUTH }} + TCACHE_URL: ${{ secrets.TCACHE_URL }} + ENABLE_TCACHE_STATS: ${{ vars.ENABLE_TCACHE_STATS }} run: | # env echo "::group::Script setup" diff --git a/.github/workflows/upgrade.yaml b/.github/workflows/upgrade.yaml index 37603e6fa..9d59c9e10 100644 --- a/.github/workflows/upgrade.yaml +++ b/.github/workflows/upgrade.yaml @@ -42,3 +42,6 @@ jobs: upgrade_prot_ver: ${{ inputs.upgrade_prot_ver }} byron_cluster: ${{ inputs.byron_cluster }} env-path: runner/env_upgrade + secrets: + TCACHE_BASIC_AUTH: ${{ secrets.TCACHE_BASIC_AUTH }} + TCACHE_URL: ${{ secrets.TCACHE_URL }} diff --git a/.github/workflows/upgrade_reusable.yaml b/.github/workflows/upgrade_reusable.yaml index 3b89810f1..9dc92f068 100644 --- a/.github/workflows/upgrade_reusable.yaml +++ b/.github/workflows/upgrade_reusable.yaml @@ -39,6 +39,10 @@ on: required: false COPILOT_GITHUB_TOKEN: required: false + TCACHE_BASIC_AUTH: + required: false + TCACHE_URL: + required: false env: PY_COLORS: "1" @@ -83,6 +87,9 @@ jobs: id: testing-step env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TCACHE_BASIC_AUTH: ${{ secrets.TCACHE_BASIC_AUTH }} + TCACHE_URL: ${{ secrets.TCACHE_URL }} + ENABLE_TCACHE_STATS: ${{ vars.ENABLE_TCACHE_STATS }} run: | # env echo "::group::Script setup" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ab5bfbac..faa05c790 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,4 +74,4 @@ repos: # the full dependency graph. pass_filenames: false require_serial: true - args: [cardano_node_tests, framework_tests, scripts/count_test_results.py, src_docs/source/conf.py] + args: [cardano_node_tests, framework_tests, scripts/count_test_results.py, scripts/stats_json.py, src_docs/source/conf.py] diff --git a/README.md b/README.md index 216f1c260..ae9bf135b 100644 --- a/README.md +++ b/README.md @@ -182,20 +182,21 @@ You can fine-tune test runs using these environment variables: ### Additional for `regression.sh` -| Variable | Description | -| ------------------- | -------------------------------------------------------- | -| `CARDANO_CLI_REV` | `cardano-cli` version. | -| `CI_BYRON_CLUSTER` | Run cluster from Byron ➝ Conway (slow start). | -| `DBSYNC_REV` | `cardano-db-sync` version. | -| `NODE_REV` | `cardano-node` version (default: `master`). | -| `TX_CENTRIFUGE_REV` | `tx-centrifuge` version (default: `bench/leios-11.0.1`). | -| `TX_FIREHOSE_REV` | `tx-firehose` version (default: `leios-prototype`). | -| `PYTEST_ARGS` | Extra options passed to pytest. | -| `TEST_THREADS` | Number of pytest workers (default: `20`). | -| `MONITOR_INTERVAL` | Resource snapshot interval in seconds (default: `60`). | -| `TIME_DRIFT_POOL` | Pool (1-3) for libfaketime clock skew; unset = off. | -| `TIME_DRIFT_SPEC` | libfaketime `FAKETIME` string, e.g. `+0 x1.008`. | -| `LIBFAKETIME_PATH` | Override path to `libfaketimeMT.so.1`. | +| Variable | Description | +| --------------------- | -------------------------------------------------------- | +| `CARDANO_CLI_REV` | `cardano-cli` version. | +| `CI_BYRON_CLUSTER` | Run cluster from Byron ➝ Conway (slow start). | +| `DBSYNC_REV` | `cardano-db-sync` version. | +| `ENABLE_TCACHE_STATS` | Upload testrun stats to the tcache (default: `false`). | +| `NODE_REV` | `cardano-node` version (default: `master`). | +| `TX_CENTRIFUGE_REV` | `tx-centrifuge` version (default: `bench/leios-11.0.1`). | +| `TX_FIREHOSE_REV` | `tx-firehose` version (default: `leios-prototype`). | +| `PYTEST_ARGS` | Extra options passed to pytest. | +| `TEST_THREADS` | Number of pytest workers (default: `20`). | +| `MONITOR_INTERVAL` | Resource snapshot interval in seconds (default: `60`). | +| `TIME_DRIFT_POOL` | Pool (1-3) for libfaketime clock skew; unset = off. | +| `TIME_DRIFT_SPEC` | libfaketime `FAKETIME` string, e.g. `+0 x1.008`. | +| `LIBFAKETIME_PATH` | Override path to `libfaketimeMT.so.1`. | ### 💡 Usage Examples diff --git a/runner/env_leios b/runner/env_leios index 961685c07..e3c6c9924 100644 --- a/runner/env_leios +++ b/runner/env_leios @@ -3,3 +3,4 @@ ENABLE_TX_FIREHOSE=true TX_TPS=15 COMMAND_ERA=dijkstra PROTOCOL_VERSION=12 +TCACHE_TESTRUN_NAME=leios diff --git a/runner/env_nightly b/runner/env_nightly index 1437ab183..0a547ba45 100644 --- a/runner/env_nightly +++ b/runner/env_nightly @@ -1 +1,2 @@ COMMAND_ERA=latest +TCACHE_TESTRUN_NAME=nightly diff --git a/runner/env_nightly_cli b/runner/env_nightly_cli index f0f6a4ed1..26aa7af03 100644 --- a/runner/env_nightly_cli +++ b/runner/env_nightly_cli @@ -1,2 +1,3 @@ COMMAND_ERA=conway CARDANO_CLI_REV=master +TCACHE_TESTRUN_NAME=nightly-cli diff --git a/runner/env_nightly_dbsync b/runner/env_nightly_dbsync index 19a755066..047465aba 100644 --- a/runner/env_nightly_dbsync +++ b/runner/env_nightly_dbsync @@ -3,3 +3,4 @@ DBSYNC_REV=13.8.0.0 DBSYNC_TAR_URL=https://github.com/IntersectMBO/cardano-db-sync/releases/download/13.8.0.0/cardano-db-sync-13.8.0.0-linux-amd64.tar.gz SMASH=true DBSYNC_SKIP_INDEXES=true +TCACHE_TESTRUN_NAME=nightly-dbsync diff --git a/runner/env_nightly_upgrade b/runner/env_nightly_upgrade index 90dfc85a2..2227817cf 100644 --- a/runner/env_nightly_upgrade +++ b/runner/env_nightly_upgrade @@ -1,2 +1,3 @@ BASE_TAR_URL=https://github.com/IntersectMBO/cardano-node/releases/download/11.1.3/cardano-node-11.1.3-linux-amd64.tar.gz CI_BYRON_CLUSTER=true +TCACHE_TESTRUN_NAME=nightly-upgrade diff --git a/runner/env_regression b/runner/env_regression index e69de29bb..fef3ad808 100644 --- a/runner/env_regression +++ b/runner/env_regression @@ -0,0 +1 @@ +TCACHE_TESTRUN_NAME=regression diff --git a/runner/env_regression_dbsync b/runner/env_regression_dbsync index dfc9a4dde..b3e23fcc9 100644 --- a/runner/env_regression_dbsync +++ b/runner/env_regression_dbsync @@ -1 +1,2 @@ DBSYNC_SKIP_INDEXES=true +TCACHE_TESTRUN_NAME=regression-dbsync diff --git a/runner/env_upgrade b/runner/env_upgrade index e69de29bb..a539ae872 100644 --- a/runner/env_upgrade +++ b/runner/env_upgrade @@ -0,0 +1 @@ +TCACHE_TESTRUN_NAME=upgrade diff --git a/runner/node_upgrade.sh b/runner/node_upgrade.sh index b87e6fadf..73900399e 100755 --- a/runner/node_upgrade.sh +++ b/runner/node_upgrade.sh @@ -89,6 +89,11 @@ export SCHEDULING_LOG="${WORKDIR}/scheduling.log" export DEV_CLUSTER_RUNNING=true FORBID_RESPIN=true CLUSTERS_COUNT=1 TEST_THREADS=10 NUM_POOLS="${NUM_POOLS:-4}" +# A local run has no CI run id. Give all its steps one id, so the tcache +# stores them as one testrun. +TCACHE_RUN_ID="local-$(TZ=UTC printf '%(%Y%m%dT%H%M%SZ)T' -1)-$(printf '%08x' "$SRANDOM")" +export TCACHE_RUN_ID + echo "::endgroup::" # end group for "Script setup" echo "::group::Nix env setup step1" diff --git a/runner/node_upgrade_pytest.sh b/runner/node_upgrade_pytest.sh index bbe4c7c2e..3d7d65713 100755 --- a/runner/node_upgrade_pytest.sh +++ b/runner/node_upgrade_pytest.sh @@ -154,6 +154,10 @@ if [ "$1" = "step1" ]; then # create results archive for step1 ./runner/create_results.sh "$reports_dir" "$WORKDIR" allure-results-step1 + # report the step stats to the tcache, without CLI coverage: it is built only + # in the finish step + ./runner/report_stats.sh "${WORKDIR}/allure-results-step1" "$retval" "" step1 || : + printf "STEP1 finish: %(%H:%M:%S)T\n" -1 # @@ -325,6 +329,9 @@ elif [ "$1" = "step2" ]; then [ "$err_retval" -gt "$retval" ] && retval=1 + # report the step stats to the tcache + ./runner/report_stats.sh "${WORKDIR}/allure-results-step2" "$retval" "" step2 || : + printf "STEP2 finish: %(%H:%M:%S)T\n" -1 @@ -510,6 +517,9 @@ elif [ "$1" = "step3" ]; then [ "$err_retval" -gt "$retval" ] && retval=1 + # report the step stats to the tcache + ./runner/report_stats.sh "${WORKDIR}/allure-results-step3" "$retval" "" step3 || : + printf "STEP3 finish: %(%H:%M:%S)T\n" -1 # diff --git a/runner/regression.sh b/runner/regression.sh index 92dbd3dd9..fd7556c87 100755 --- a/runner/regression.sh +++ b/runner/regression.sh @@ -483,6 +483,9 @@ fi # Save testing artifacts ./runner/save_artifacts.sh "$ARTIFACTS_DIR" "$WORKDIR" || : +# Report testrun stats to the tcache +./runner/report_stats.sh "${WORKDIR}/allure-results" "$retval" "${WORKDIR}/cli_coverage.json" || : + kill_monitor if [ "$retval" -eq 0 ]; then diff --git a/runner/report_stats.sh b/runner/report_stats.sh new file mode 100755 index 000000000..737fd3164 --- /dev/null +++ b/runner/report_stats.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +# Upload testrun statistics to the tcache. Best effort: call it with `|| :` +# so a failure cannot change the caller's exit code. + +set -euo pipefail + +if [ "$#" -lt 2 ] || [ "$#" -gt 4 ]; then + echo "Usage: $0 [cli_coverage_json] [step]" >&2 + exit 1 +fi + +results_dir="$1" +exit_code="$2" +coverage_json="${3:-}" +step="${4:-main}" + +tests_repo="$(cd "$(dirname "$0")/.." && pwd)" || { echo "Cannot determine test repo dir, exiting." >&2; exit 1; } + +# shellcheck disable=SC1091 +. "${tests_repo}/scripts/common.sh" + +if ! is_truthy "${ENABLE_TCACHE_STATS:-}"; then + echo "ENABLE_TCACHE_STATS is not enabled, not reporting stats." + exit 0 +fi + +if [ -z "${TCACHE_BASIC_AUTH:-}" ] || [ -z "${TCACHE_URL:-}" ]; then + echo "TCACHE_BASIC_AUTH or TCACHE_URL is not set, not reporting stats." + exit 0 +fi + +if [ ! -d "$results_dir" ]; then + echo "No Allure results in '$results_dir', not reporting stats." >&2 + exit 1 +fi + +# regression.sh adds PATH_PREPEND to PATH only inside its nix shell, so add it +# again here for the `--version` calls. +if [ -n "${PATH_PREPEND:-}" ]; then + PATH="${PATH_PREPEND}:${PATH}" + export PATH +fi + +# `TCACHE_URL` ends in `/results`. +root="${TCACHE_URL%/}" +stats_url="${root%/results}/stats" + +stats_json="$(mktemp)" +# shellcheck disable=SC2064 +trap "rm -f '$stats_json'" EXIT + +extra_args=() +if [ -n "$coverage_json" ] && [ -e "$coverage_json" ]; then + extra_args+=(--cli-coverage "$coverage_json") +fi +if [ -n "${MARKEXPR:-}" ] || [ -n "${DESELECT_FROM_FILE:-}" ]; then + extra_args+=(--filtered) +fi + +if ! "${tests_repo}/scripts/stats_json.py" "$results_dir" \ + --exit-code "$exit_code" --step "$step" "${extra_args[@]}" \ + --output "$stats_json"; then + echo "Could not build the stats document, not reporting stats." >&2 + exit 1 +fi + +echo "Reporting testrun stats to $stats_url" +# Time out so a tcache that never answers cannot hold up the testrun. +curl -sS -X PUT --fail-with-body --connect-timeout 10 --max-time 60 \ + -u "$TCACHE_BASIC_AUTH" "$stats_url" \ + -H "Content-Type: application/json" --data-binary "@$stats_json" diff --git a/scripts/stats_json.py b/scripts/stats_json.py new file mode 100755 index 000000000..ffd3bc177 --- /dev/null +++ b/scripts/stats_json.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python3 +"""Emit a testrun statistics document for upload to the tcache. + +Usage: + scripts/stats_json.py [options] + +The counts come from the grouping logic in `count_test_results.py`. This +script adds the run identity, timing, software versions and CLI coverage. +""" + +import argparse +import contextlib +import datetime +import json +import os +import pathlib as pl +import re +import subprocess +import sys +import typing as tp +import uuid + +# `scripts` is not a package, so the sibling module is imported by path. +sys.path.insert(0, str(pl.Path(__file__).resolve().parent)) + +import count_test_results as counter + +SCHEMA = 1 +DEFAULT_STEP = "main" +DEFAULT_TESTRUN_NAME = "local" +DEFAULT_PROJECT = "cardano-node-tests" + +# Allure timestamps are milliseconds since the epoch. +MS_PER_SECOND = 1000 + + +def _utc_now() -> datetime.datetime: + """Return the current time as a tz-aware UTC datetime. + + Returns: + The current UTC time. + """ + return datetime.datetime.now(tz=datetime.UTC) + + +def _local_run_id() -> str: + """Build a run id, with no host or user name, for a run that was given none. + + Returns: + A sortable id, e.g. `local-20260929T120000Z-1a2b3c4d`. + """ + stamp = _utc_now().strftime("%Y%m%dT%H%M%SZ") + return f"local-{stamp}-{uuid.uuid4().hex[:8]}" + + +def _project() -> str: + """Return the project the testrun belongs to. + + Returns: + The CI repository name, else the default. + """ + repository = os.environ.get("GITHUB_REPOSITORY") + if repository: + return repository.rsplit("/", maxsplit=1)[-1] + return DEFAULT_PROJECT + + +def _scrub(name: str) -> str: + """Keep only ASCII letters, digits, `_` and `-` in a testrun name. + + The same set the workflows keep with `${CI_TESTRUN_NAME//[!a-zA-Z0-9_-]/}`, + without depending on the locale. + + Args: + name: The raw name. + + Returns: + The scrubbed name. + """ + return re.sub(r"[^A-Za-z0-9_-]", "", name) + + +def _typed_testrun_name() -> str: + """Return the testrun name typed for a manual run, scrubbed. + + Returns: + The scrubbed name, or an empty string when none was typed. + """ + return _scrub(os.environ.get("CI_TESTRUN_NAME", "")) + + +def _identity(step: str) -> dict[str, str]: + """Gather the five fields that identify a testrun. + + Args: + step: The step name, for the upgrade path. + + Returns: + The identity fields. + """ + on_ci = bool(os.environ.get("GITHUB_ACTIONS")) + # A name typed for a release run wins over the default in the workflow's env file. + testrun_name = ( + _typed_testrun_name() + or _scrub(os.environ.get("TCACHE_TESTRUN_NAME", "")) + or DEFAULT_TESTRUN_NAME + ) + return { + "project": _project(), + "testrun_name": testrun_name, + "run_id": os.environ.get("GITHUB_RUN_ID") + or os.environ.get("TCACHE_RUN_ID") + or _local_run_id(), + "step": step, + "origin": "ci" if on_ci else "local", + } + + +def _tool_version(command: str) -> str | None: + """Read a tool's version from its `--version` output. + + Args: + command: The executable to ask. + + Returns: + The version field, or None when the tool is missing or silent. + """ + try: + completed = subprocess.run( + [command, "--version"], capture_output=True, check=False, text=True, timeout=30 + ) + except (OSError, subprocess.SubprocessError): + return None + fields = completed.stdout.split() + return fields[1] if len(fields) > 1 else None + + +def _versions() -> dict[str, str | None]: + """Collect the software versions the counts were produced with. + + Returns: + A version per tool, with None where the tool is not installed. + """ + return { + "cardano_node": _tool_version("cardano-node"), + "cardano_cli": _tool_version("cardano-cli"), + "db_sync": _tool_version("cardano-db-sync"), + } + + +def _submits(node: dict[str, tp.Any]) -> int: + """Sum the `transaction submit` invocations in a coverage report subtree. + + Args: + node: A subtree of the CLI coverage report. + + Returns: + The number of times `transaction submit` ran, in every era group. + """ + total = 0 + for key, child in node.items(): + if not isinstance(child, dict): + continue + submit = child.get("submit") if key == "transaction" else None + if isinstance(submit, dict): + total += int(submit.get("_count_submit", 0)) + total += _submits(child) + return total + + +def _commands(coverage_file: pl.Path | None) -> dict[str, int | float | None]: + """Read the CLI coverage summary written by `runner/cli_coverage.sh`. + + Args: + coverage_file: Path to `cli_coverage.json`, or None to skip. + + Returns: + The invocation count, the transactions submitted and the coverage + percentage, all None when the file is missing or empty. + """ + empty: dict[str, int | float | None] = {"count": None, "submits": None, "coverage_pct": None} + if coverage_file is None or not coverage_file.is_file(): + return empty + try: + report = json.loads(coverage_file.read_text(encoding="utf-8")) + # One top-level entry, named after the tool it covers. + tool, covered = next(iter(report.items())) + count = int(covered[f"_count_{tool}"]) + coverage = float(covered[f"_coverage_{tool}"]) + except (OSError, ValueError, TypeError, KeyError, StopIteration, AttributeError): + return empty + return {"count": count, "submits": _submits(covered), "coverage_pct": coverage} + + +def _timing(results_dir: pl.Path) -> tuple[str | None, float]: + """Derive when the testrun started and its wall clock duration. + + Args: + results_dir: The Allure results directory. + + Returns: + Tuple of (ISO-8601 start time or None, duration in seconds). + """ + starts: list[float] = [] + stops: list[float] = [] + for fpath in results_dir.iterdir(): + if not fpath.name.endswith("-result.json"): + continue + with contextlib.suppress(OSError, ValueError, TypeError, AttributeError): + record = json.loads(fpath.read_text(encoding="utf-8")) + message = (record.get("statusDetails") or {}).get("message") or "" + if message == counter.SKIPALL_MSG: + continue + start, stop = record.get("start"), record.get("stop") + if isinstance(start, (int, float)) and isinstance(stop, (int, float)): + starts.append(start) + stops.append(stop) + + if not starts: + return None, 0.0 + began = datetime.datetime.fromtimestamp(min(starts) / MS_PER_SECOND, tz=datetime.UTC) + return began.isoformat(), (max(stops) - min(starts)) / MS_PER_SECOND + + +def _counts(best: dict[str, counter._TestResult]) -> dict[str, int]: + """Count the grouped results by Allure status. + + Args: + best: The authoritative result per test, from `count_test_results`. + + Returns: + The total and the four status buckets. + """ + tally: dict[str, int] = {} + for result in best.values(): + tally[result.status] = tally.get(result.status, 0) + 1 + return { + "total": len(best), + "passed": tally.get("passed", 0), + "failed": tally.get("failed", 0), + "broken": tally.get("broken", 0), + "skipped": tally.get("skipped", 0), + } + + +def _quality( + best: dict[str, counter._TestResult], no_history_id: int, read_errors: int +) -> dict[str, int]: + """Report how far the counts can be trusted. + + `never_run` counts registered tests that never got a real result, which + means the testrun was interrupted. + + Args: + best: The authoritative result per test. + no_history_id: Result files that carried no `historyId`. + read_errors: Result files that could not be read. + + Returns: + The three counters that qualify the counts. + """ + return { + "never_run": sum(1 for result in best.values() if not result.is_real), + "no_history_id": no_history_id, + "read_errors": read_errors, + } + + +def build_document( + results_dir: pl.Path, + exit_code: int, + step: str = DEFAULT_STEP, + coverage_file: pl.Path | None = None, + filtered: bool = False, +) -> dict[str, tp.Any]: + """Build the statistics document for one testrun. + + Args: + results_dir: The Allure results directory. + exit_code: The testrun's exit code. + step: The step name, for the upgrade path. + coverage_file: Path to `cli_coverage.json`, or None. + filtered: True when the run covered only a subset of the tests. + + Returns: + The document to upload. + + Raises: + NotADirectoryError: When the results directory does not exist. + """ + if not results_dir.is_dir(): + err = f"Not a directory: {results_dir}" + raise NotADirectoryError(err) + + result_files = sorted(f for f in results_dir.iterdir() if f.name.endswith("-result.json")) + # Reuse the private grouping helper so the two scripts cannot drift. + best, no_history_id, read_errors = counter._group_records(result_files) + began, duration = _timing(results_dir) + + document = { + "schema": SCHEMA, + **_identity(step), + "timestamp": began or _utc_now().isoformat(), + "duration": duration, + "exit_code": exit_code, + "filtered": filtered, + "release": bool(_typed_testrun_name()), + "counts": _counts(best), + "quality": _quality(best, no_history_id, read_errors), + "versions": _versions(), + "commands": _commands(coverage_file), + } + return document + + +def get_args() -> argparse.Namespace: + """Parse the command line. + + Returns: + The parsed arguments. + """ + parser = argparse.ArgumentParser(description=__doc__.partition("\n")[0]) + parser.add_argument("results_dir", type=pl.Path, help="Allure results directory") + parser.add_argument("-e", "--exit-code", type=int, default=0, help="the testrun's exit code") + parser.add_argument("-s", "--step", default=DEFAULT_STEP, help="step name, for upgrade testing") + parser.add_argument( + "-c", "--cli-coverage", type=pl.Path, default=None, help="path to cli_coverage.json" + ) + parser.add_argument( + "-f", "--filtered", action="store_true", help="the run covered a subset of the tests" + ) + parser.add_argument( + "-o", "--output", type=pl.Path, default=None, help="write here instead of stdout" + ) + return parser.parse_args() + + +def main() -> int: + """Write the statistics document. + + Returns: + 0 on success, 1 when the results directory cannot be read. + """ + args = get_args() + try: + document = build_document( + results_dir=args.results_dir, + exit_code=args.exit_code, + step=args.step, + coverage_file=args.cli_coverage, + filtered=args.filtered, + ) + except OSError as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + + rendered = json.dumps(document, indent=2, sort_keys=True) + if args.output: + args.output.write_text(f"{rendered}\n", encoding="utf-8") + else: + print(rendered) + return 0 + + +if __name__ == "__main__": + sys.exit(main())