feat(stats): report testrun stats to the tcache

.github/workflows/nightly_upgrade.yaml

@@ -20,3 +20,5 @@ jobs:
       GMAIL_USERNAME: ${{ secrets.GMAIL_USERNAME }}
       GMAIL_PASSWORD: ${{ secrets.GMAIL_PASSWORD }}
       COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
+      TCACHE_BASIC_AUTH: ${{ secrets.TCACHE_BASIC_AUTH }}
+      TCACHE_URL: ${{ secrets.TCACHE_URL }}

.github/workflows/regression_reusable.yaml

@@ -147,6 +147,9 @@ jobs:
         id: testing-step
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TCACHE_BASIC_AUTH: ${{ secrets.TCACHE_BASIC_AUTH }}
+          TCACHE_URL: ${{ secrets.TCACHE_URL }}
+          ENABLE_TCACHE_STATS: ${{ vars.ENABLE_TCACHE_STATS }}
         run: |
           # env
           echo "::group::Script setup"

.github/workflows/upgrade.yaml

@@ -42,3 +42,6 @@ jobs:
       upgrade_prot_ver: ${{ inputs.upgrade_prot_ver }}
       byron_cluster: ${{ inputs.byron_cluster }}
       env-path: runner/env_upgrade
+    secrets:
+      TCACHE_BASIC_AUTH: ${{ secrets.TCACHE_BASIC_AUTH }}
+      TCACHE_URL: ${{ secrets.TCACHE_URL }}

.github/workflows/upgrade_reusable.yaml

@@ -39,6 +39,10 @@ on:
         required: false
       COPILOT_GITHUB_TOKEN:
         required: false
+      TCACHE_BASIC_AUTH:
+        required: false
+      TCACHE_URL:
+        required: false
 
 env:
   PY_COLORS: "1"
@@ -83,6 +87,9 @@ jobs:
         id: testing-step
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TCACHE_BASIC_AUTH: ${{ secrets.TCACHE_BASIC_AUTH }}
+          TCACHE_URL: ${{ secrets.TCACHE_URL }}
+          ENABLE_TCACHE_STATS: ${{ vars.ENABLE_TCACHE_STATS }}
         run: |
           # env
           echo "::group::Script setup"

.pre-commit-config.yaml

@@ -74,4 +74,4 @@ repos:
     # the full dependency graph.
     pass_filenames: false
     require_serial: true
-    args: [cardano_node_tests, framework_tests, scripts/count_test_results.py, src_docs/source/conf.py]
+    args: [cardano_node_tests, framework_tests, scripts/count_test_results.py, scripts/stats_json.py, src_docs/source/conf.py]

README.md

@@ -182,20 +182,21 @@ You can fine-tune test runs using these environment variables:
 
 ### Additional for `regression.sh`
 
-| Variable            | Description                                              |
-| ------------------- | -------------------------------------------------------- |
-| `CARDANO_CLI_REV`   | `cardano-cli` version.                                   |
-| `CI_BYRON_CLUSTER`  | Run cluster from Byron ➝ Conway (slow start).            |
-| `DBSYNC_REV`        | `cardano-db-sync` version.                               |
-| `NODE_REV`          | `cardano-node` version (default: `master`).              |
-| `TX_CENTRIFUGE_REV` | `tx-centrifuge` version (default: `bench/leios-11.0.1`). |
-| `TX_FIREHOSE_REV`   | `tx-firehose` version (default: `leios-prototype`).      |
-| `PYTEST_ARGS`       | Extra options passed to pytest.                          |
-| `TEST_THREADS`      | Number of pytest workers (default: `20`).                |
-| `MONITOR_INTERVAL`  | Resource snapshot interval in seconds (default: `60`).   |
-| `TIME_DRIFT_POOL`   | Pool (1-3) for libfaketime clock skew; unset = off.      |
-| `TIME_DRIFT_SPEC`   | libfaketime `FAKETIME` string, e.g. `+0 x1.008`.         |
-| `LIBFAKETIME_PATH`  | Override path to `libfaketimeMT.so.1`.                   |
+| Variable              | Description                                              |
+| --------------------- | -------------------------------------------------------- |
+| `CARDANO_CLI_REV`     | `cardano-cli` version.                                   |
+| `CI_BYRON_CLUSTER`    | Run cluster from Byron ➝ Conway (slow start).            |
+| `DBSYNC_REV`          | `cardano-db-sync` version.                               |
+| `ENABLE_TCACHE_STATS` | Upload testrun stats to the tcache (default: `false`).   |
+| `NODE_REV`            | `cardano-node` version (default: `master`).              |
+| `TX_CENTRIFUGE_REV`   | `tx-centrifuge` version (default: `bench/leios-11.0.1`). |
+| `TX_FIREHOSE_REV`     | `tx-firehose` version (default: `leios-prototype`).      |
+| `PYTEST_ARGS`         | Extra options passed to pytest.                          |
+| `TEST_THREADS`        | Number of pytest workers (default: `20`).                |
+| `MONITOR_INTERVAL`    | Resource snapshot interval in seconds (default: `60`).   |
+| `TIME_DRIFT_POOL`     | Pool (1-3) for libfaketime clock skew; unset = off.      |
+| `TIME_DRIFT_SPEC`     | libfaketime `FAKETIME` string, e.g. `+0 x1.008`.         |
+| `LIBFAKETIME_PATH`    | Override path to `libfaketimeMT.so.1`.                   |
 
 ### 💡 Usage Examples
 

runner/env_leios

@@ -3,3 +3,4 @@ ENABLE_TX_FIREHOSE=true
 TX_TPS=15
 COMMAND_ERA=dijkstra
 PROTOCOL_VERSION=12
+TCACHE_TESTRUN_NAME=leios

runner/env_nightly

@@ -1 +1,2 @@
 COMMAND_ERA=latest
+TCACHE_TESTRUN_NAME=nightly

runner/env_nightly_cli

@@ -1,2 +1,3 @@
 COMMAND_ERA=conway
 CARDANO_CLI_REV=master
+TCACHE_TESTRUN_NAME=nightly-cli

runner/env_nightly_dbsync

@@ -3,3 +3,4 @@ DBSYNC_REV=13.8.0.0
 DBSYNC_TAR_URL=https://github.com/IntersectMBO/cardano-db-sync/releases/download/13.8.0.0/cardano-db-sync-13.8.0.0-linux-amd64.tar.gz
 SMASH=true
 DBSYNC_SKIP_INDEXES=true
+TCACHE_TESTRUN_NAME=nightly-dbsync

runner/env_nightly_upgrade

@@ -1,2 +1,3 @@
 BASE_TAR_URL=https://github.com/IntersectMBO/cardano-node/releases/download/11.1.3/cardano-node-11.1.3-linux-amd64.tar.gz
 CI_BYRON_CLUSTER=true
+TCACHE_TESTRUN_NAME=nightly-upgrade

runner/env_regression

@@ -0,0 +1 @@
+TCACHE_TESTRUN_NAME=regression

runner/env_regression_dbsync

@@ -1 +1,2 @@
 DBSYNC_SKIP_INDEXES=true
+TCACHE_TESTRUN_NAME=regression-dbsync

runner/env_upgrade

@@ -0,0 +1 @@
+TCACHE_TESTRUN_NAME=upgrade

runner/node_upgrade.sh

@@ -89,6 +89,11 @@ export SCHEDULING_LOG="${WORKDIR}/scheduling.log"
 
 export DEV_CLUSTER_RUNNING=true FORBID_RESPIN=true CLUSTERS_COUNT=1 TEST_THREADS=10 NUM_POOLS="${NUM_POOLS:-4}"
 
+# A local run has no CI run id. Give all its steps one id, so the tcache
+# stores them as one testrun.
+TCACHE_RUN_ID="local-$(TZ=UTC printf '%(%Y%m%dT%H%M%SZ)T' -1)-$(printf '%08x' "$SRANDOM")"
+export TCACHE_RUN_ID
+
 echo "::endgroup::"  # end group for "Script setup"
 
 echo "::group::Nix env setup step1"

runner/node_upgrade_pytest.sh

@@ -154,6 +154,10 @@ if [ "$1" = "step1" ]; then
   # create results archive for step1
   ./runner/create_results.sh "$reports_dir" "$WORKDIR" allure-results-step1
 
+  # report the step stats to the tcache, without CLI coverage: it is built only
+  # in the finish step
+  ./runner/report_stats.sh "${WORKDIR}/allure-results-step1" "$retval" "" step1 || :
+
   printf "STEP1 finish: %(%H:%M:%S)T\n" -1
 
 #
@@ -325,6 +329,9 @@ elif [ "$1" = "step2" ]; then
 
   [ "$err_retval" -gt "$retval" ] && retval=1
 
+  # report the step stats to the tcache
+  ./runner/report_stats.sh "${WORKDIR}/allure-results-step2" "$retval" "" step2 || :
+
   printf "STEP2 finish: %(%H:%M:%S)T\n" -1
 
 
@@ -510,6 +517,9 @@ elif [ "$1" = "step3" ]; then
 
   [ "$err_retval" -gt "$retval" ] && retval=1
 
+  # report the step stats to the tcache
+  ./runner/report_stats.sh "${WORKDIR}/allure-results-step3" "$retval" "" step3 || :
+
   printf "STEP3 finish: %(%H:%M:%S)T\n" -1
 
 #

runner/regression.sh

@@ -483,6 +483,9 @@ fi
 # Save testing artifacts
 ./runner/save_artifacts.sh "$ARTIFACTS_DIR" "$WORKDIR" || :
 
+# Report testrun stats to the tcache
+./runner/report_stats.sh "${WORKDIR}/allure-results" "$retval" "${WORKDIR}/cli_coverage.json" || :
+
 kill_monitor
 
 if [ "$retval" -eq 0 ]; then

runner/report_stats.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+# Upload testrun statistics to the tcache. Best effort: call it with `|| :`
+# so a failure cannot change the caller's exit code.
+
+set -euo pipefail
+
+if [ "$#" -lt 2 ] || [ "$#" -gt 4 ]; then
+  echo "Usage: $0 <allure_results_dir> <exit_code> [cli_coverage_json] [step]" >&2
+  exit 1
+fi
+
+results_dir="$1"
+exit_code="$2"
+coverage_json="${3:-}"
+step="${4:-main}"
+
+tests_repo="$(cd "$(dirname "$0")/.." && pwd)" || { echo "Cannot determine test repo dir, exiting." >&2; exit 1; }
+
+# shellcheck disable=SC1091
+. "${tests_repo}/scripts/common.sh"
+
+if ! is_truthy "${ENABLE_TCACHE_STATS:-}"; then
+  echo "ENABLE_TCACHE_STATS is not enabled, not reporting stats."
+  exit 0
+fi
+
+if [ -z "${TCACHE_BASIC_AUTH:-}" ] || [ -z "${TCACHE_URL:-}" ]; then
+  echo "TCACHE_BASIC_AUTH or TCACHE_URL is not set, not reporting stats."
+  exit 0
+fi
+
+if [ ! -d "$results_dir" ]; then
+  echo "No Allure results in '$results_dir', not reporting stats." >&2
+  exit 1
+fi
+
+# regression.sh adds PATH_PREPEND to PATH only inside its nix shell, so add it
+# again here for the `--version` calls.
+if [ -n "${PATH_PREPEND:-}" ]; then
+  PATH="${PATH_PREPEND}:${PATH}"
+  export PATH
+fi
+
+# `TCACHE_URL` ends in `/results`.
+root="${TCACHE_URL%/}"
+stats_url="${root%/results}/stats"
+
+stats_json="$(mktemp)"
+# shellcheck disable=SC2064
+trap "rm -f '$stats_json'" EXIT
+
+extra_args=()
+if [ -n "$coverage_json" ] && [ -e "$coverage_json" ]; then
+  extra_args+=(--cli-coverage "$coverage_json")
+fi
+if [ -n "${MARKEXPR:-}" ] || [ -n "${DESELECT_FROM_FILE:-}" ]; then
+  extra_args+=(--filtered)
+fi
+
+if ! "${tests_repo}/scripts/stats_json.py" "$results_dir" \
+       --exit-code "$exit_code" --step "$step" "${extra_args[@]}" \
+       --output "$stats_json"; then
+  echo "Could not build the stats document, not reporting stats." >&2
+  exit 1
+fi
+
+echo "Reporting testrun stats to $stats_url"
+# Time out so a tcache that never answers cannot hold up the testrun.
+curl -sS -X PUT --fail-with-body --connect-timeout 10 --max-time 60 \
+  -u "$TCACHE_BASIC_AUTH" "$stats_url" \
+  -H "Content-Type: application/json" --data-binary "@$stats_json"