feat(digest): surface standards-check warnings in the digest

scripts/dependabot-digest/render.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 # Render classify.sh output as the Markdown body of the digest issue.
 #
-# Usage: render.sh [--run-url URL] < classified.ndjson > body.md
+# Usage: render.sh [--run-url URL] [--standards FILE] < classified.ndjson > body.md
+# --standards takes standards.sh output.
 #
 # The body leads with what a human must do and states, per PR, the fact that
 # put it in its bucket. It never says a PR will merge: the merge path runs
@@ -11,9 +12,11 @@ set -uo pipefail
 unset CDPATH
 
 run_url=""
+standards_file=""
 while [[ $# -gt 0 ]]; do
   case "$1" in
     --run-url) run_url="${2-}"; shift 2 ;;
+    --standards) standards_file="${2-}"; shift 2 ;;
     *) echo "render.sh: unknown argument: $1" >&2; exit 2 ;;
   esac
 done
@@ -118,6 +121,99 @@ else
   fi
 fi
 
+# dev-env#179. Keep three outcomes apart: warnings, clean, not checked.
+render_standards() {
+  local file="$1" recs checked clean warned unchecked archived rows
+  echo "## Standards warnings"
+  echo
+  if [[ ! -r "${file}" ]]; then
+    echo "**Not checked.** The standards survey produced no readable result, so nothing below the Dependabot queue was verified."
+    echo
+    return 0
+  fi
+  # jq stops at a malformed line, dropping every repo after it.
+  if ! jq -e -s 'type == "array"' "${file}" >/dev/null 2>&1; then
+    echo "**Not checked.** The standards survey output is malformed, so nothing was verified."
+    echo
+    return 0
+  fi
+  recs="$(jq -c 'select(type == "object" and (.state | type == "string"))' "${file}" 2>/dev/null)"
+  if [[ -z "${recs}" ]]; then
+    echo "**Not checked.** The standards survey returned no repositories, so nothing was verified."
+    echo
+    return 0
+  fi
+  echo "Warning- and notice-level annotations from each repository's latest \`standards-check\` run that vetted its default branch. Fleet callers run only on pull requests, so that is usually the run on the head commit of the PR that produced the default-branch head. A PR run lints only the files that PR changed (node-floor always checks the whole repo), so \"no warnings\" means that run was clean, not that the whole repository is."
+  echo
+  checked="$(jq -s '[.[] | select(.state == "ok")] | length' <<<"${recs}")"
+  warned="$(jq -s '[.[] | select(.state == "ok" and (.annotations | length) > 0)] | length' <<<"${recs}")"
+  clean=$((checked - warned))
+  unchecked="$(jq -s '[.[] | select(.state != "ok" and .state != "archived")] | length' <<<"${recs}")"
+  echo "**${checked} repositories checked**: ${warned} with warnings or notices, ${clean} with none. **${unchecked} not checked.**"
+  echo
+
+  if [[ "${checked}" -eq 0 ]]; then
+    # Zero warnings out of zero repositories read is not "no warnings".
+    echo "**No repository could be checked**, so no warnings were looked for."
+    echo
+  elif [[ "${warned}" -eq 0 ]]; then
+    echo "No warnings: no checked repository's latest run carried a warning or notice annotation."
+    echo
+  else
+    # A failed jq prints nothing; an empty table would read as clean.
+    if ! rows="$(jq -r '
+      def cell: tostring | gsub("[\r\n]+"; " ") | gsub("\\|"; "\\|");
+      # GitHub puts file-less annotations under path .github.
+      def where: if .path == "" or .path == ".github" then "—"
+                 else "`" + .path + (if .line then ":" + (.line | tostring) else "" end) + "`" end;
+      select(.state == "ok") | . as $r | .annotations[]
+      | "| " + (if $r.runUrl then "[" + $r.repo + "](" + $r.runUrl + ")" else $r.repo end)
+        + " | " + .level + " | " + where + " | " + (.message | cell) + " |"
+    ' <<<"${recs}")" || [[ -z "${rows}" ]]; then
+      echo "**Warnings found but could not be rendered.** ${warned} repositories carry annotations; see the run log."
+      echo
+    else
+      echo "| Repo | Level | File | Message |"
+      echo "| --- | --- | --- | --- |"
+      echo "${rows}"
+      echo
+    fi
+  fi
+
+  if [[ "${unchecked}" -gt 0 ]]; then
+    echo "### Not checked"
+    echo
+    echo "These repositories have no readable standards-check result. Their warnings, if any, are unknown — not absent."
+    echo
+    if ! rows="$(jq -r '
+      def cell: tostring | gsub("[\r\n]+"; " ") | gsub("\\|"; "\\|");
+      def state_name: {"unreadable": "UNREADABLE", "unlisted": "OWNER NOT LISTED",
+                       "no-run": "no standards-check run", "no-pr": "no run, no merged PR",
+                       "in-progress": "run in progress"}[.state] // .state;
+      select(.state != "ok" and .state != "archived")
+      | "| " + (.repo // ("all of " + .owner)) + " | " + state_name + ": " + (.detail | cell) + " |"
+    ' <<<"${recs}")" || [[ -z "${rows}" ]]; then
+      echo "**${unchecked} repositories were not checked, and the list could not be rendered.** See the run log."
+      echo
+    else
+      echo "| Repo | Why |"
+      echo "| --- | --- |"
+      echo "${rows}"
+      echo
+    fi
+  fi
+
+  archived="$(jq -rs '[.[] | select(.state == "archived") | .repo] | join(", ")' <<<"${recs}")"
+  if [[ -n "${archived}" ]]; then
+    echo "Archived, not surveyed: ${archived}."
+    echo
+  fi
+}
+
+if [[ -n "${standards_file}" ]]; then
+  render_standards "${standards_file}"
+fi
+
 echo "---"
 echo
 generated_at="$(date -u '+%Y-%m-%d %H:%M UTC')"

scripts/dependabot-digest/run-digest.sh

@@ -45,6 +45,8 @@ work="$(mktemp -d)"
 trap 'rm -rf "${work}"' EXIT
 collected="${work}/collected.ndjson"
 : >"${collected}"
+standards="${work}/standards.ndjson"
+: >"${standards}"
 
 issue_token="${GH_TOKEN-}"
 
@@ -67,6 +69,14 @@ for owner in ${OWNERS}; do
     echo "run-digest.sh: collection failed for ${owner}; not publishing a partial digest" >&2
     exit 1
   fi
+
+  # Standards never block the queue report. A crash renders as not checked.
+  echo "run-digest.sh: reading standards-check annotations for ${owner}" >&2
+  if ! GH_TOKEN="${token}" bash "${HERE}/standards.sh" "${owner}" >>"${standards}"; then
+    jq -cn --arg o "${owner}" '{owner: $o, repo: null, state: "unlisted",
+      detail: "standards.sh failed; see the run log", source: null,
+      runUrl: null, annotations: []}' >>"${standards}"
+  fi
 done
 
 classified="${work}/classified.ndjson"
@@ -76,8 +86,8 @@ if ! bash "${HERE}/classify.sh" <"${collected}" >"${classified}"; then
 fi
 
 body="${work}/body.md"
-render_args=()
-[[ -n "${run_url}" ]] && render_args=(--run-url "${run_url}")
+render_args=(--standards "${standards}")
+[[ -n "${run_url}" ]] && render_args+=(--run-url "${run_url}")
 if ! bash "${HERE}/render.sh" "${render_args[@]}" <"${classified}" >"${body}"; then
   echo "run-digest.sh: rendering failed" >&2
   exit 1

scripts/dependabot-digest/standards.sh

@@ -0,0 +1,184 @@
+#!/usr/bin/env bash
+# Usage: standards.sh <owner>. Prints one JSON record per repo: standards-check
+# annotations, or the reason none were read.
+set -uo pipefail
+unset CDPATH
+
+owner="${1-}"
+if [[ -z "${owner}" ]]; then
+  echo "standards.sh: usage: standards.sh <owner>" >&2
+  exit 2
+fi
+
+# Match the reusable job name only: callers may name their job differently.
+read -r -d '' pick_run <<'JQ'
+[.check_runs[]
+ | select(.app.slug == "github-actions")
+ | select(.name == "run-standards-check"
+          or (.name | endswith(" / run-standards-check")))]
+| sort_by(.id) | last
+JQ
+
+# Nulled entries or a short page mean the list cannot be trusted.
+read -r -d '' runs_sane <<'JQ'
+(.check_runs | type == "array")
+and (.total_count | type == "number")
+and (.total_count <= (.check_runs | length))
+and all(.check_runs[]; (.id | type == "number") and (.name | type == "string"))
+JQ
+
+emit() {
+  # emit <repo> <state> <detail> [source] [run_url] [annotations_json]
+  local line
+  line="$(jq -cn --arg owner "${owner}" --arg repo "$1" --arg state "$2" \
+    --arg detail "$3" --arg source "${4-}" --arg url "${5-}" \
+    --argjson ann "${6:-[]}" \
+    '{owner: $owner, repo: (if $repo == "" then null else $repo end),
+      state: $state, detail: $detail,
+      source: (if $source == "" then null else $source end),
+      runUrl: (if $url == "" then null else $url end),
+      annotations: $ann}' 2>/dev/null)"
+  # If jq fails, the repo must still appear.
+  if [[ -z "${line}" ]]; then
+    printf '{"owner":"%s","repo":"%s","state":"unreadable","detail":"could not encode the result","source":null,"runUrl":null,"annotations":[]}\n' \
+      "${owner}" "$1"
+    return 0
+  fi
+  printf '%s\n' "${line}"
+}
+
+# Callers run api in a subshell, so the failure reason goes to a file.
+ERRF="$(mktemp)"
+trap 'rm -f "${ERRF}"' EXIT
+api() {
+  : >"${ERRF}"
+  gh api "$@" 2>"${ERRF}"
+}
+# emit_err <repo> <state> <what failed> <fallback reason> [source] [run_url]
+emit_err() {
+  local why
+  why="$(last_err "$4")"
+  emit "$1" "$2" "$3: ${why}" "${5-}" "${6-}"
+}
+set_err() { printf '%s' "$1" >"${ERRF}"; }
+last_err() {
+  local e
+  e="$(tr '\n' ' ' <"${ERRF}" | cut -c1-160)"
+  printf '%s' "${e:-$1}"
+}
+
+# Prints the commit's standards-check run, or nothing. Returns 1 if unreadable.
+run_on_commit() {
+  local nwo="$1" sha="$2" body
+  if ! body="$(api "repos/${nwo}/commits/${sha}/check-runs?per_page=100")"; then
+    return 1
+  fi
+  if ! jq -e "${runs_sane}" >/dev/null 2>&1 <<<"${body}"; then
+    set_err "check-run list for ${sha:0:7} is incomplete or nulled"
+    return 1
+  fi
+  jq -c "${pick_run} // empty" <<<"${body}"
+}
+
+survey_repo() {
+  local nwo="$1" branch="$2" head pulls pr pr_head run source ann count url
+  if ! head="$(api "repos/${nwo}/commits/${branch}" --jq '.sha')" \
+    || [[ ! "${head}" =~ ^[0-9a-f]{40}$ ]]; then
+    emit_err "${nwo}" unreadable "cannot read the head of ${branch}" "no sha returned"
+    return 0
+  fi
+
+  if ! run="$(run_on_commit "${nwo}" "${head}")"; then
+    emit_err "${nwo}" unreadable "cannot read check runs on ${branch} head ${head:0:7}" "no detail"
+    return 0
+  fi
+  source="${branch} head ${head:0:7}"
+
+  if [[ -z "${run}" ]]; then
+    if ! pulls="$(api "repos/${nwo}/commits/${head}/pulls")" \
+      || ! jq -e 'type == "array"' >/dev/null 2>&1 <<<"${pulls}"; then
+      emit_err "${nwo}" unreadable "cannot read the pull request for ${branch} head ${head:0:7}" "non-array body"
+      return 0
+    fi
+    # Only the PR whose merge produced this commit vetted it.
+    pr="$(jq -c --arg sha "${head}" '[.[] | select(.merge_commit_sha == $sha and .merged_at != null)] | first // empty' <<<"${pulls}" 2>/dev/null)"
+    if [[ -z "${pr}" ]]; then
+      emit "${nwo}" no-pr "${branch} head ${head:0:7} has no standards-check run and was not produced by a merged pull request" "${source}"
+      return 0
+    fi
+    pr_head="$(jq -r '.head.sha // empty' <<<"${pr}")"
+    source="#$(jq -r '.number' <<<"${pr}"), merged as ${branch} head ${head:0:7}"
+    if [[ ! "${pr_head}" =~ ^[0-9a-f]{40}$ ]]; then
+      emit "${nwo}" unreadable "pull request for ${head:0:7} has no readable head commit" "${source}"
+      return 0
+    fi
+    if ! run="$(run_on_commit "${nwo}" "${pr_head}")"; then
+      emit_err "${nwo}" unreadable "cannot read check runs on PR head ${pr_head:0:7}" "no detail" "${source}"
+      return 0
+    fi
+    if [[ -z "${run}" ]]; then
+      emit "${nwo}" no-run "no standards-check run on ${branch} head ${head:0:7} or on the head of the PR that produced it" "${source}"
+      return 0
+    fi
+  fi
+
+  url="$(jq -r '.html_url // ""' <<<"${run}")"
+  local status run_id
+  status="$(jq -r '.status' <<<"${run}")"
+  run_id="$(jq -r '.id' <<<"${run}")"
+  if [[ "${status}" != "completed" ]]; then
+    emit "${nwo}" in-progress "latest run has not completed, so its annotations are partial" "${source}" "${url}"
+    return 0
+  fi
+
+  count="$(jq -r '.output.annotations_count // empty' <<<"${run}")"
+  if [[ ! "${count}" =~ ^[0-9]+$ ]]; then
+    emit "${nwo}" unreadable "run reports no annotation count" "${source}" "${url}"
+    return 0
+  fi
+  if ! ann="$(api --paginate --slurp "repos/${nwo}/check-runs/${run_id}/annotations?per_page=100")"; then
+    emit_err "${nwo}" unreadable "cannot read annotations" "no detail" "${source}" "${url}"
+    return 0
+  fi
+  # Fewer annotations than the run reports means some were withheld.
+  ann="$(jq -c --argjson want "${count}" '
+    (add // []) as $all
+    | if all($all[]; (.annotation_level | type == "string") and (.message | type == "string"))
+         and ($all | length) >= $want
+      then [$all[] | select(.annotation_level == "warning" or .annotation_level == "notice")
+            | {level: .annotation_level, path: (.path // ""),
+               line: (.start_line // null), message: .message}]
+      else "short" end' <<<"${ann}" 2>/dev/null)"
+  if [[ -z "${ann}" ]] || ! jq -e 'type == "array"' >/dev/null 2>&1 <<<"${ann}"; then
+    emit "${nwo}" unreadable "annotations are incomplete or malformed (run reports ${count})" "${source}" "${url}"
+    return 0
+  fi
+  emit "${nwo}" ok "" "${source}" "${url}" "${ann}"
+}
+
+# The installation list is exactly the set of repos this token can read.
+if ! repos="$(api --paginate --slurp "installation/repositories?per_page=100")" \
+  || ! repo_list="$(jq -r '
+      if type == "array" and all(.[]; (.repositories | type == "array"))
+         and ([.[].repositories[]] | length) >= (.[0].total_count // 0)
+      then [.[].repositories[]] | sort_by(.full_name)[]
+           | [.full_name, (.default_branch // ""), (.archived | tostring)] | @tsv
+      else error("bad shape") end' <<<"${repos}" 2>/dev/null)"; then
+  emit_err "" unlisted "cannot list the repositories this token can read" "unexpected response shape"
+  exit 0
+fi
+if [[ -z "${repo_list}" ]]; then
+  emit "" unlisted "the token can read no repositories, so nothing was checked"
+  exit 0
+fi
+
+while IFS=$'\t' read -r nwo branch archived; do
+  [[ -z "${nwo}" ]] && continue
+  if [[ "${archived}" == "true" ]]; then
+    emit "${nwo}" archived "archived; not surveyed"
+  elif [[ -z "${branch}" ]]; then
+    emit "${nwo}" unreadable "no default branch reported"
+  else
+    survey_repo "${nwo}" "${branch}"
+  fi
+done <<<"${repo_list}"

scripts/dependabot-digest/tests/fixtures/standards/installation_repositories.json.o

@@ -0,0 +1,52 @@
+[
+ {
+  "total_count": 8,
+  "repositories": [
+   {
+    "full_name": "o/warned",
+    "default_branch": "main",
+    "archived": false
+   },
+   {
+    "full_name": "o/clean",
+    "default_branch": "main",
+    "archived": false
+   },
+   {
+    "full_name": "o/norun",
+    "default_branch": "main",
+    "archived": false
+   },
+   {
+    "full_name": "o/nulled",
+    "default_branch": "main",
+    "archived": false
+   }
+  ]
+ },
+ {
+  "total_count": 8,
+  "repositories": [
+   {
+    "full_name": "o/short",
+    "default_branch": "main",
+    "archived": false
+   },
+   {
+    "full_name": "o/direct",
+    "default_branch": "main",
+    "archived": false
+   },
+   {
+    "full_name": "o/old",
+    "default_branch": "main",
+    "archived": true
+   },
+   {
+    "full_name": "o/forbidden",
+    "default_branch": "main",
+    "archived": false
+   }
+  ]
+ }
+]