fix(gateway): make passive connection failures diagnosable

.github/workflows/ci.yml

@@ -31,6 +31,7 @@ jobs:
           node tests/test_parse_protected_gateway_index.cjs
           node tests/test_gateway_workflow_resolve.cjs
           python3 -m unittest discover -s tests -p 'test_match_gateway_metadata.py'
+          python3 -m unittest discover -s tests -p 'test_gateway_ssh_failure_diagnostics.py'
 
   docker-build:
     runs-on: ubuntu-latest

.github/workflows/read-only-vm-metadata-diagnostic.yml

@@ -364,16 +364,18 @@ jobs:
           GCP_PROJECT_ID: ${{ steps.metadata.outputs.gcp_secret_project_id }}
         run: |
           set -euo pipefail
-          key_file="$(mktemp "${RUNNER_TEMP}/gateway-ssh-key.XXXXXX")"
-          error_file="$(mktemp "${RUNNER_TEMP}/gateway-secret-error.XXXXXX")"
+          key_file="$(mktemp "${RUNNER_TEMP}/gateway-ssh-key.XXXXXX" 2>/dev/null)"
+          error_file="$(mktemp "${RUNNER_TEMP}/gateway-secret-error.XXXXXX" 2>/dev/null)"
           key_handed_off=false
           cleanup_secret_fetch() {
-            rm -f "${error_file}"
-            if [ "${key_handed_off}" != "true" ]; then rm -f "${key_file}"; fi
+            rm -f "${error_file}" 2>/dev/null || true
+            if [ "${key_handed_off}" != "true" ]; then rm -f "${key_file}" 2>/dev/null || true; fi
           }
           trap cleanup_secret_fetch EXIT
-          echo "SSH_KEY_FILE=${key_file}" >> "${GITHUB_ENV}"
-          chmod 600 "${key_file}"
+          if ! chmod 600 "${key_file}" 2>/dev/null; then
+            echo "GATEWAY_CONNECTION_INSPECTION=blocked reason=credential_unavailable"
+            exit 1
+          fi
           if ! gcloud secrets versions access latest \
             --secret="${SSH_PRIVATE_KEY_SECRET_NAME}" \
             --project="${GCP_PROJECT_ID}" >"${key_file}" 2>"${error_file}"; then
@@ -384,7 +386,11 @@ jobs:
             echo "GATEWAY_CONNECTION_INSPECTION=blocked reason=credential_unavailable"
             exit 1
           fi
-          chmod 600 "${key_file}"
+          if ! SSH_KEY_FILE="${key_file}" bash scripts/prepare_gateway_ssh_key.sh >/dev/null 2>"${error_file}"; then
+            echo "GATEWAY_CONNECTION_INSPECTION=blocked reason=credential_invalid"
+            exit 1
+          fi
+          echo "SSH_KEY_FILE=${key_file}" >> "${GITHUB_ENV}"
           key_handed_off=true
 
       - name: Inspect Gateway connections through one IAP SSH session
@@ -394,11 +400,14 @@ jobs:
           TARGET_INDEX: ${{ matrix.target_index }}
         run: |
           set -euo pipefail
-          known_hosts="$(mktemp "${RUNNER_TEMP}/gateway-known-hosts.XXXXXX")"
-          output_file="$(mktemp "${RUNNER_TEMP}/gateway-inspection-output.XXXXXX")"
-          error_file="$(mktemp "${RUNNER_TEMP}/gateway-ssh-error.XXXXXX")"
-          trap 'rm -f "${known_hosts}" "${output_file}" "${error_file}" "${SSH_KEY_FILE}"' EXIT
-          chmod 600 "${known_hosts}" "${output_file}" "${error_file}"
+          known_hosts="$(mktemp "${RUNNER_TEMP}/gateway-known-hosts.XXXXXX" 2>/dev/null)"
+          output_file="$(mktemp "${RUNNER_TEMP}/gateway-inspection-output.XXXXXX" 2>/dev/null)"
+          error_file="$(mktemp "${RUNNER_TEMP}/gateway-ssh-error.XXXXXX" 2>/dev/null)"
+          trap 'rm -f "${known_hosts}" "${output_file}" "${error_file}" "${SSH_KEY_FILE}" 2>/dev/null || true' EXIT
+          if ! chmod 600 "${known_hosts}" "${output_file}" "${error_file}" 2>/dev/null; then
+            echo "GATEWAY_CONNECTION_INSPECTION=blocked reason=inspection_unavailable"
+            exit 1
+          fi
           target_alias="gateway-target-${TARGET_INDEX}"
           # The temporary SSH host-key record is bound to this exact IAP target alias.
           proxy_command="gcloud compute start-iap-tunnel '${GCE_INSTANCE_NAME}' 22 --listen-on-stdin --project='${GCP_PROJECT_ID}' --zone='${GCE_ZONE}' --quiet"
@@ -419,45 +428,19 @@ jobs:
           else
             ssh_status=$?
           fi
-          python3 - "${output_file}" "${ssh_status}" <<'PY'
-          import re
-          import sys
-          from pathlib import Path
-
-          lines = Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace").splitlines()
-          ssh_status = int(sys.argv[2])
-          if len(lines) == 1 and re.fullmatch(r"GATEWAY_CONNECTION_INSPECTION=blocked reason=[a-z_]+", lines[0]):
-              print(lines[0])
-              raise SystemExit(1)
-          patterns = {
-              "GATEWAY_CONNECTION_INSPECTION": r"GATEWAY_CONNECTION_INSPECTION=observed",
-              "GATEWAY_CONTAINER_RUNNING": r"GATEWAY_CONTAINER_RUNNING=(true|false)",
-              "GATEWAY_API_LISTENER": r"GATEWAY_API_LISTENER=(true|false)",
-              "GATEWAY_ESTABLISHED_CONNECTION_COUNT": r"GATEWAY_ESTABLISHED_CONNECTION_COUNT=[0-9]+",
-              "GATEWAY_CONNECTION_OBSERVED_AT_UTC": r"GATEWAY_CONNECTION_OBSERVED_AT_UTC=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z",
-              "GATEWAY_CONNECTION_INSPECTION_LIMIT": r"GATEWAY_CONNECTION_INSPECTION_LIMIT=OBSERVATION_ONLY_NO_AUTH_OR_CONCURRENCY_ASSERTION",
-          }
-          names = [line.split("=", 1)[0] for line in lines]
-          if ssh_status != 0:
-              print("GATEWAY_CONNECTION_INSPECTION=blocked reason=ssh_unavailable")
-              raise SystemExit(1)
-          if len(lines) != len(patterns) or set(names) != set(patterns) or len(set(names)) != len(names) or any(
-              not re.fullmatch(patterns.get(name, ""), line) for name, line in zip(names, lines)
-          ):
-              print("GATEWAY_CONNECTION_INSPECTION=blocked reason=remote_response_invalid")
-              raise SystemExit(1)
-          for line in lines:
-              print(line)
-          PY
+          python3 scripts/classify_gateway_ssh_failure.py \
+            --stdout-file "${output_file}" \
+            --stderr-file "${error_file}" \
+            --exit-code "${ssh_status}"
 
       - name: Remove temporary SSH key
         if: ${{ always() && inputs.inspect_connections }}
         env:
           SSH_KEY_FILE: ${{ env.SSH_KEY_FILE }}
         run: |
           set -euo pipefail
-          if [ -n "${SSH_KEY_FILE:-}" ]; then rm -f -- "${SSH_KEY_FILE}"; fi
+          if [ -n "${SSH_KEY_FILE:-}" ]; then rm -f -- "${SSH_KEY_FILE}" 2>/dev/null || true; fi
           for key_file in "${RUNNER_TEMP}"/gateway-ssh-key.*; do
             [ -e "${key_file}" ] || continue
-            rm -f -- "${key_file}"
+            rm -f -- "${key_file}" 2>/dev/null || true
           done

scripts/classify_gateway_ssh_failure.py

@@ -0,0 +1,126 @@
+"""Classify captured SSH/IAP errors into fixed, non-sensitive categories."""
+
+from __future__ import annotations
+
+import argparse
+import re
+from pathlib import Path
+
+
+_PATTERNS = {
+    "timeout": re.compile(r"(?:connection|operation|command|connect) timed out|\btimeout\b", re.I),
+    "key_invalid": re.compile(
+        r"invalid format|error in libcrypto|incorrect passphrase|bad passphrase|load key .*: invalid",
+        re.I,
+    ),
+    "auth_denied": re.compile(
+        r"permission denied \(publickey[^)]*\)|no supported authentication methods|authentication failed|too many authentication failures",
+        re.I,
+    ),
+    "iap_denied": re.compile(
+        r"(?:start-iap-tunnel|iap tunnel|iap-tunnel).{0,160}(?:4033|not authorized|permission denied|forbidden)|"
+        r"(?:4033|not authorized|permission denied|forbidden).{0,160}(?:start-iap-tunnel|iap tunnel|iap-tunnel)",
+        re.I | re.S,
+    ),
+    "iap_transport": re.compile(
+        r"(?:start-iap-tunnel|iap tunnel|iap-tunnel).{0,160}(?:failed to connect|connection refused|backend|unreachable|closed)|"
+        r"(?:failed to connect|connection refused|backend|unreachable|closed).{0,160}(?:start-iap-tunnel|iap tunnel|iap-tunnel)",
+        re.I | re.S,
+    ),
+}
+
+
+def classify_gateway_ssh_failure(message: str) -> str:
+    """Return a category only when the captured text identifies exactly one."""
+
+    matches = [name for name, pattern in _PATTERNS.items() if pattern.search(message)]
+    return matches[0] if len(matches) == 1 else "unknown"
+
+
+_RESPONSE_PATTERNS = {
+    "GATEWAY_CONNECTION_INSPECTION": r"GATEWAY_CONNECTION_INSPECTION=observed",
+    "GATEWAY_CONTAINER_RUNNING": r"GATEWAY_CONTAINER_RUNNING=(true|false)",
+    "GATEWAY_API_LISTENER": r"GATEWAY_API_LISTENER=(true|false)",
+    "GATEWAY_ESTABLISHED_CONNECTION_COUNT": r"GATEWAY_ESTABLISHED_CONNECTION_COUNT=[0-9]+",
+    "GATEWAY_CONNECTION_OBSERVED_AT_UTC": r"GATEWAY_CONNECTION_OBSERVED_AT_UTC=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z",
+    "GATEWAY_CONNECTION_INSPECTION_LIMIT": (
+        "GATEWAY_CONNECTION_INSPECTION_LIMIT="
+        "OBSERVATION_ONLY_NO_AUTH_OR_CONCURRENCY_ASSERTION"
+    ),
+}
+_HELPER_BLOCKED_REASONS = {
+    "configuration_invalid",
+    "container_inspection_failed",
+    "container_state_invalid",
+    "container_pid_invalid",
+    "socket_inspection_failed",
+    "socket_result_invalid",
+}
+
+
+def validate_gateway_ssh_response(lines: list[str]) -> str | None:
+    """Return safe helper output or a fixed local response error."""
+
+    if len(lines) == 1 and any(
+        lines[0] == f"GATEWAY_CONNECTION_INSPECTION=blocked reason={reason}"
+        for reason in _HELPER_BLOCKED_REASONS
+    ):
+        return lines[0]
+    names = [line.split("=", 1)[0] for line in lines]
+    if (
+        len(lines) != len(_RESPONSE_PATTERNS)
+        or set(names) != set(_RESPONSE_PATTERNS)
+        or len(set(names)) != len(names)
+        or any(
+            not re.fullmatch(_RESPONSE_PATTERNS.get(name, ""), line)
+            for name, line in zip(names, lines)
+        )
+    ):
+        return None
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--stdout-file", required=True, type=Path)
+    parser.add_argument("--stderr-file", required=True, type=Path)
+    parser.add_argument("--exit-code", required=True, type=int)
+    args = parser.parse_args(argv)
+    if not 0 <= args.exit_code <= 255:
+        print("GATEWAY_CONNECTION_INSPECTION=blocked reason=ssh_unknown exit_code=1")
+        return 1
+    if args.exit_code == 0:
+        try:
+            lines = args.stdout_file.read_text(encoding="utf-8", errors="replace").splitlines()
+        except OSError:
+            lines = []
+        response = validate_gateway_ssh_response(lines)
+        if response is None:
+            print("GATEWAY_CONNECTION_INSPECTION=blocked reason=remote_response_invalid")
+            return 1
+        print(response)
+        return 1 if response.startswith("GATEWAY_CONNECTION_INSPECTION=blocked reason=") else 0
+    try:
+        lines = args.stdout_file.read_text(encoding="utf-8", errors="replace").splitlines()
+    except OSError:
+        lines = []
+    helper_response = validate_gateway_ssh_response(lines)
+    if helper_response is not None and helper_response.startswith(
+        "GATEWAY_CONNECTION_INSPECTION=blocked reason="
+    ):
+        print(helper_response)
+        return 1
+    try:
+        message = args.stderr_file.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        message = ""
+    category = "timeout" if args.exit_code == 124 else classify_gateway_ssh_failure(message)
+    print(
+        "GATEWAY_CONNECTION_INSPECTION=blocked "
+        f"reason=ssh_{category} exit_code={args.exit_code}"
+    )
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/prepare_gateway_ssh_key.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+key_file="${SSH_KEY_FILE:-}"
+if [[ -z "${key_file}" || ! -f "${key_file}" ]]; then
+  echo "GATEWAY_CONNECTION_INSPECTION=blocked reason=credential_invalid"
+  exit 1
+fi
+
+normalized_file=""
+cleanup() {
+  if [[ -n "${normalized_file}" ]]; then rm -f -- "${normalized_file}" 2>/dev/null || true; fi
+}
+trap cleanup EXIT
+
+normalized_file="$(mktemp "${RUNNER_TEMP:-/tmp}/gateway-ssh-key-normalized.XXXXXX" 2>/dev/null)" || {
+  rm -f -- "${key_file}" 2>/dev/null || true
+  echo "GATEWAY_CONNECTION_INSPECTION=blocked reason=credential_invalid"
+  exit 1
+}
+chmod 600 "${normalized_file}" 2>/dev/null || {
+  rm -f -- "${key_file}" 2>/dev/null || true
+  echo "GATEWAY_CONNECTION_INSPECTION=blocked reason=credential_invalid"
+  exit 1
+}
+if ! tr -d '\r' <"${key_file}" >"${normalized_file}" 2>/dev/null || [[ ! -s "${normalized_file}" ]]; then
+  rm -f -- "${key_file}" 2>/dev/null || true
+  echo "GATEWAY_CONNECTION_INSPECTION=blocked reason=credential_invalid"
+  exit 1
+fi
+if ! ssh-keygen -y -P '' -f "${normalized_file}" >/dev/null 2>/dev/null; then
+  rm -f -- "${key_file}" 2>/dev/null || true
+  echo "GATEWAY_CONNECTION_INSPECTION=blocked reason=credential_invalid"
+  exit 1
+fi
+if ! cat "${normalized_file}" >"${key_file}" 2>/dev/null || ! chmod 600 "${key_file}" 2>/dev/null; then
+  rm -f -- "${key_file}"
+  echo "GATEWAY_CONNECTION_INSPECTION=blocked reason=credential_invalid"
+  exit 1
+fi
+
+echo "GATEWAY_SSH_KEY_STATUS=ready"