From d19afe8e5cfd1c75535289cc129ee61be4f6dee5 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Thu, 19 Nov 2026 13:54:34 +0100 Subject: [PATCH] docs(self-hosting): close GKE guide gaps from infra #495 - Step 4: leave desktop_feed_bucket_name out during the install; the bucket without a signing key makes relay-control exit. Recovery in Troubleshooting. - Step 7: the services, psql and chart pins belong to agent-relay-2026.11.16; how to read them for another release. - Step 14: the Ingress takes 15 to 20 minutes; a one-off BackendConfig translation warning is harmless; 502 on closed paths is intended. - Step 15: first-admin.sh gains optional API_HOST and PASSWORD_FILE (copied from infra at the #495 head, keeping the website's own line 2). - Regenerate SHA256SUMS and the downloads table. Co-Authored-By: Claude --- web/content/self-hosting/downloads.md | 2 +- web/content/self-hosting/install-gke.md | 81 +++++++++++++++++-- web/content/self-hosting/troubleshooting.md | 35 ++++++++ web/public/self-hosting/files/gke/SHA256SUMS | 2 +- .../self-hosting/files/gke/first-admin.sh | 56 ++++++++++--- 5 files changed, 155 insertions(+), 21 deletions(-) diff --git a/web/content/self-hosting/downloads.md b/web/content/self-hosting/downloads.md index 35165655..3d08e46a 100644 --- a/web/content/self-hosting/downloads.md +++ b/web/content/self-hosting/downloads.md @@ -40,7 +40,7 @@ Used by [this guide](/docs/self-hosting/install-gke). Checksums: [`SHA256SUMS`]( | [`add-user.sh`](https://agentrelay.com/self-hosting/files/gke/add-user.sh) | `64656f743f1c04b69044f5403bb70914d824179edfb3de596f84e559e7105303` | | [`cloud-sql-setup.sh`](https://agentrelay.com/self-hosting/files/gke/cloud-sql-setup.sh) | `044f4227a119438cf4bf42b5eb32a2bcac112dcc223dbd4a5bb99df290639a0b` | | [`create-workload-identity.sh`](https://agentrelay.com/self-hosting/files/gke/create-workload-identity.sh) | `b474bf2ddd9e01ca67c905a1d9652ccaf80b74eb47a56ddffb46071b2fafff7e` | -| [`first-admin.sh`](https://agentrelay.com/self-hosting/files/gke/first-admin.sh) | `35a1d8eab7855d451c5dae2eb3072afd6411dea4ccde8f188936e08cbe38386f` | +| [`first-admin.sh`](https://agentrelay.com/self-hosting/files/gke/first-admin.sh) | `7991981ce1bab75fd4385f851f7a1e67bdc9733a51ad3ab9951ca3fbd7c3e2a0` | | [`helm-secrets-prepare.sh`](https://agentrelay.com/self-hosting/files/gke/helm-secrets-prepare.sh) | `2ae3074e1577fd32f3b10c2d104a239b144454068a37fb0cecdb62e31de3d124` | | [`managed-values-byo.sh`](https://agentrelay.com/self-hosting/files/gke/managed-values-byo.sh) | `9fa8e76b98949612855498795c36a77557717a905b18d7cfe34dca78aeac56ea` | | [`okta-setup.sh`](https://agentrelay.com/self-hosting/files/gke/okta-setup.sh) | `d2208cdf9011cd06fb103250da1fc9129b69dc6f4119456da531593c2728f1d7` | diff --git a/web/content/self-hosting/install-gke.md b/web/content/self-hosting/install-gke.md index ae6b7473..d5a30e0f 100644 --- a/web/content/self-hosting/install-gke.md +++ b/web/content/self-hosting/install-gke.md @@ -178,7 +178,7 @@ network_self_link = "projects//global/networks/" sessions_bucket_name = "" deletion_protection = true image_mirror_repository = "agent-relay-mirror" # step 5's registry, readable by the GKE nodes -# desktop_feed_bucket_name = "" # optional: the desktop app's update feed +# desktop_feed_bucket_name = "" # optional: leave it out for now (see below) # gke_node_service_account = "" # only if the node pool has its own service account # kms_key_name = "projects/

/locations//keyRings//cryptoKeys/" # optional CMEK labels = { app = "agent-relay" } @@ -200,12 +200,19 @@ exists. signed update feed in your project: a private bucket, and a Google service account that relay-control uses through Workload Identity to write releases there and sign download links. Creating that service account needs Service -Account Admin. `managed-values.json` carries the feed's settings into step 11. -If your `values.yaml` sets `relay-control.extraEnv`, copy the `env` entries -from `terraform output desktop_feed` into it: Helm replaces a list instead of -merging it. The feed also needs a signing key: see +Account Admin. The feed also needs a signing key, so **leave this line +commented out during the install** and add it later with the key, in [Signed desktop update feed](#signed-desktop-update-feed-optional). +The bucket puts the feed's storage settings into `managed-values.json`, and +relay-control won't start with storage but no signing key. It exits with +`SELFHOST_DESKTOP_FEED_SIGNER and SELFHOST_DESKTOP_FEED_STORAGE must be set together`, +and the pod restarts again and again. Already hit it? See +[Troubleshooting](/docs/self-hosting/troubleshooting#relay-control-exits-signer-and-storage-must-be-set-together). +If your `values.yaml` sets `relay-control.extraEnv`, copy the `env` entries +from `terraform output desktop_feed` into it once the feed is on: Helm +replaces a list instead of merging it. + **Where the Terraform state lives.** The example has no `backend` block, so `terraform init` keeps the state in a local file. To keep it in Cloud Storage instead, create a private, versioned bucket and add a backend before @@ -332,6 +339,25 @@ source settings.sh bash -c 'source ./settings.sh; for v in DOMAIN ORG_ID ORG_NAME ADMIN_EMAIL GCP_PROJECT_ID REGION MIRROR_ROOT CLOUD_SQL_INSTANCE CLOUD_SQL_HOST CLOUD_SQL_ADMIN_USER CLOUD_SQL_ADMIN_SECRET_ID CLOUD_SQL_CA_SHA256 SESSIONS_BUCKET; do [ -n "${!v}" ] && [ "${!v}" != null ] && [[ "${!v}" != *"<"* ]] || echo "MISSING: $v"; done; echo "settings checked"' ``` +**`SERVICES_IMAGE`, `PSQL_IMAGE` and `agent-relay-0.7.54.tgz` belong to +`agent-relay-2026.11.16`** (chart 0.7.54), the release this guide installs. +Installing another release? Don't reuse these digests: the Jobs in steps 8, 13 +and 16 would run an older services image than the rest of the install. Read +the three values from that release's own files in `release/` (step 1, with its +tag): + +```sh +ls release/agent-relay-*.tgz +jq -r '.images[] | select(.name == "sessions-pipeline" or .name == "sessions-migration-postgresql") | "\(.name) \(.ref)"' release/release-manifest.json +``` + +`SERVICES_IMAGE` is `$MIRROR_ROOT/` followed by the `sessions-pipeline` ref, +and `PSQL_IMAGE` is `$MIRROR_ROOT/` followed by the +`sessions-migration-postgresql` ref. The `.tgz` name replaces +`agent-relay-0.7.54.tgz` in steps 11 and 13. The infra checkout also records +the services image for each release, in +`infra/releases//pins.json` (`.pins["sessions-pipeline"]`). + **Brought your own resources (step 3)?** Replace the six lines between `PG=` and `unset` with the values from [Bring your own Google Cloud resources](/docs/self-hosting/bring-your-own-gcp#then-the-settings-file). @@ -512,7 +538,8 @@ with `echo $CLOUD_SQL_CA_SHA256`. kubectl -n $NAMESPACE get managedcertificate agent-relay -o jsonpath='{.status.certificateStatus}{"\n"}' ``` -Repeat until it prints `Active`. Then: +Repeat until it prints `Active` (what is normal while you wait is below). +Then: ```sh curl -fsS "https://api.$DOMAIN/readyz" && echo " api ok" @@ -524,12 +551,32 @@ curl -s -o /dev/null -w "history %{http_code}\n" "https://history.$DOMAIN/v1/ses **Done when:** `Active`, then `api ok`, `cast ok`, `dashboard 200` and `history 401`. +**The Ingress can take 15 to 20 minutes to get its address** after step 11. +The certificate can't leave `Provisioning` before that. To look: + +```sh +kubectl -n $NAMESPACE get ingress +kubectl -n $NAMESPACE describe ingress | sed -n '/^Events:/,$p' +``` + +A `Translation failed ... no BackendConfig for service port exists` warning +on the first sync is harmless if it doesn't repeat. The load balancer +controller read the Services before their BackendConfigs, and a later sync +fixes it by itself. Worry only if the `ADDRESS` column is still empty, or the +warning still repeats, after about 20 minutes; then send us the events. + - **Still `Provisioning` after an hour:** one of the four names doesn't resolve to the address from step 6, or a CAA record blocks Google. Fix DNS; the certificate retries by itself. - **`FailedNotVisible`:** the same: DNS doesn't point at the load balancer yet. - **502 for a few minutes after `Active`:** the load balancer's health checks are still converging. Wait 5 minutes. +- **502 on `/internal` and other closed paths is not an outage.** The chart + sends paths that must not be public (for example + `https://api./internal/...`) to the `...-cast-edge-deny` Service, + which has no endpoints on purpose. Without a Cloud Armor policy the load + balancer answers those paths with 502, not 403 or 404. Only the four checks + above tell you whether the install is up. ## 16. Turn on agent search (once, after first sign-in) @@ -749,6 +812,10 @@ token it publishes with. The token itself stays with Agent Relay. (cd infra/terraform/examples/gcp && terraform plan -out tfplan && terraform apply tfplan \ && terraform output -json chart_values > ../../../../managed-values.json) ``` + + Don't run step 11's or step 13's `helm upgrade` again until item 3 below: + until the signing key is in place, relay-control won't start with these + settings. 2. **Signing key:** create an Ed25519 key for the profile and store it in the cluster. The private key never leaves your cluster and Secret Manager: diff --git a/web/content/self-hosting/troubleshooting.md b/web/content/self-hosting/troubleshooting.md index ba553f90..45ea1ad1 100644 --- a/web/content/self-hosting/troubleshooting.md +++ b/web/content/self-hosting/troubleshooting.md @@ -36,6 +36,17 @@ HUMAN before running anything that changes the cluster or cloud account. with `permission denied for schema sessions`. It recovers by itself. - **`helm upgrade --install` prints `STATUS: deployed` with a failed migration Job.** Helm doesn't wait for Jobs here: always run the migration check. +- **On GKE, the Ingress has no address for 15 to 20 minutes after step 11,** + and its events may show one `Translation failed ... no BackendConfig for + service port exists` warning. The load balancer controller read the Services + before their BackendConfigs; a later sync fixes it by itself. Worry only if + the address is still empty, or the warning still repeats, after about 20 + minutes (see [step 14 of the GKE guide](/docs/self-hosting/install-gke#14-wait-for-the-certificate-and-check-up-to-60-minutes)). +- **On GKE, `/internal` and other closed paths answer 502.** The chart sends + them to the `...-cast-edge-deny` Service, which has no endpoints on purpose. + Without a Cloud Armor policy the load balancer answers 502 there, not 403 or + 404. This is not an outage: the four checks in step 14 tell you whether the + install is up. ## Common problems @@ -120,6 +131,30 @@ kubectl -n agent-relay get managedcertificate -o wide Don't roll back mid-provisioning: that starts another replacement. +### relay-control exits: signer and storage must be set together + +relay-control restarts again and again, and its log ends with +`SELFHOST_DESKTOP_FEED_SIGNER and SELFHOST_DESKTOP_FEED_STORAGE must be set together`. +`desktop_feed_bucket_name` is set in `terraform.tfvars`, so +`managed-values.json` carries the desktop feed's storage settings, but the +feed's signing key isn't in place yet. + +Keep the bucket, and take the storage settings out until the key exists. From +the install folder: + +```sh +jq 'del(.["relay-control"].extraEnv)' managed-values.json > managed-values.tmp && mv managed-values.tmp managed-values.json +``` + +Also remove any `SELFHOST_DESKTOP_FEED_*` entries you copied into +`values.yaml`. The service account annotations can stay. Then run step 11's +`helm upgrade` command again. The Terraform in `agent-relay-2026.11.16` has +no setting that does this for you, so use the `jq` line. + +When the key is in place, follow +[Signed desktop update feed](/docs/self-hosting/install-gke#signed-desktop-update-feed-optional): +its first item writes `managed-values.json` again with the storage settings. + ### Other GKE issues - **Certificate stuck in `Provisioning` or `FailedNotVisible`:** a hostname diff --git a/web/public/self-hosting/files/gke/SHA256SUMS b/web/public/self-hosting/files/gke/SHA256SUMS index b0f4684e..a355d595 100644 --- a/web/public/self-hosting/files/gke/SHA256SUMS +++ b/web/public/self-hosting/files/gke/SHA256SUMS @@ -1,7 +1,7 @@ 64656f743f1c04b69044f5403bb70914d824179edfb3de596f84e559e7105303 add-user.sh 044f4227a119438cf4bf42b5eb32a2bcac112dcc223dbd4a5bb99df290639a0b cloud-sql-setup.sh b474bf2ddd9e01ca67c905a1d9652ccaf80b74eb47a56ddffb46071b2fafff7e create-workload-identity.sh -35a1d8eab7855d451c5dae2eb3072afd6411dea4ccde8f188936e08cbe38386f first-admin.sh +7991981ce1bab75fd4385f851f7a1e67bdc9733a51ad3ab9951ca3fbd7c3e2a0 first-admin.sh 2ae3074e1577fd32f3b10c2d104a239b144454068a37fb0cecdb62e31de3d124 helm-secrets-prepare.sh 9fa8e76b98949612855498795c36a77557717a905b18d7cfe34dca78aeac56ea managed-values-byo.sh d2208cdf9011cd06fb103250da1fc9129b69dc6f4119456da531593c2728f1d7 okta-setup.sh diff --git a/web/public/self-hosting/files/gke/first-admin.sh b/web/public/self-hosting/files/gke/first-admin.sh index 8d07c1fd..8234a136 100755 --- a/web/public/self-hosting/files/gke/first-admin.sh +++ b/web/public/self-hosting/files/gke/first-admin.sh @@ -6,24 +6,41 @@ # # bash first-admin.sh [namespace] # +# Optional environment: +# API_HOST the API's host name. Default: api.. Set it when +# the API has another name. The Origin header stays the app +# hostname. +# PASSWORD_FILE keep the password in this new local file (mode 0600) instead +# of Secret Manager. gcloud is then not needed and +# is not used (pass -). The file must not exist yet. +# # It generates a password, stores it in Secret Manager as # agent-relay-first-admin-password (before using it, so it is never lost), then # posts it with the setup key from the agent-relay-secrets Secret. On 200 the # admin reads the password with # gcloud secrets versions access latest --secret agent-relay-first-admin-password --project -# and signs in at https:///login. +# and signs in at https:///login. Without PASSWORD_FILE this +# needs secretmanager.googleapis.com enabled and permission to create a secret +# in the project. # # The setup key works once, within 24 hours of the install: this script never # retries. A 403 csrf_rejected is refused before the key is read, so the key # stays unused; any other failure: read its message, fix the cause, run again. -# Needs gcloud, kubectl (on the cluster), curl, jq and openssl. +# Needs kubectl (on the cluster), curl, jq and openssl, and gcloud unless +# PASSWORD_FILE is set. set -euo pipefail -DOMAIN="${1:?usage: bash first-admin.sh [namespace]}" -PROJECT="${2:?usage: bash first-admin.sh [namespace]}" +USAGE='usage: [API_HOST=] [PASSWORD_FILE=] bash first-admin.sh [namespace]' +DOMAIN="${1:?$USAGE}" +PROJECT="${2:?$USAGE}" NAMESPACE="${3:-agent-relay}" +API_HOST="${API_HOST:-api.$DOMAIN}" +PASSWORD_FILE="${PASSWORD_FILE:-}" SECRET=agent-relay-first-admin-password case "$DOMAIN" in *[!A-Za-z0-9.-]*|"") echo "app hostname must be a bare host name, e.g. relay.example.com" >&2; exit 2 ;; esac -for tool in gcloud kubectl curl jq openssl; do +case "$API_HOST" in *[!A-Za-z0-9.-]*|"") echo "API_HOST must be a bare host name, e.g. relay-api.example.com" >&2; exit 2 ;; esac +tools="kubectl curl jq openssl" +[ -n "$PASSWORD_FILE" ] || tools="gcloud $tools" +for tool in $tools; do command -v "$tool" >/dev/null || { echo "$tool is required" >&2; exit 2; } done @@ -33,15 +50,30 @@ kubectl -n "$NAMESPACE" get secret agent-relay-secrets -o jsonpath='{.data.selfh # 2. Store the password first. An existing secret means a run already stored # one: stop rather than replace a password the admin may already use. -if gcloud secrets describe "$SECRET" --project "$PROJECT" >/dev/null 2>&1; then +if [ -n "$PASSWORD_FILE" ]; then + if [ -e "$PASSWORD_FILE" ] || [ -L "$PASSWORD_FILE" ]; then + echo "$PASSWORD_FILE already exists: the first sign-in may already be done." >&2 + echo "Sign in with that password; to start over, delete the file yourself first." >&2 + exit 1 + fi + STORED="the file $PASSWORD_FILE" +elif gcloud secrets describe "$SECRET" --project "$PROJECT" >/dev/null 2>&1; then echo "Secret Manager already holds $SECRET in $PROJECT: the first sign-in may already be done." >&2 echo "Sign in with that password; to start over, delete the secret yourself first." >&2 exit 1 +else + STORED="$SECRET" fi PW="$(openssl rand -base64 33 | tr -d '\n/+=' | cut -c1-32)" -printf '%s' "$PW" | gcloud secrets create "$SECRET" --project "$PROJECT" \ - --replication-policy automatic --labels app=agent-relay --data-file=- >/dev/null -echo "stored $SECRET in $PROJECT" +if [ -n "$PASSWORD_FILE" ]; then + # noclobber: never write through a file or link created since the check. + (umask 077; set -o noclobber; printf '%s\n' "$PW" > "$PASSWORD_FILE") + echo "stored the password in $PASSWORD_FILE (mode 0600)" +else + printf '%s' "$PW" | gcloud secrets create "$SECRET" --project "$PROJECT" \ + --replication-policy automatic --labels app=agent-relay --data-file=- >/dev/null + echo "stored $SECRET in $PROJECT" +fi # 3. Post the key and password. The Origin header is required: sign-in routes # refuse a request without the dashboard's origin (403 csrf_rejected). @@ -50,14 +82,14 @@ CODE="$(PW="$PW" KEY="$(kubectl -n "$NAMESPACE" get secret agent-relay-secrets - jq -n '{setupKey:env.KEY,password:env.PW}' | curl -sS -o "$BODY" -w '%{http_code}' -X POST \ -H 'Content-Type: application/json' -H "Origin: https://$DOMAIN" \ - --data-binary @- "https://api.$DOMAIN/api/v1/auth/password/setup-key")" || CODE="${CODE:-000}" + --data-binary @- "https://$API_HOST/api/v1/auth/password/setup-key")" || CODE="${CODE:-000}" unset PW if [ "$CODE" = 200 ]; then - echo "first admin set: sign in at https://$DOMAIN/login with the password in $SECRET" + echo "first admin set: sign in at https://$DOMAIN/login with the password in $STORED" exit 0 fi # Error bodies carry a code and a message, never the key or the password. # 000: curl could not reach the API (DNS, certificate, network); the key was not sent. echo "setup-key sign-in failed: HTTP $CODE $(jq -c '{error,message}' "$BODY" 2>/dev/null || true)" >&2 -echo "The stored password was not accepted; delete $SECRET before running again." >&2 +echo "The stored password was not accepted (API: https://$API_HOST); delete $STORED before running again." >&2 exit 1