diff --git a/.agents/skills/setting-up-self-hosted-agent-relay-gcp/SKILL.md b/.agents/skills/setting-up-self-hosted-agent-relay-gcp/SKILL.md index 51d62529..0c65b324 100644 --- a/.agents/skills/setting-up-self-hosted-agent-relay-gcp/SKILL.md +++ b/.agents/skills/setting-up-self-hosted-agent-relay-gcp/SKILL.md @@ -58,6 +58,10 @@ Obtain these from the operator; do not discover or guess cloud identifiers: - release and namespace, normally `agent-relay`; - names of the global static IP, managed certificate, FrontendConfig, GCS bucket, Kubernetes service accounts, and existing Kubernetes Secret; +- name of the Compute SSL policy (TLS 1.2 minimum) for `gkeIngress.sslPolicy`, + and, when the customer wants them, the Cloud Armor policy + (`gkeIngress.securityPolicy`) or a TLS Secret (`gkeIngress.tlsSecretName`); + see `docs/gcp-install/3-EDGE-HARDENING.md`; - Artifact Registry repositories and immutable `sha256:` image digests; - a customer-specific, non-secret Helm overlay exported as `CUSTOMER_VALUES_FILE`; diff --git a/charts/agent-relay/Chart.yaml b/charts/agent-relay/Chart.yaml index f816bac8..b9b126bc 100644 --- a/charts/agent-relay/Chart.yaml +++ b/charts/agent-relay/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: agent-relay description: Self-hosted Agent Relay messaging and sessions suite type: application -version: 0.7.57 +version: 0.7.59 appVersion: "0.1.0" kubeVersion: ">=1.28.0-0" annotations: diff --git a/charts/agent-relay/README.md b/charts/agent-relay/README.md index 2181c1ff..b421da75 100644 --- a/charts/agent-relay/README.md +++ b/charts/agent-relay/README.md @@ -63,6 +63,7 @@ before installing. The required value overrides are: | `global.imageRegistry` | Artifact Registry hostname for the selected region, such as `us-central1-docker.pkg.dev`. | | `global.tenant.orgId` | Stable customer organization ID. Do not change it after data has been written. | | `gkeIngress.name`, `staticIpName`, `managedCertificateName`, `frontendConfigName` | Customer-selected names. `staticIpName` must name a reserved global external IPv4 address; the chart creates the other resources. | +| `gkeIngress.sslPolicy` | Recommended. Name of a Compute SSL policy with a TLS 1.2 minimum; see [GKE edge hardening](#gke-edge-hardening). | | `gkeIngress.hosts.{root,api,cast,history}` | Four customer-controlled public DNS names whose A records point at the reserved address. | | `relaycast.image.{repository,digest}` | Customer-accessible Relaycast image mirror and its real immutable digest. | | `sessions-pipeline.worker.image.{repository,digest}` and `sessions-pipeline.alleyoop.image.{repository,digest}` | Repository path under `global.imageRegistry` and the same immutable Sessions services digest. | @@ -252,6 +253,48 @@ For a base domain `relay.example.com`: Leaf ingresses are disabled and each ClusterIP backend uses a GKE standalone NEG. Sensitive Relaycast paths still route to an endpoint-less deny Service. +## GKE edge hardening + +Three optional `gkeIngress` values, each empty by default. Each names +something the chart cannot create, which must exist before the upgrade; +[`terraform/modules/gcp/edge-security`](https://github.com/AgentWorkforce/infra/blob/main/terraform/modules/gcp/edge-security) +creates the two policies. The operator steps and checks are in +[`docs/gcp-install/3-EDGE-HARDENING.md`](https://github.com/AgentWorkforce/infra/blob/main/docs/gcp-install/3-EDGE-HARDENING.md). + +| Value | Effect | +| --- | --- | +| `gkeIngress.sslPolicy` | Compute SSL policy for the HTTPS listener (`FrontendConfig.spec.sslPolicy`). Without it Google's default policy applies, which still accepts TLS 1.0 and 1.1. Use a policy with `min_tls_version = TLS_1_2`. | +| `gkeIngress.securityPolicy` | Cloud Armor backend security policy, set as `spec.securityPolicy.name` on the BackendConfig of **every** backend behind the Ingress. | +| `gkeIngress.tlsSecretName` | A `kubernetes.io/tls` Secret served instead of the ManagedCertificate (which is then not rendered). For an edge whose DNS points at a proxy or CDN, where Google cannot issue a managed certificate. The certificate must cover every host. | + +The chart leaves `sslPolicy` empty by default because a name that does not +exist in the project stops the Ingress from syncing. `terraform/examples/gcp` +creates the policy and exports it in `chart_values`, so an install made from +the GKE guide has it set. + +With `securityPolicy`, the api, cast, history, actors and relaygate +BackendConfigs gain the policy, and the chart renders policy-only BackendConfigs +for the backends that had none: `-dashboard`, +`-edge-deny` and, with `gkeIngress.hosts.files`, +`-files`. The dashboard and Relayfile Services belong to child +charts, so bind them in values (the render fails with the exact value +otherwise): + +```yaml +gkeIngress: + sslPolicy: agent-relay-tls + securityPolicy: agent-relay-edge +relay-webapp: + service: + annotations: + cloud.google.com/neg: '{"ingress": true}' + cloud.google.com/backend-config: '{"default": "agent-relay-dashboard"}' +``` + +Behind a proxy that fronts the load balancer (Cloudflare), also raise +`relay-control.auth.trustedProxies` by one (the Google load balancer alone +needs 2; the guide shows how to check). + ## Customer-neutral EKS with RDS native History `values-eks-rds-native.yaml` is the reusable EKS overlay for the supported diff --git a/charts/agent-relay/templates/gke-ingress.yaml b/charts/agent-relay/templates/gke-ingress.yaml index 7f87fd1e..6194d04e 100644 --- a/charts/agent-relay/templates/gke-ingress.yaml +++ b/charts/agent-relay/templates/gke-ingress.yaml @@ -1,6 +1,25 @@ {{- include "agent-relay.validate" . -}} {{- if dig "enabled" false (.Values.gkeIngress | default dict) }} {{- $gke := .Values.gkeIngress | default dict -}} +{{- /* Edge hardening, all optional and off by default (each names a Google +Cloud resource this chart cannot create; terraform/modules/gcp/edge-security +creates them): + sslPolicy Compute SSL policy for the load balancer's HTTPS listener + (FrontendConfig spec.sslPolicy). Without it Google's default + policy applies, which still accepts TLS 1.0. + securityPolicy Cloud Armor backend security policy, set on the + BackendConfig of every backend Service behind the Ingress. + tlsSecretName kubernetes.io/tls Secret served instead of the + ManagedCertificate, for an edge whose public DNS points at a + proxy in front of the load balancer (a Google-managed + certificate cannot be issued there). */ -}} +{{- $computeName := "^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$" -}} +{{- $sslPolicy := dig "sslPolicy" "" $gke | toString -}} +{{- if and $sslPolicy (not (regexMatch $computeName $sslPolicy)) }}{{ fail "gkeIngress.sslPolicy must be the name of a Compute Engine SSL policy in the cluster's project (lowercase letters, digits and hyphens, at most 63 characters), not a URL" }}{{ end -}} +{{- $securityPolicy := dig "securityPolicy" "" $gke | toString -}} +{{- if and $securityPolicy (not (regexMatch $computeName $securityPolicy)) }}{{ fail "gkeIngress.securityPolicy must be the name of a Cloud Armor security policy in the cluster's project (lowercase letters, digits and hyphens, at most 63 characters), not a URL" }}{{ end -}} +{{- $tlsSecretName := dig "tlsSecretName" "" $gke | toString -}} +{{- if and $tlsSecretName (not (regexMatch "^[a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?$" $tlsSecretName)) }}{{ fail "gkeIngress.tlsSecretName must be the name of a kubernetes.io/tls Secret in the release namespace" }}{{ end -}} {{- $components := .Values.components | default dict -}} {{- $relaycast := .Values.relaycast | default dict -}} {{- $cloudApi := index .Values "relay-control" | default dict -}} @@ -59,7 +78,8 @@ shape (hosts.files without components.relayfile fails further down). */ -}} {{- $nativeReadEdge := printf "%s-history-read" ($gkeName | trunc 50 | trimSuffix "-") -}} {{- $nativeIngestEdge := printf "%s-history-ingest" ($gkeName | trunc 48 | trimSuffix "-") -}} {{- $staticIpName := required "gkeIngress.staticIpName is required when gkeIngress.enabled=true" (dig "staticIpName" "" $gke) -}} -{{- $managedCertificateName := required "gkeIngress.managedCertificateName is required when gkeIngress.enabled=true" (dig "managedCertificateName" "" $gke) -}} +{{- $managedCertificateName := dig "managedCertificateName" "" $gke -}} +{{- if not (or $tlsSecretName $managedCertificateName) }}{{ fail "gkeIngress.managedCertificateName is required when gkeIngress.enabled=true (or set gkeIngress.tlsSecretName to serve a certificate from a Secret)" }}{{ end -}} {{- $frontendConfigName := required "gkeIngress.frontendConfigName is required when gkeIngress.enabled=true" (dig "frontendConfigName" "" $gke) -}} {{- $hostRoot := dig "hosts" "root" "" $gke -}} {{- $hostApi := dig "hosts" "api" "" $gke -}} @@ -172,6 +192,38 @@ derived from gkeIngress.name. The name is truncated to 59 characters before {{- $haveAnnotation := dig "service" "annotations" "cloud.google.com/backend-config" "" $cloudApi -}} {{- if ne (toString $haveAnnotation | nospace) (nospace $wantBackendConfig) }}{{ fail (printf "gkeIngress requires relay-control.service.backendConfig=%s (-api) so the api backend gets the BackendConfig timeout" $apiBackendConfig) }}{{ end -}} {{- end }} +{{- /* Cloud Armor attaches per backend service, through the BackendConfig +its Service names. The dashboard, Relayfile and edge-deny Services have no +BackendConfig otherwise, so with gkeIngress.securityPolicy each gets one that +carries only the policy. The edge-deny Service is rendered here and binds its +own; the dashboard and Relayfile Services belong to child charts, whose +values a parent cannot template, so their backend-config annotation is +checked against the derived name (the render fails with the value to set). */ -}} +{{- $dashboardBackendConfig := printf "%s-dashboard" ($gkeName | trunc 53 | trimSuffix "-") -}} +{{- $filesBackendConfig := printf "%s-files" ($gkeName | trunc 57 | trimSuffix "-") -}} +{{- $denyBackendConfig := printf "%s-edge-deny" ($gkeName | trunc 53 | trimSuffix "-") -}} +{{- if $securityPolicy }} +{{- $armorOnly := dict $dashboardBackendConfig (list "relay-webapp" $dashboard) -}} +{{- if $hostFiles }}{{ $_ := set $armorOnly $filesBackendConfig (list "relayfile" $relayfile) }}{{ end -}} +{{- range $bc, $child := $armorOnly }} +{{- $have := dig "service" "annotations" "cloud.google.com/backend-config" "" (index $child 1) | toString | nospace -}} +{{- if ne $have (printf "{\"default\":\"%s\"}" $bc) }}{{ fail (printf "gkeIngress.securityPolicy requires %s.service.annotations[cloud.google.com/backend-config]='{\"default\": \"%s\"}' so the Cloud Armor policy also covers that backend" (index $child 0) $bc) }}{{ end -}} +{{- end }} +{{- range $bc := concat (list $dashboardBackendConfig $denyBackendConfig) (ternary (list $filesBackendConfig) (list) (not (empty $hostFiles))) }} +apiVersion: cloud.google.com/v1 +kind: BackendConfig +metadata: + name: {{ $bc }} + namespace: {{ $.Release.Namespace | quote }} + labels: + app.kubernetes.io/name: agent-relay + app.kubernetes.io/instance: {{ $.Release.Name }} +spec: + securityPolicy: + name: {{ $securityPolicy | quote }} +--- +{{- end }} +{{- end }} apiVersion: cloud.google.com/v1 kind: BackendConfig metadata: @@ -181,6 +233,10 @@ metadata: app.kubernetes.io/name: agent-relay app.kubernetes.io/instance: {{ .Release.Name }} spec: + {{- with $securityPolicy }} + securityPolicy: + name: {{ . | quote }} + {{- end }} timeoutSec: {{ $apiTimeout }} --- {{- /* Relaycast WebSockets (/v1/ws, /v1/node/ws) through the cast backend are @@ -208,6 +264,10 @@ metadata: app.kubernetes.io/name: agent-relay app.kubernetes.io/instance: {{ .Release.Name }} spec: + {{- with $securityPolicy }} + securityPolicy: + name: {{ . | quote }} + {{- end }} timeoutSec: {{ $castTimeout }} --- {{- $gateEdge := printf "%s-gate" ($gkeName | trunc 58 | trimSuffix "-") -}} @@ -226,6 +286,10 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: gate-edge spec: + {{- with $securityPolicy }} + securityPolicy: + name: {{ . | quote }} + {{- end }} timeoutSec: {{ $castTimeout }} healthCheck: type: HTTP @@ -253,6 +317,7 @@ spec: - { name: http, port: 8080, targetPort: http, protocol: TCP } --- {{- end }} +{{- if not $tlsSecretName }} apiVersion: networking.gke.io/v1 kind: ManagedCertificate metadata: @@ -273,6 +338,7 @@ spec: - {{ . | quote }} {{- end }} --- +{{- end }} apiVersion: networking.gke.io/v1beta1 kind: FrontendConfig metadata: @@ -282,6 +348,9 @@ metadata: app.kubernetes.io/name: agent-relay app.kubernetes.io/instance: {{ .Release.Name }} spec: + {{- with $sslPolicy }} + sslPolicy: {{ . | quote }} + {{- end }} redirectToHttps: enabled: true responseCodeName: MOVED_PERMANENTLY_DEFAULT @@ -300,6 +369,9 @@ metadata: app.kubernetes.io/component: edge-deny annotations: cloud.google.com/neg: '{"ingress": true}' + {{- if $securityPolicy }} + cloud.google.com/backend-config: {{ printf "{\"default\": \"%s\"}" $denyBackendConfig | squote }} + {{- end }} spec: type: ClusterIP ports: @@ -323,6 +395,10 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: actors-edge spec: + {{- with $securityPolicy }} + securityPolicy: + name: {{ . | quote }} + {{- end }} timeoutSec: {{ $actorsTimeout | int }} healthCheck: type: HTTP @@ -362,6 +438,10 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: native-history-read-edge spec: + {{- with $securityPolicy }} + securityPolicy: + name: {{ . | quote }} + {{- end }} healthCheck: type: HTTP requestPath: /healthz @@ -377,6 +457,10 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: native-history-ingest-edge spec: + {{- with $securityPolicy }} + securityPolicy: + name: {{ . | quote }} + {{- end }} healthCheck: type: HTTP requestPath: /healthz @@ -435,9 +519,15 @@ metadata: annotations: kubernetes.io/ingress.class: gce kubernetes.io/ingress.global-static-ip-name: {{ $staticIpName | quote }} + {{- if not $tlsSecretName }} networking.gke.io/managed-certificates: {{ $managedCertificateName | quote }} + {{- end }} networking.gke.io/v1beta1.FrontendConfig: {{ $frontendConfigName | quote }} spec: + {{- if $tlsSecretName }} + tls: + - secretName: {{ $tlsSecretName | quote }} + {{- end }} rules: - host: {{ $hostRoot | quote }} http: diff --git a/charts/agent-relay/values-gcp.yaml b/charts/agent-relay/values-gcp.yaml index 6e6d8034..fdb2dd0e 100644 --- a/charts/agent-relay/values-gcp.yaml +++ b/charts/agent-relay/values-gcp.yaml @@ -39,6 +39,18 @@ gkeIngress: staticIpName: example-agent-relay-ingress managedCertificateName: example-agent-relay frontendConfigName: example-agent-relay + # RECOMMENDED: the name of a Compute SSL policy with min TLS 1.2 (create it + # first: docs/gcp-install step 7 or terraform/modules/gcp/edge-security). + # Left empty, the load balancer uses Google's default policy, which still + # accepts TLS 1.0 and 1.1. + # sslPolicy: example-agent-relay-tls + # Optional: a Cloud Armor security policy for every backend (README: GKE + # edge hardening). Also set the dashboard Service's backend-config + # annotation, relay-webapp.service.annotations below. + # securityPolicy: example-agent-relay-edge + # Optional: serve a certificate from a kubernetes.io/tls Secret instead of + # the ManagedCertificate (an edge behind a proxy or CDN). + # tlsSecretName: example-agent-relay-tls hosts: # REQUIRED OVERRIDE: four public DNS names controlled by the customer. root: dashboard.placeholder.company @@ -495,6 +507,8 @@ relay-webapp: service: annotations: cloud.google.com/neg: '{"ingress": true}' + # Required with gkeIngress.securityPolicy: -dashboard. + # cloud.google.com/backend-config: '{"default": "example-agent-relay-dashboard"}' ingress: enabled: false networkPolicy: diff --git a/charts/agent-relay/values.yaml b/charts/agent-relay/values.yaml index 661015f5..d8e9657c 100644 --- a/charts/agent-relay/values.yaml +++ b/charts/agent-relay/values.yaml @@ -130,6 +130,25 @@ gkeIngress: staticIpName: agent-relay-ingress managedCertificateName: agent-relay frontendConfigName: agent-relay + # Optional edge hardening. Each names a Google Cloud resource in the + # cluster's project that must exist before the upgrade (the chart cannot + # create it; terraform/modules/gcp/edge-security does). README: GKE edge + # hardening. + # + # Compute SSL policy for the HTTPS listener. Empty leaves Google's default + # policy, which still accepts TLS 1.0 and 1.1; set a policy with + # min TLS 1.2 (profile MODERN or RESTRICTED). + sslPolicy: "" + # Cloud Armor backend security policy, applied to every backend Service + # behind the Ingress. Needs the backend-config annotation on the + # relay-webapp Service (and relayfile with hosts.files); the render fails + # with the exact value when it is missing. + securityPolicy: "" + # kubernetes.io/tls Secret served instead of the ManagedCertificate (which + # is then not rendered). For an edge whose DNS points at a proxy or CDN in + # front of the load balancer, where Google cannot issue a managed + # certificate. The certificate must cover every host below. + tlsSecretName: "" hosts: root: "" api: "" diff --git a/ci/agent-relay/render.sh b/ci/agent-relay/render.sh index 6aba2e12..1d9a0a44 100755 --- a/ci/agent-relay/render.sh +++ b/ci/agent-relay/render.sh @@ -902,6 +902,68 @@ test "$(helm template proof charts/agent-relay "${chief_args[@]}" --set componen || { echo "relay-chief is not a suspended CronJob when enabled" >&2; exit 1; } rm -f "$chief_values" +# GKE edge hardening: gkeIngress.sslPolicy, securityPolicy and tlsSecretName +# are off by default (the default render has none of them) and, when set, reach +# the FrontendConfig, every backend's BackendConfig and the Ingress. +edge_fail() { echo "gke edge hardening: $*" >&2; exit 1; } +edge_default=$(helm template proof charts/agent-relay -n agent-relay --no-hooks \ + -f charts/agent-relay/values-gcp-proof.yaml -f charts/agent-relay/ci/gcp-proof-values.yaml) +test "$(yq 'select(.kind == "BackendConfig") | .metadata.name' <<<"$edge_default" | grep -c '^agent-relay-gcp-proof-')" = 4 || edge_fail "default render must keep exactly the api, cast and two history BackendConfigs" +test -z "$(yq 'select(.kind == "FrontendConfig") | .spec.sslPolicy // ""' <<<"$edge_default")" || edge_fail "default render sets an sslPolicy" +test -z "$(yq 'select(.kind == "BackendConfig") | .spec.securityPolicy.name // ""' <<<"$edge_default" | sed '/^---$/d;/^$/d')" || edge_fail "default render sets a securityPolicy" +test -z "$(yq 'select(.kind == "Ingress") | .spec.tls // ""' <<<"$edge_default")" || edge_fail "default render sets spec.tls" +edge_dashboard='relay-webapp.service.annotations={"cloud.google.com/neg":"{\"ingress\": true}","cloud.google.com/backend-config":"{\"default\": \"agent-relay-gcp-proof-dashboard\"}"}' +edge_out=$(helm template proof charts/agent-relay -n agent-relay --no-hooks \ + -f charts/agent-relay/values-gcp-proof.yaml -f charts/agent-relay/ci/gcp-proof-values.yaml \ + --set gkeIngress.sslPolicy=relay-tls --set gkeIngress.securityPolicy=relay-edge --set-json "$edge_dashboard") +test "$(yq 'select(.kind == "FrontendConfig") | .spec.sslPolicy + " " + (.spec.redirectToHttps.enabled | tostring)' <<<"$edge_out")" = "relay-tls true" \ + || edge_fail "FrontendConfig must carry sslPolicy and keep the HTTPS redirect" +# Every BackendConfig carries the policy, and the timeouts and health checks stay. +test "$(yq 'select(.kind == "BackendConfig") | .metadata.name + "=" + (.spec.securityPolicy.name // "none")' <<<"$edge_out" | sed '/^---$/d;/^$/d' | sort | tr '\n' ' ')" \ + = "agent-relay-gcp-proof-api=relay-edge agent-relay-gcp-proof-cast=relay-edge agent-relay-gcp-proof-dashboard=relay-edge agent-relay-gcp-proof-edge-deny=relay-edge agent-relay-gcp-proof-history-ingest=relay-edge agent-relay-gcp-proof-history-read=relay-edge " \ + || edge_fail "securityPolicy must be on the api, cast, dashboard, edge-deny and both history BackendConfigs" +test "$(yq 'select(.kind == "BackendConfig" and .metadata.name == "agent-relay-gcp-proof-api") | .spec.timeoutSec' <<<"$edge_out")" = 3600 || edge_fail "api BackendConfig lost its timeout" +test "$(yq 'select(.kind == "BackendConfig" and .metadata.name == "agent-relay-gcp-proof-history-read") | .spec.healthCheck.requestPath' <<<"$edge_out")" = /healthz || edge_fail "history read BackendConfig lost its health check" +# Every Service the Ingress routes to is bound to a BackendConfig that has the +# policy: no backend is left outside Cloud Armor. +edge_backends=$(yq 'select(.kind == "Ingress") | .spec.rules[].http.paths[].backend.service.name' <<<"$edge_out" | sort -u) +test "$(grep -c . <<<"$edge_backends")" -ge 6 || edge_fail "expected at least six Ingress backends, got: $edge_backends" +for svc in $edge_backends; do + bc=$(SVC="$svc" yq 'select(.kind == "Service" and .metadata.name == strenv(SVC)) | .metadata.annotations["cloud.google.com/backend-config"] // ""' <<<"$edge_out" | yq '.default // ""') + [ -n "$bc" ] || edge_fail "Ingress backend Service $svc has no backend-config annotation" + test "$(BC="$bc" yq 'select(.kind == "BackendConfig" and .metadata.name == strenv(BC)) | .spec.securityPolicy.name' <<<"$edge_out")" = relay-edge \ + || edge_fail "Ingress backend Service $svc is bound to $bc, which has no securityPolicy" +done +expect_umbrella_failure "gkeIngress.securityPolicy requires relay-webapp.service.annotations[cloud.google.com/backend-config]='{\"default\": \"agent-relay-gcp-proof-dashboard\"}'" \ + --set gkeIngress.securityPolicy=relay-edge +expect_umbrella_failure "gkeIngress.securityPolicy must be the name of a Cloud Armor security policy" \ + --set gkeIngress.securityPolicy=projects/p/global/securityPolicies/relay-edge +expect_umbrella_failure "gkeIngress.sslPolicy must be the name of a Compute Engine SSL policy" --set gkeIngress.sslPolicy=Relay_TLS +# tlsSecretName replaces the ManagedCertificate (an edge behind a proxy or CDN). +tls_out=$(helm template proof charts/agent-relay -n agent-relay --no-hooks \ + -f charts/agent-relay/values-gcp-proof.yaml -f charts/agent-relay/ci/gcp-proof-values.yaml \ + --set gkeIngress.tlsSecretName=relay-origin-tls --set gkeIngress.managedCertificateName=) +test -z "$(yq 'select(.kind == "ManagedCertificate") | .metadata.name' <<<"$tls_out")" || edge_fail "tlsSecretName must not render a ManagedCertificate" +test "$(yq 'select(.kind == "Ingress") | .spec.tls[0].secretName + " " + (.metadata.annotations | has("networking.gke.io/managed-certificates") | tostring) + " " + .metadata.annotations["networking.gke.io/v1beta1.FrontendConfig"]' <<<"$tls_out")" \ + = "relay-origin-tls false agent-relay-gcp-proof" || edge_fail "tlsSecretName must set spec.tls, drop the managed-certificates annotation and keep the FrontendConfig" +expect_umbrella_failure "gkeIngress.tlsSecretName must be the name of a kubernetes.io/tls Secret" --set gkeIngress.tlsSecretName=Bad_Name +expect_umbrella_failure "gkeIngress.managedCertificateName is required" --set gkeIngress.managedCertificateName= + +# With the optional actors and files hosts, their backends are covered too: +# the actors edge BackendConfig takes the policy, and Relayfile needs its own. +edge_files='relayfile.service.annotations={"cloud.google.com/neg":"{\"ingress\": true}","cloud.google.com/backend-config":"{\"default\": \"agent-relay-gcp-proof-files\"}"}' +edge_da=$(helm template agent-relay charts/agent-relay -n agent-relay --no-hooks "${da_args[@]}" \ + --set gkeIngress.securityPolicy=relay-edge --set-json "$edge_dashboard" --set-json "$edge_files") +test -z "$(yq 'select(.kind == "BackendConfig" and .spec.securityPolicy.name != "relay-edge") | .metadata.name' <<<"$edge_da" | sed '/^---$/d;/^$/d')" \ + || edge_fail "a BackendConfig in the actors/files overlay has no securityPolicy" +for bc in agent-relay-gcp-proof-actors-edge agent-relay-gcp-proof-files; do + test "$(BC="$bc" yq 'select(.kind == "BackendConfig" and .metadata.name == strenv(BC)) | .spec.securityPolicy.name' <<<"$edge_da")" = relay-edge || edge_fail "$bc has no securityPolicy" +done +test "$(yq 'select(.kind == "BackendConfig" and .metadata.name == "agent-relay-gcp-proof-actors-edge") | .spec.timeoutSec' <<<"$edge_da")" = 3600 || edge_fail "actors edge BackendConfig lost its timeout" +if edge_err=$(helm template agent-relay charts/agent-relay -n agent-relay --no-hooks "${da_args[@]}" \ + --set gkeIngress.securityPolicy=relay-edge --set-json "$edge_dashboard" 2>&1); then edge_fail "securityPolicy rendered without the relayfile backend-config annotation"; fi +grep -Fq "gkeIngress.securityPolicy requires relayfile.service.annotations[cloud.google.com/backend-config]" <<<"$edge_err" || edge_fail "unexpected error without the relayfile annotation: $edge_err" + # With single-host mode off, every default (subdomain) render is unchanged. ci/agent-relay/subdomain-golden.sh check diff --git a/docs/gcp-install/0-START-HERE.md b/docs/gcp-install/0-START-HERE.md index c6486f6c..1bc06dbe 100644 --- a/docs/gcp-install/0-START-HERE.md +++ b/docs/gcp-install/0-START-HERE.md @@ -581,6 +581,11 @@ curl -s -o /dev/null -w "history %{http_code}\n" "https://history.$DOMAIN/v1/ses - **502 for a few minutes after `Active`:** the load balancer's health checks are still converging. Wait 5 minutes. +The load balancer accepts TLS 1.0 and later until it has an SSL policy. To +require TLS 1.2, add Cloud Armor, or put Cloudflare in front, see +[3-EDGE-HARDENING.md](3-EDGE-HARDENING.md) (from the first release with chart +0.7.59; not the release this guide pins). + ## 15. First sign-in, then add users The first admin does this **before signing in any other way**, including the diff --git a/docs/gcp-install/3-EDGE-HARDENING.md b/docs/gcp-install/3-EDGE-HARDENING.md new file mode 100644 index 00000000..b47732b2 --- /dev/null +++ b/docs/gcp-install/3-EDGE-HARDENING.md @@ -0,0 +1,307 @@ +# Hardening the GKE edge + +Three optional controls on the load balancer in front of the install. Do them +after step 14 of [0-START-HERE.md](0-START-HERE.md), in any order. + +| Control | What it does | Default | +| --- | --- | --- | +| [TLS policy](#1-tls-policy) | Refuses TLS 1.0 and 1.1 | **On** when this release's Terraform (step 4) created your resources | +| [Cloud Armor](#2-cloud-armor) | OWASP rules, a sign-in rate limit, and optionally an allowlist of source addresses | Off | +| [A proxy in front (Cloudflare)](#3-cloudflare-in-front) | Only the proxy may reach the load balancer | Off | + +**Which release.** These controls need the `agent-relay` chart 0.7.59 or +later and the Terraform from the same release. The release that +0-START-HERE.md currently pins (chart 0.7.54) has neither: its chart ignores +`gkeIngress.sslPolicy`, `securityPolicy` and `tlsSecretName` without an +error, and its Terraform has no `edge_*` inputs. Check with +`helm show chart "$CHART" | grep '^version'` before you start. + +The chart and Terraform sides are covered by render and plan tests in this +repository. They have not yet been proven on a live GKE install; follow the +"Done when" checks below on yours. + +## 1. TLS policy + +Without an SSL policy a Google load balancer accepts TLS 1.0 and later. +`terraform/examples/gcp` creates the policy `-tls` (profile `MODERN`, +TLS 1.2 minimum) and puts its name in `managed-values.json` +(`gkeIngress.sslPolicy`), so an install made with this release's step 4 and +step 11 has it already (step 4's plan then shows one resource more than the +count the guide states for the earlier release). + +**An install made with an earlier release:** upgrade to this release's chart +and Terraform first, then run step 4's `terraform plan`/`apply` again (it +adds one resource), write `managed-values.json` again, and run step 11's +`helm upgrade`. + +**Brought your own resources?** Create the policy and name it in `values.yaml`: + +```sh +gcloud compute ssl-policies create agent-relay-tls --profile MODERN --min-tls-version 1.2 --global +``` + +```yaml +gkeIngress: + sslPolicy: agent-relay-tls +``` + +**Done when** (allow 5 minutes after the upgrade): + +```sh +kubectl -n $NAMESPACE get frontendconfig agent-relay -o jsonpath='{.spec.sslPolicy}{"\n"}' +openssl s_client -connect "$DOMAIN:443" -servername "$DOMAIN" -tls1_1 &1 | grep -c 'Cipher is (NONE)\|alert protocol version\|no protocols available' +openssl s_client -connect "$DOMAIN:443" -servername "$DOMAIN" -tls1_2 /dev/null | grep 'Protocol' +``` + +The first prints the policy name, the second a number above 0 (TLS 1.1 is +refused), the third `TLSv1.2`. + +- **`Error syncing to GCP ... sslPolicies/... was not found`** in + `kubectl -n $NAMESPACE describe ingress agent-relay`: the policy does not + exist in the cluster's project. The load balancer keeps its previous + settings until it does. +- **Stricter ciphers:** `edge_ssl_policy = { profile = "RESTRICTED" }` in + `terraform.tfvars`. Check the desktop app and your CLIs connect before you + roll it out. +- **Opt out:** `edge_ssl_policy = { create = false }`. + +## 2. Cloud Armor + +The policy applies to every backend behind the load balancer: the dashboard, +the API, Relaycast, both history services, and Relayfile and durable actors +when you run them. + +1. Create the policy. In `infra/terraform/examples/gcp/terraform.tfvars`: + + ```hcl + edge_security_policy = { create = true } + ``` + + Run `terraform apply`. `terraform output edge` prints its name + (`-edge`). This policy has the OWASP rules in **preview** (they log + and do not block) and a limit of 60 sign-in requests a minute per client. + +2. Name it in `values.yaml`, and bind the dashboard Service to its + BackendConfig (`-dashboard`): + + ```yaml + gkeIngress: + securityPolicy: agent-relay-edge + relay-webapp: + service: + annotations: + cloud.google.com/neg: '{"ingress": true}' + cloud.google.com/backend-config: '{"default": "agent-relay-dashboard"}' + ``` + + Run step 11's `helm upgrade`. If an annotation is missing, the upgrade + stops and prints the value to set. + +**Done when** (allow 10 minutes): + +```sh +kubectl -n $NAMESPACE get backendconfig -o custom-columns=NAME:.metadata.name,POLICY:.spec.securityPolicy.name +gcloud compute backend-services list --format='table(name,securityPolicy.basename())' | grep -c agent-relay-edge +``` + +Every BackendConfig shows the policy, and the count equals the number of +backend services of the Ingress (6 for the default install). + +**Turn the OWASP rules on.** Session transcripts and messages contain code, +SQL and shell text, which is what those rules look for. Use the install +normally for a week, then look for requests the rules would have blocked: + +```sh +gcloud logging read 'resource.type="http_load_balancer" AND jsonPayload.previewSecurityPolicy.outcome="DENY"' \ + --freshness 7d --limit 50 --format='table(httpRequest.requestUrl,jsonPayload.previewSecurityPolicy.preconfiguredExprIds)' +``` + +When it prints nothing you care about, set +`edge_security_policy = { create = true, waf = { preview = false } }` and +apply. To drop a rule that only matches your own traffic, list the ones you +keep in `waf.rules` (`terraform/modules/gcp/edge-security`). + +**Only some source addresses** (for example your VPN's egress range): + +```hcl +edge_security_policy = { create = true, source_ip_allowlist = ["198.51.100.0/24"] } +``` + +Everything else gets 403. The desktop app, CLIs and any agent that uploads +sessions must come from those addresses too. + +## 3. Cloudflare in front + +With Cloudflare proxying the four names, lock the load balancer so that only +Cloudflare can reach it. + +1. **Certificate.** Google cannot issue the managed certificate while the + names resolve to Cloudflare. Serve a certificate from a Secret instead, for + example a [Cloudflare Origin CA](https://developers.cloudflare.com/ssl/origin-configuration/origin-ca/) + certificate that covers all four names: + + ```sh + kubectl -n $NAMESPACE create secret tls agent-relay-origin-tls --cert origin.pem --key origin.key + ``` + + ```yaml + gkeIngress: + tlsSecretName: agent-relay-origin-tls + ``` + + Step 14's `managedcertificate` check no longer applies. Renew the Secret + before the certificate expires; nothing renews it for you. In Cloudflare + set the zone's SSL mode to **Full (strict)**. + +2. **Names.** The install uses ``, `api.`, `cast.` and + `history.`. Cloudflare's free Universal certificate covers a zone's + apex and one level below it. If `` is itself a subdomain + (`ar.example.com`, so `api.ar.example.com`), order an Advanced certificate + for the four names or turn on Total TLS. + +3. **Policy.** In `terraform.tfvars`: + + ```hcl + edge_security_policy = { create = true, cloudflare = { enabled = true } } + edge_origin_secret_header = { name = "x-origin-auth", values = [""] } + ``` + + In Cloudflare, add a **request header transform rule** for the four names + that sets `x-origin-auth` to the same value. Apply Terraform, then set + `gkeIngress.securityPolicy` as in section 2. Cloudflare's addresses are + shared by all its customers; the header is what proves a request came + through your zone. Its value is in the Terraform state and in the Cloud + Armor policy, so keep the state private. **To rotate it:** put the new + value second in `values` and apply, change the Cloudflare rule to the new + value, then remove the old value and apply. Requests keep flowing + throughout. + +4. **Client addresses.** relay-control finds the client's address by + counting `relay-control.auth.trustedProxies` entries from the right of the + `X-Forwarded-For` header. The Google load balancer appends two entries: + the address it received the request from, and its own. Measured on a live + GKE install, the header is `, ` and the right value + is 2. Cloudflare adds one more, so behind Cloudflare the header is + `, , ` and the value is 3. The + Cloudflare case is expected, not yet measured; check it as below. + + ```yaml + relay-control: + auth: + trustedProxies: 3 # Cloudflare, then the Google load balancer's two entries + ``` + + To check: load the dashboard from a network whose public address you know + (`curl -s https://ifconfig.me`), then read the header the pods received: + + ```sh + kubectl -n $NAMESPACE logs deploy/relay-webapp -c dashboard --tail 5 + ``` + + The last quoted field of each line is `X-Forwarded-For`. Count from the + right to your address: that count is `trustedProxies`. A number that is + too low makes every user share one sign-in limit and lockout; a number + that is too high lets a client forge its address. + + **Raise the number only after item 3 is applied and the "Done when" below + shows `direct 403`.** While the load balancer still answers requests that + skip Cloudflare, a caller who connects to it directly can put any address + in `X-Forwarded-For`, and with 3 the API would believe it. + +5. **Which clients may reach which paths** (optional). To let a vendor's + published ranges reach only the session upload paths, your own ranges + reach everything, and nobody else anything, add `client_ip_rules` to the + policy. Cloud Armor then decides by the client address Cloudflare reports + (`CF-Connecting-IP`), which the secret header from item 3 makes + trustworthy: + + ```hcl + edge_security_policy = { + create = true + cloudflare = { enabled = true } + rate_limits = [] + client_ip_rules = [ + { name = "vendors", hosts = ["history."], path_prefixes = ["/v1/delivery", "/v1/turns", "/v1/auth/token/refresh"], cidrs = ["", ""], rate_limit = { count = 600, interval_sec = 60 } }, + { name = "sign-in", path_prefixes = ["/login", "/api/v1/auth", "/cloud/api/v1/auth"], cidrs = [""], rate_limit = { count = 60, interval_sec = 60 } }, + { name = "staff", path_prefixes = ["/"], cidrs = [""], unthrottled = true }, + ] + } + ``` + + The first rule that matches a request's path decides it: its listed + clients are admitted, everyone else gets 403. The last rule closes every + other path. A request that a rule admits is not checked by any later + rule, so each rule carries its own `rate_limit` (per client), and the + sign-in limit is a rule of its own ahead of the closing one. The module's + README (`terraform/modules/gcp/edge-security`) explains the limits. A + vendor that publishes hundreds of ranges needs a higher Cloud Armor quota, + or that list enforced as a Cloudflare WAF custom rule instead. + +**Done when:** + +```sh +curl -s -o /dev/null -w "through Cloudflare %{http_code}\n" "https://api.$DOMAIN/readyz" +curl -sk -o /dev/null -w "direct %{http_code}\n" --resolve "api.$DOMAIN:443:$(gcloud compute addresses describe $STATIC_IP_NAME --global --format='value(address)')" "https://api.$DOMAIN/readyz" +``` + +`through Cloudflare 200` and `direct 403`, and the count in item 4 matches +`trustedProxies`. + +- **Long-lived connections.** The desktop app, Relaycast and the flows worker + queue hold WebSocket and event-stream connections open. Cloudflare proxies + WebSockets by default and closes a connection that carries no data for 100 + seconds; the services send keep-alives more often than that, but this path + has not been proven end to end. +- **Upload size.** Cloudflare limits a request body to 100 MB on its Free and + Pro plans. +- **Labels.** The Cloud Armor policy carries your `labels`. SSL policies, and + the forwarding rules and backend services that GKE creates for the Ingress, + have no labels in Google's API. For cost reports use + [GKE cost allocation](https://cloud.google.com/kubernetes-engine/docs/how-to/cost-allocations), + which covers what runs in the cluster. +- **Cloudflare changes its ranges** rarely and announces it. Before each + `terraform apply`, compare with + `cloudflare.ipv4_cidrs` in the module (the list there is dated). + +## Removing these + +**Cloud Armor.** Clearing `gkeIngress.securityPolicy` alone does not detach +the policy: GKE leaves a backend's policy in place when the field is absent, +and detaches only when it is set to an empty name. A policy that a backend +still uses cannot be deleted. So, in this order: + +```sh +# 1. Detach from every backend, while the value is still set. +for bc in $(kubectl -n $NAMESPACE get backendconfig -o name); do + kubectl -n $NAMESPACE patch "$bc" --type merge -p '{"spec":{"securityPolicy":{"name":""}}}' +done +# 2. Wait until no backend service uses it (prints 0; allow 10 minutes). +gcloud compute backend-services list --format='value(securityPolicy)' | grep -c agent-relay-edge +``` + +3. Clear `gkeIngress.securityPolicy` (and the dashboard's `backend-config` + annotation) in `values.yaml` and run step 11's `helm upgrade`. +4. Set `edge_security_policy = { create = false }` and `terraform apply`. + +**TLS policy.** The name reaches the chart from `managed-values.json`, which +only changes after a Terraform apply, and Terraform cannot delete a policy +that is still attached. So detach it through `values.yaml` first: + +1. Add `sslPolicy: ""` under `gkeIngress:` in `values.yaml` (it is read after + `managed-values.json`, so it wins) and run step 11's `helm upgrade`. +2. Wait until the load balancer no longer uses the policy: + + ```sh + gcloud compute target-https-proxies list --format='table(name,sslPolicy.basename())' + ``` + + If the policy is still listed after 10 minutes, clear it on that proxy: + `gcloud compute target-https-proxies update --clear-ssl-policy`. +3. Set `edge_ssl_policy = { create = false }`, run `terraform apply`, and + write `managed-values.json` again (step 4). The line added in 1 can then + be removed. + +The load balancer goes back to Google's default policy, which accepts TLS 1.0. + +The detach behaviour is from Google's documentation and has not been run on a +live install here. diff --git a/docs/gcp-install/README.md b/docs/gcp-install/README.md index a34ae2fa..a3875e03 100644 --- a/docs/gcp-install/README.md +++ b/docs/gcp-install/README.md @@ -9,6 +9,7 @@ cluster, the GCP counterpart of the AWS EKS guide. Start with | `0-START-HERE.md` | Numbered copy-paste steps, each with a "Done when" check | | `1-BRING-YOUR-OWN.md` | Specs for creating Cloud SQL, GCS, Memorystore, Pub/Sub, search and certificates yourself instead of Terraform | | `2-SERVICE-ACCOUNTS.md` | Workload Identity: every grant the install needs, and who needs which roles | +| `3-EDGE-HARDENING.md` | Optional: the load balancer's TLS policy, Cloud Armor, and Cloudflare in front | | `NOT-QUALIFIED.md` | What 2026.11.16 does not yet qualify on GKE. Read before a production rollout | | `images.md` | The release images and how they are mirrored | | `values.yaml` | Site values template (placeholders filled in step 9) | diff --git a/docs/gcp-install/values.yaml b/docs/gcp-install/values.yaml index 2db306c3..9a1c9b2f 100644 --- a/docs/gcp-install/values.yaml +++ b/docs/gcp-install/values.yaml @@ -61,6 +61,12 @@ gkeIngress: staticIpName: __STATIC_IP_NAME__ managedCertificateName: agent-relay frontendConfigName: agent-relay + # Chart 0.7.59 or later only (an earlier chart ignores these keys): + # sslPolicy (TLS 1.2 minimum) then comes from managed-values.json (step 4). + # Optional, 3-EDGE-HARDENING.md: Cloud Armor on every backend, and a + # certificate from a Secret when a proxy such as Cloudflare is in front. + # securityPolicy: agent-relay-edge + # tlsSecretName: agent-relay-origin-tls hosts: root: __DOMAIN__ api: api.__DOMAIN__ diff --git a/docs/self-hosted-requirements.md b/docs/self-hosted-requirements.md index d4c3650c..e16fa64a 100644 --- a/docs/self-hosted-requirements.md +++ b/docs/self-hosted-requirements.md @@ -14,7 +14,7 @@ registered, every shipped entry must exist in the charts, planned entries must not be referenced by any template, and the compatibility matrix must equal the pinned digests. -Source of truth: [`self-hosted/requirements.yaml`](../self-hosted/requirements.yaml). 199 entries: 55 requirements (55 shipped, 0 planned), 144 options (143 shipped, 1 planned). +Source of truth: [`self-hosted/requirements.yaml`](../self-hosted/requirements.yaml). 200 entries: 55 requirements (55 shipped, 0 planned), 145 options (144 shipped, 1 planned). ## Contents @@ -167,7 +167,8 @@ Services image (sessions-pipeline-worker, alleyoop and the privileged SQL Jobs): | relayauth-0.1.2 | shipped | relayauth 0.1.2 | relayauth: `ghcr.io/agentworkforce/relayauth@sha256:c23c92d25fdf…` | | | License-annotation patch bump of 0.1.1. Superseded by 0.1.3. | | relayauth-0.1.1 | shipped | relayauth 0.1.1 | relayauth: `ghcr.io/agentworkforce/relayauth@sha256:c23c92d25fdf…` | | | relayauth 784582ef (machine-bound History assertion contract, relayauth #113). Superseded by 0.1.2. | | relayauth-0.1.0 | shipped | relayauth 0.1.0 | relayauth: `ghcr.io/agentworkforce/relayauth@sha256:eb71daa28e30…` | | | superseded by 0.1.1. | -| agent-relay-0.7.57 | unreleased | agent-relay 0.7.57
relay-control 0.4.22
relay-webapp 0.2.7
(deps: relaycast 0.3.16, relayauth 0.1.5, sessions-pipeline 0.8.49, relay-control 0.4.22, relay-webapp 0.2.7, relayflows 0.2.5, durable-actors 0.14.6-awf.4, relayfile 0.2.2, telemetry-collector 0.1.7, relay-chief 0.1.0, slack-relay 0.3.11, relaygate 0.1.10) | relaycast: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
relaycast-postgres: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
sessions-pipeline-worker: `ghcr.io/agentworkforce/sessions-pipeline@sha256:77022c5b6a1d…`
relay-control: `ghcr.io/agentworkforce/relay-control@sha256:d1b6ca95fab2…`
relay-webapp: `ghcr.io/agentworkforce/relay-webapp@sha256:bd295ccb7d13…` | | | Carries 0.7.56. Opt-in externalSecrets (default off): renders one External Secrets Operator ExternalSecret (external-secrets.io/v1) per externalSecrets.secrets entry, targeting the Kubernetes Secret of that name, so install Secrets can come from AWS SSM Parameter Store, AWS Secrets Manager or another ESO provider without chart edits; store.create renders a namespaced SecretStore from store.provider. Default renders differ only in the helm.sh/chart labels. | +| agent-relay-0.7.59 | unreleased | agent-relay 0.7.59
relay-control 0.4.22
relay-webapp 0.2.7
(deps: relaycast 0.3.16, relayauth 0.1.5, sessions-pipeline 0.8.49, relay-control 0.4.22, relay-webapp 0.2.7, relayflows 0.2.5, durable-actors 0.14.6-awf.4, relayfile 0.2.2, telemetry-collector 0.1.7, relay-chief 0.1.0, slack-relay 0.3.11, relaygate 0.1.10) | relaycast: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
relaycast-postgres: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
sessions-pipeline-worker: `ghcr.io/agentworkforce/sessions-pipeline@sha256:77022c5b6a1d…`
relay-control: `ghcr.io/agentworkforce/relay-control@sha256:d1b6ca95fab2…`
relay-webapp: `ghcr.io/agentworkforce/relay-webapp@sha256:bd295ccb7d13…` | | | Carries 0.7.57 (0.7.58 is taken by another open change). GKE edge hardening, all opt-in and empty by default: gkeIngress.sslPolicy names a Compute SSL policy that the FrontendConfig applies to the HTTPS listener (without one Google's default policy still accepts TLS 1.0 and 1.1); gkeIngress.securityPolicy names a Cloud Armor backend security policy set on the BackendConfig of every backend behind the Ingress (api, cast, both history edges, the actors and relaygate edges, and new policy-only BackendConfigs -dashboard, -edge-deny and -files; the relay-webapp and relayfile Services need their backend-config annotation and the render fails with the exact value otherwise); gkeIngress.tlsSecretName serves a kubernetes.io/tls Secret instead of the ManagedCertificate, for an edge whose DNS points at a proxy or CDN. The resources come from terraform/modules/gcp/edge-security; terraform/examples/gcp now creates the TLS 1.2 SSL policy by default and exports it in chart_values. Default renders differ only in the helm.sh/chart labels (subdomain golden unchanged); images unchanged. | +| agent-relay-0.7.57 | **planned** | agent-relay 0.7.57
relay-control 0.4.22
relay-webapp 0.2.7
(deps: relaycast 0.3.16, relayauth 0.1.5, sessions-pipeline 0.8.49, relay-control 0.4.22, relay-webapp 0.2.7, relayflows 0.2.5, durable-actors 0.14.6-awf.4, relayfile 0.2.2, telemetry-collector 0.1.7, relay-chief 0.1.0, slack-relay 0.3.11, relaygate 0.1.10) | relaycast: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
relaycast-postgres: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
sessions-pipeline-worker: `ghcr.io/agentworkforce/sessions-pipeline@sha256:77022c5b6a1d…`
relay-control: `ghcr.io/agentworkforce/relay-control@sha256:d1b6ca95fab2…`
relay-webapp: `ghcr.io/agentworkforce/relay-webapp@sha256:bd295ccb7d13…` | | | Carries 0.7.56. Opt-in externalSecrets (default off): renders one External Secrets Operator ExternalSecret (external-secrets.io/v1) per externalSecrets.secrets entry, targeting the Kubernetes Secret of that name, so install Secrets can come from AWS SSM Parameter Store, AWS Secrets Manager or another ESO provider without chart edits; store.create renders a namespaced SecretStore from store.provider. Default renders differ only in the helm.sh/chart labels. | | agent-relay-0.7.56 | **planned** | agent-relay 0.7.56
relay-control 0.4.22
relay-webapp 0.2.7
(deps: relaycast 0.3.16, relayauth 0.1.5, sessions-pipeline 0.8.49, relay-control 0.4.22, relay-webapp 0.2.7, relayflows 0.2.5, durable-actors 0.14.6-awf.4, relayfile 0.2.2, telemetry-collector 0.1.7, relay-chief 0.1.0, slack-relay 0.3.11, relaygate 0.1.10) | relaycast: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
relaycast-postgres: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
sessions-pipeline-worker: `ghcr.io/agentworkforce/sessions-pipeline@sha256:77022c5b6a1d…`
relay-control: `ghcr.io/agentworkforce/relay-control@sha256:d1b6ca95fab2…`
relay-webapp: `ghcr.io/agentworkforce/relay-webapp@sha256:bd295ccb7d13…` | | | Carries sessions-pipeline 0.8.49 (searchIndexer default memory sized for the streaming reconcile: request 1Gi, limit 3Gi). | | agent-relay-0.7.55 | **planned** | agent-relay 0.7.55
relay-control 0.4.22
relay-webapp 0.2.7
(deps: relaycast 0.3.16, relayauth 0.1.5, sessions-pipeline 0.8.48, relay-control 0.4.22, relay-webapp 0.2.7, relayflows 0.2.5, durable-actors 0.14.6-awf.4, relayfile 0.2.2, telemetry-collector 0.1.7, relay-chief 0.1.0, slack-relay 0.3.11, relaygate 0.1.10) | relaycast: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
relaycast-postgres: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
sessions-pipeline-worker: `ghcr.io/agentworkforce/sessions-pipeline@sha256:77022c5b6a1d…`
relay-control: `ghcr.io/agentworkforce/relay-control@sha256:d1b6ca95fab2…`
relay-webapp: `ghcr.io/agentworkforce/relay-webapp@sha256:bd295ccb7d13…` | | | Carries 0.7.54 and sessions-pipeline 0.8.48: opt-in ingestEdge.admission.spread, ingestEdge.admission.spreadGraceMs and ingestEdge.livenessTimeoutSeconds. Default renders differ only in the helm.sh/chart labels. | | agent-relay-0.7.54 | **planned** | agent-relay 0.7.54
relay-control 0.4.22
relay-webapp 0.2.7
(deps: relaycast 0.3.16, relayauth 0.1.5, sessions-pipeline 0.8.47, relay-control 0.4.22, relay-webapp 0.2.7, relayflows 0.2.5, durable-actors 0.14.6-awf.4, relayfile 0.2.2, telemetry-collector 0.1.7, relay-chief 0.1.0, slack-relay 0.3.11, relaygate 0.1.10) | relaycast: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
relaycast-postgres: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
sessions-pipeline-worker: `ghcr.io/agentworkforce/sessions-pipeline@sha256:77022c5b6a1d…`
relay-control: `ghcr.io/agentworkforce/relay-control@sha256:d1b6ca95fab2…`
relay-webapp: `ghcr.io/agentworkforce/relay-webapp@sha256:bd295ccb7d13…` | | | Carries 0.7.53 and relaygate 0.1.10: the default policy gives inboxes no prefix covers (Desktop agents enrolled with a device token, named cursor-, claude-, codex-) the gid-11 inboxDefaults mirroring the local- rule, so local agents DM each other and the pit may read, never write. Needs relaygate with inboxDefaults (relaygate#43); older images ignore it. Still default-off (components.relaygate). Default renders differ in the relaygate policy ConfigMap and the helm.sh/chart labels. | @@ -1202,6 +1203,7 @@ chart `agent-relay` · values [`values-gcp-proof.yaml`](../charts/agent-relay/va | shipped | [Separate Control introspection credential Secret](#opt-relay-control-external-introspection-secret) | `disabled`, `enabled` | `disabled` | values `introspection.existingSecret`, `introspection.key`; env `RELAYCONTROL_INTROSPECT_SECRET_FILE`; Secret `` keys `introspect-secret` | | shipped | [Single-host mode for the dashboard and the control API](#opt-agent-relay-single-host) | `disabled`, `enabled` | `disabled` | values `global.singleHost.host`, `relay-control:global.singleHost.host`, `relay-control:publicUrl`, `relay-webapp:global.singleHost.host`, `relay-webapp:runtime.publicUrl`, `relay-webapp:ingress.host`; env `SELFHOST_PUBLIC_URL`, `DASHBOARD_PUBLIC_URL` | | shipped | [Single GKE ingress for the proof hosts](#opt-agent-relay-gke-ingress) | `enabled`, `disabled` | `disabled` | values `gkeIngress.enabled`, `gkeIngress.staticIpName`, `gkeIngress.managedCertificateName`, `gkeIngress.frontendConfigName`, `gkeIngress.hosts.*`, `gkeIngress.backends.*`, `relayauth.jwksIngress.enabled` | +| shipped | [GKE edge TLS policy, Cloud Armor and certificate from a Secret](#opt-agent-relay-gke-edge-hardening) | `none`, `sslPolicy`, `securityPolicy`, `tlsSecretName` | `none` | values `gkeIngress.sslPolicy`, `gkeIngress.securityPolicy`, `gkeIngress.tlsSecretName` | | shipped | [Optional Relayflows component in the suite](#opt-agent-relay-relayflows) | `enabled`, `disabled` | `disabled` | values `components.relayflows`, `relayflows.*` | | shipped | [Control legacy RelayAuth client environment](#opt-control-legacy-relayauth) | `true`, `false` | `true` | values `services.relayauthEnabled`; env `RELAYAUTH_URL`, `RELAYAUTH_API_KEY_FILE` | | shipped | [Native pipeline collector adapters](#opt-native-pipeline-analytics) | `false`, `true` | `false` | values `ingestEdge.analytics.*`, `nativeConsumer.analytics.*`, `session-read-api:analytics.*`; env `ANALYTICS_SINK`, `ANALYTICS_DEPLOYMENT_ID`, `ANALYTICS_COLLECTOR_URL`, `ANALYTICS_COLLECTOR_TOKEN`, `NODE_EXTRA_CA_CERTS`; Secret `` keys `token`; Secret `` keys `token`; Secret `` keys `token` | @@ -3534,6 +3536,39 @@ relaycast.service.backendConfig, so Relaycast WebSockets stay open. - Verify: render contains exactly one Ingress, the four expected hosts, ManagedCertificate, FrontendConfig, NEG annotations, the api BackendConfig and the /internal edge-deny on the api host - Sources: [`charts/agent-relay/templates/gke-ingress.yaml`](../charts/agent-relay/templates/gke-ingress.yaml), [`charts/agent-relay/values-gcp-proof.yaml`](../charts/agent-relay/values-gcp-proof.yaml), [`ci/agent-relay/render.sh`](../ci/agent-relay/render.sh) +#### opt.agent-relay.gke-edge-hardening + + + +**GKE edge TLS policy, Cloud Armor and certificate from a Secret** · shipped · chart `agent-relay` · profiles agent-relay + +Three optional gkeIngress settings, each empty by default and each +naming something that must exist before the upgrade. +gkeIngress.sslPolicy is a Compute SSL policy in the cluster's project; +the FrontendConfig applies it to the load balancer's HTTPS listener. +Without one Google's default policy applies, which still accepts TLS 1.0 +and 1.1; a policy with min TLS 1.2 (profile MODERN or RESTRICTED) is +recommended. gkeIngress.securityPolicy is a Cloud Armor backend security +policy; it is set on the BackendConfig of every backend behind the +Ingress, including policy-only BackendConfigs for the dashboard +(-dashboard), Relayfile (-files) and +the edge-deny Service. The relay-webapp and relayfile Services belong to +child charts, so their cloud.google.com/backend-config annotation must be +set in values; the render fails with the exact value when it is missing. +gkeIngress.tlsSecretName is a kubernetes.io/tls Secret served instead of +the ManagedCertificate, for an edge whose public DNS points at a proxy or +CDN in front of the load balancer, where Google cannot issue a managed +certificate. terraform/modules/gcp/edge-security creates the SSL policy +and the Cloud Armor policy (OWASP rules, sign-in rate limit, source-IP +allowlist, Cloudflare origin lockdown). + +- Default: `none` +- Choices: `none`: Google default TLS policy, no Cloud Armor, ManagedCertificate; `sslPolicy`: recommended: TLS 1.2 minimum; `securityPolicy`: Cloud Armor on every backend; `tlsSecretName`: certificate from a Secret instead of ManagedCertificate +- Provided by: operator +- Values: `gkeIngress.sslPolicy`, `gkeIngress.securityPolicy`, `gkeIngress.tlsSecretName` +- Verify: render with the values set shows spec.sslPolicy on the FrontendConfig, spec.securityPolicy.name on every BackendConfig, every Ingress backend Service bound to one of them, and spec.tls without a ManagedCertificate +- Sources: [`charts/agent-relay/templates/gke-ingress.yaml`](../charts/agent-relay/templates/gke-ingress.yaml), [`charts/agent-relay/values.yaml`](../charts/agent-relay/values.yaml), [`ci/agent-relay/render.sh`](../ci/agent-relay/render.sh), [`terraform/modules/gcp/edge-security/main.tf`](../terraform/modules/gcp/edge-security/main.tf) + #### opt.agent-relay.relayflows diff --git a/self-hosted/requirements.yaml b/self-hosted/requirements.yaml index 07205563..2d5bf01d 100644 --- a/self-hosted/requirements.yaml +++ b/self-hosted/requirements.yaml @@ -1956,9 +1956,22 @@ compatibility: images: relayauth: { repository: ghcr.io/agentworkforce/relayauth, digest: sha256:eb71daa28e30eb7305f666d0bccc5a4002e53e8a574e3c4c32163d907a1b71df } notes: superseded by 0.1.1. - - id: agent-relay-0.7.57 + - id: agent-relay-0.7.59 status: unreleased current: true + charts: { agent-relay: 0.7.59, relay-control: 0.4.22, relay-webapp: 0.2.7 } + app_versions: { agent-relay: 0.1.0, relay-control: 0.1.0, relay-webapp: 0.1.0 } + depends: + agent-relay: { relaycast: 0.3.16, relayauth: 0.1.5, sessions-pipeline: 0.8.49, relay-control: 0.4.22, relay-webapp: 0.2.7, relayflows: 0.2.5, durable-actors: 0.14.6-awf.4, relayfile: 0.2.2, telemetry-collector: 0.1.7, relay-chief: 0.1.0, slack-relay: 0.3.11, relaygate: 0.1.10 } + images: + relaycast: { repository: ghcr.io/agentworkforce/relaycast-self-host, digest: sha256:ec062ef901af15cf575b53a71d7eb09f2e6ab6e7c0d5be2da35339a275a5c6cc } + relaycast-postgres: { repository: ghcr.io/agentworkforce/relaycast-self-host, digest: sha256:ec062ef901af15cf575b53a71d7eb09f2e6ab6e7c0d5be2da35339a275a5c6cc } + sessions-pipeline-worker: { repository: ghcr.io/agentworkforce/sessions-pipeline, digest: sha256:77022c5b6a1d30f96f921d703eb27a1acada8ec273ad84aebdde24f8eddce305 } + relay-control: { repository: ghcr.io/agentworkforce/relay-control, digest: sha256:d1b6ca95fab240a9d41208d83de3b1f4245408b0765038762f1c419ee60f43bc } + relay-webapp: { repository: ghcr.io/agentworkforce/relay-webapp, digest: sha256:bd295ccb7d1323965dce890f19d0b7f857d706c00d7756f719c713763fdb49f3 } + notes: 'Carries 0.7.57 (0.7.58 is taken by another open change). GKE edge hardening, all opt-in and empty by default: gkeIngress.sslPolicy names a Compute SSL policy that the FrontendConfig applies to the HTTPS listener (without one Google''s default policy still accepts TLS 1.0 and 1.1); gkeIngress.securityPolicy names a Cloud Armor backend security policy set on the BackendConfig of every backend behind the Ingress (api, cast, both history edges, the actors and relaygate edges, and new policy-only BackendConfigs -dashboard, -edge-deny and -files; the relay-webapp and relayfile Services need their backend-config annotation and the render fails with the exact value otherwise); gkeIngress.tlsSecretName serves a kubernetes.io/tls Secret instead of the ManagedCertificate, for an edge whose DNS points at a proxy or CDN. The resources come from terraform/modules/gcp/edge-security; terraform/examples/gcp now creates the TLS 1.2 SSL policy by default and exports it in chart_values. Default renders differ only in the helm.sh/chart labels (subdomain golden unchanged); images unchanged.' + - id: agent-relay-0.7.57 + status: planned charts: { agent-relay: 0.7.57, relay-control: 0.4.22, relay-webapp: 0.2.7 } app_versions: { agent-relay: 0.1.0, relay-control: 0.1.0, relay-webapp: 0.1.0 } depends: @@ -5830,6 +5843,50 @@ entries: - "charts/agent-relay/values-gcp-proof.yaml" - "ci/agent-relay/render.sh" + - id: opt.agent-relay.gke-edge-hardening + kind: option + title: GKE edge TLS policy, Cloud Armor and certificate from a Secret + description: | + Three optional gkeIngress settings, each empty by default and each + naming something that must exist before the upgrade. + gkeIngress.sslPolicy is a Compute SSL policy in the cluster's project; + the FrontendConfig applies it to the load balancer's HTTPS listener. + Without one Google's default policy applies, which still accepts TLS 1.0 + and 1.1; a policy with min TLS 1.2 (profile MODERN or RESTRICTED) is + recommended. gkeIngress.securityPolicy is a Cloud Armor backend security + policy; it is set on the BackendConfig of every backend behind the + Ingress, including policy-only BackendConfigs for the dashboard + (-dashboard), Relayfile (-files) and + the edge-deny Service. The relay-webapp and relayfile Services belong to + child charts, so their cloud.google.com/backend-config annotation must be + set in values; the render fails with the exact value when it is missing. + gkeIngress.tlsSecretName is a kubernetes.io/tls Secret served instead of + the ManagedCertificate, for an edge whose public DNS points at a proxy or + CDN in front of the load balancer, where Google cannot issue a managed + certificate. terraform/modules/gcp/edge-security creates the SSL policy + and the Cloud Armor policy (OWASP rules, sign-in rate limit, source-IP + allowlist, Cloudflare origin lockdown). + chart: agent-relay + values: + - gkeIngress.sslPolicy + - gkeIngress.securityPolicy + - gkeIngress.tlsSecretName + profiles: [agent-relay] + status: shipped + provided_by: operator + default: none + choices: + - { value: none, note: "Google default TLS policy, no Cloud Armor, ManagedCertificate" } + - { value: sslPolicy, note: "recommended: TLS 1.2 minimum" } + - { value: securityPolicy, note: "Cloud Armor on every backend" } + - { value: tlsSecretName, note: "certificate from a Secret instead of ManagedCertificate" } + verify: render with the values set shows spec.sslPolicy on the FrontendConfig, spec.securityPolicy.name on every BackendConfig, every Ingress backend Service bound to one of them, and spec.tls without a ManagedCertificate + sources: + - "charts/agent-relay/templates/gke-ingress.yaml" + - "charts/agent-relay/values.yaml" + - "ci/agent-relay/render.sh" + - "terraform/modules/gcp/edge-security/main.tf" + - id: opt.agent-relay.relayflows kind: option title: Optional Relayflows component in the suite diff --git a/terraform/README.md b/terraform/README.md index 0679f5fd..62462191 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -15,6 +15,7 @@ terraform/ modules/aws/{foundation,postgres,object-storage,search,functions} modules/aws/relay-chief-slots # EC2 hosts for Relay Chief modules/gcp/{foundation,postgres,object-storage,search,functions} + modules/gcp/edge-security # GKE Ingress TLS policy and Cloud Armor examples/{aws,gcp} # thin roots: four capabilities together examples/{aws,gcp}-foundation # network and Kubernetes foundation roots examples/aws-environments # environment-scoped AWS data composition diff --git a/terraform/examples/gcp/README.md b/terraform/examples/gcp/README.md index b5b41e6c..eedfed0a 100644 --- a/terraform/examples/gcp/README.md +++ b/terraform/examples/gcp/README.md @@ -60,6 +60,7 @@ Terraform or OpenTofu >= 1.11 (the Cloud SQL admin password is write-only). | Name | Source | Version | | ---- | ------ | ------- | | desktop\_feed\_bucket | ../../modules/gcp/object-storage | n/a | +| edge\_security | ../../modules/gcp/edge-security | n/a | | functions | ../../modules/gcp/functions | n/a | | postgres | ../../modules/gcp/postgres | n/a | | search | ../../modules/gcp/search | n/a | @@ -98,6 +99,9 @@ Terraform or OpenTofu >= 1.11 (the Cloud SQL admin password is write-only). | deletion\_protection | Prevent deletion of Cloud SQL, Memorystore and Pub/Sub data resources. Disable only for a disposable proof environment. | `bool` | `true` | no | | desktop\_feed\_bucket\_name | Globally unique bucket for relay-control's signed Desktop feed (SELFHOST\_DESKTOP\_FEED\_GCS\_BUCKET). Empty disables it. | `string` | `""` | no | | desktop\_feed\_service\_account\_id | account\_id of the Google service account relay-control uses for the Desktop feed bucket (6-30 characters). | `string` | `"agent-relay-control"` | no | +| edge\_origin\_secret\_header | Shared-secret request header a fronting proxy adds; requests without one of the accepted values (1-3, for rotation) are denied (modules/gcp/edge-security origin\_secret\_header). Stored in state. | ```object({ name = string values = list(string) })``` | `null` | no | +| edge\_security\_policy | Cloud Armor policy for every GKE Ingress backend (modules/gcp/edge-security security\_policy: create, waf, rate\_limits, source\_ip\_allowlist, cloudflare, client\_ip\_header, client\_ip\_rules). Off unless create = true. It is exported as gkeIngress.securityPolicy in output.edge, not in chart\_values, because the chart then also needs the dashboard Service's backend-config annotation. | `any` | `{}` | no | +| edge\_ssl\_policy | Compute SSL policy for the GKE Ingress HTTPS listener, exported as gkeIngress.sslPolicy in chart\_values. create = false leaves Google's default policy (TLS 1.0 and later). | ```object({ create = optional(bool, true) profile = optional(string, "MODERN") min_tls_version = optional(string, "TLS_1_2") })``` | `{}` | no | | enable\_functions | Create the optional event-driven pipeline (Cloud Run functions). | `bool` | `false` | no | | enable\_search | Install the optional OpenSearch cluster on GKE. | `bool` | `false` | no | | gke\_cluster\_location | Its region or zone. | `string` | `null` | no | @@ -129,6 +133,7 @@ Terraform or OpenTofu >= 1.11 (the Cloud SQL admin password is write-only). | ---- | ----------- | | chart\_values | Complete non-secret managed-service overlay for values-gcp.yaml plus values-gcp-native.yaml. Save with: terraform output -json chart\_values > managed-values.json. | | desktop\_feed | Values for relay-control's signed Desktop feed on GCS (null unless desktop\_feed\_bucket\_name is set). Put env into relay-control.extraEnv and annotations into relay-control.serviceAccount.annotations; signing keys go through relay-control.desktopFeed.signingKeys. | +| edge | Edge policy names (edge.tf). security\_policy is null unless edge\_security\_policy.create is set; put it in gkeIngress.securityPolicy together with the backend-config annotation the chart asks for. client\_ip\_header set means a proxy fronts the load balancer: relay-control.auth.trustedProxies is 3 instead of 2. | | functions | Pipeline function, topics and secret (null unless enable\_functions). | | image\_mirror\_root | MIRROR\_ROOT for install guide step 5 (null unless image\_mirror\_repository is set). | | pipeline | Native Sessions pipeline values. The Redis CA is public certificate material, not a credential. | diff --git a/terraform/examples/gcp/edge.tf b/terraform/examples/gcp/edge.tf new file mode 100644 index 00000000..62ea6956 --- /dev/null +++ b/terraform/examples/gcp/edge.tf @@ -0,0 +1,21 @@ +# Edge controls for the chart's GKE Ingress (modules/gcp/edge-security). +# +# The SSL policy (MODERN, TLS 1.2 minimum) is created by default and reaches +# the chart through chart_values (gkeIngress.sslPolicy): without one, the load +# balancer uses Google's default policy, which still accepts TLS 1.0 and 1.1. +# It is a free global resource and changes nothing until a Helm upgrade +# applies the regenerated managed-values.json; edge_ssl_policy.create = false +# opts out. +# +# The Cloud Armor policy is off by default (edge_security_policy.create). + +module "edge_security" { + source = "../../modules/gcp/edge-security" + + project_id = var.project_id + name = var.name + labels = var.labels + ssl_policy = var.edge_ssl_policy + security_policy = var.edge_security_policy + origin_secret_header = var.edge_origin_secret_header +} diff --git a/terraform/examples/gcp/outputs.tf b/terraform/examples/gcp/outputs.tf index 212cd12c..e2390480 100644 --- a/terraform/examples/gcp/outputs.tf +++ b/terraform/examples/gcp/outputs.tf @@ -104,6 +104,20 @@ output "chart_values" { key = "ca.crt" } } + # The TLS policy of the load balancer's HTTPS listener (edge.tf). Empty + # with edge_ssl_policy.create = false, which is the chart's default. + gkeIngress = { + sslPolicy = module.edge_security.ssl_policy_name != null ? module.edge_security.ssl_policy_name : "" + } + } +} + +output "edge" { + description = "Edge policy names (edge.tf). security_policy is null unless edge_security_policy.create is set; put it in gkeIngress.securityPolicy together with the backend-config annotation the chart asks for. client_ip_header set means a proxy fronts the load balancer: relay-control.auth.trustedProxies is 3 instead of 2." + value = { + ssl_policy = module.edge_security.ssl_policy_name + security_policy = module.edge_security.security_policy_name + client_ip_header = module.edge_security.client_ip_header } } diff --git a/terraform/examples/gcp/terraform.tfvars.example b/terraform/examples/gcp/terraform.tfvars.example index 8925aa32..f16574e4 100644 --- a/terraform/examples/gcp/terraform.tfvars.example +++ b/terraform/examples/gcp/terraform.tfvars.example @@ -19,6 +19,12 @@ deletion_protection = true # vpc_connector = "projects/my-project/locations/us-central1/connectors/my-connector" # pipeline_source_archive = { bucket = "my-artifacts", object = "session-pipeline.zip" } +# The load balancer's TLS policy (MODERN, TLS 1.2 minimum) is created by +# default. A Cloud Armor policy for every backend is opt-in: +# edge_security_policy = { create = true } # OWASP rules (preview) + sign-in rate limit +# edge_security_policy = { create = true, source_ip_allowlist = ["198.51.100.0/24"] } # only these addresses +# edge_security_policy = { create = true, cloudflare = { enabled = true } } # only Cloudflare may connect + labels = { app = "agent-relay" } diff --git a/terraform/examples/gcp/tests/native.tftest.hcl b/terraform/examples/gcp/tests/native.tftest.hcl index d7546525..90c000a1 100644 --- a/terraform/examples/gcp/tests/native.tftest.hcl +++ b/terraform/examples/gcp/tests/native.tftest.hcl @@ -160,3 +160,41 @@ run "desktop_feed_bucket_and_identity" { error_message = "Enabling the feed must keep relay-control's Cloud SQL CA values." } } + +run "edge_tls_policy_is_on_by_default" { + command = apply + + assert { + condition = output.chart_values.gkeIngress.sslPolicy == "agent-relay-tls" && output.edge.ssl_policy == "agent-relay-tls" + error_message = "managed-values.json must name the TLS 1.2 SSL policy for the GKE Ingress." + } + assert { + condition = output.edge.security_policy == null && output.edge.client_ip_header == null + error_message = "Cloud Armor is off by default." + } +} + +run "edge_tls_policy_opt_out" { + command = apply + variables { + edge_ssl_policy = { create = false } + } + + assert { + condition = output.chart_values.gkeIngress.sslPolicy == "" && output.edge.ssl_policy == null + error_message = "Opting out must leave the chart default (no sslPolicy)." + } +} + +run "edge_cloud_armor_behind_cloudflare" { + command = apply + variables { + edge_security_policy = { create = true, cloudflare = { enabled = true } } + edge_origin_secret_header = { name = "x-origin-auth", values = ["0123456789abcdef0123456789abcdef"] } + } + + assert { + condition = output.edge.security_policy == "agent-relay-edge" && output.edge.client_ip_header == "CF-Connecting-IP" + error_message = "The Cloud Armor policy and the client-address header must be reported." + } +} diff --git a/terraform/examples/gcp/variables.tf b/terraform/examples/gcp/variables.tf index 94d25d26..071dd526 100644 --- a/terraform/examples/gcp/variables.tf +++ b/terraform/examples/gcp/variables.tf @@ -209,3 +209,29 @@ variable "relay_control_service_account" { type = string default = "relay-control" } + +variable "edge_ssl_policy" { + description = "Compute SSL policy for the GKE Ingress HTTPS listener, exported as gkeIngress.sslPolicy in chart_values. create = false leaves Google's default policy (TLS 1.0 and later)." + type = object({ + create = optional(bool, true) + profile = optional(string, "MODERN") + min_tls_version = optional(string, "TLS_1_2") + }) + default = {} +} + +variable "edge_security_policy" { + description = "Cloud Armor policy for every GKE Ingress backend (modules/gcp/edge-security security_policy: create, waf, rate_limits, source_ip_allowlist, cloudflare, client_ip_header, client_ip_rules). Off unless create = true. It is exported as gkeIngress.securityPolicy in output.edge, not in chart_values, because the chart then also needs the dashboard Service's backend-config annotation." + type = any + default = {} +} + +variable "edge_origin_secret_header" { + description = "Shared-secret request header a fronting proxy adds; requests without one of the accepted values (1-3, for rotation) are denied (modules/gcp/edge-security origin_secret_header). Stored in state." + type = object({ + name = string + values = list(string) + }) + default = null + sensitive = true +} diff --git a/terraform/modules/gcp/edge-security/README.md b/terraform/modules/gcp/edge-security/README.md new file mode 100644 index 00000000..87f6eee7 --- /dev/null +++ b/terraform/modules/gcp/edge-security/README.md @@ -0,0 +1,164 @@ +# gcp/edge-security + +Edge controls for the GKE Ingress of the `agent-relay` chart. Both resources are global Compute resources that the chart refers to by name; GKE attaches them to the load balancer. + +| Resource | Chart value | Default | +| --- | --- | --- | +| Compute SSL policy `-tls` (`MODERN`, TLS 1.2 minimum) | `gkeIngress.sslPolicy` | created | +| Cloud Armor backend security policy `-edge` | `gkeIngress.securityPolicy` | not created (`security_policy.create = true`) | + +Without an SSL policy a Google load balancer uses its default policy, which still accepts TLS 1.0 and 1.1. + +`labels` reach the Cloud Armor policy (plus `managed-by = terraform`). Compute SSL policies and security policy rules have no labels in the API. + +## Cloud Armor policy + +Rules are evaluated in priority order and the first match decides. A rule that admits a request ends evaluation, so the order below matters: the secret header and the WAF rules can only deny, and come first. + +| Priority | Rule | Action | +| --- | --- | --- | +| 100 | `origin_secret_header`: the shared-secret header is missing or wrong | deny 403 | +| 1000+ | `waf`: OWASP preconfigured rules (SQLi, XSS, LFI, RFI, RCE, scanner detection, protocol attack, session fixation, CVE canary) at sensitivity 1 | deny 403, **preview by default** | +| 2000+ | `client_ip_rules`: on the listed paths, admit the listed client addresses (with the rule's own rate limit) and deny everyone else | allow or throttle, then deny 403 | +| 3000+ | `rate_limits`: per-client throttle on path regexes (default: sign-in and password paths, 60 requests a minute) | 429 beyond the limit | +| 9000+ | the allowlist of connecting addresses (`source_ip_allowlist`, Cloudflare's ranges) | allow | +| last | default | allow, or **deny 403 when an allowlist is set** | + +**The WAF rules start in preview.** Session transcripts, messages and flows carry code, SQL and shell text, which is what the OWASP signatures look for. Preview logs each match without blocking. Read the load balancer's request log (`jsonPayload.previewSecurityPolicy`) for a week of normal use, then set `waf.preview = false`, or remove the rule names that only match your own traffic from `waf.rules`. + +A throttle rule admits the requests under its limit, so when an allowlist is set each throttle rule also requires an allowlisted connecting address. Cloud Armor expressions take five subexpressions, which is why an allowlist appears there in groups of four. + +### Only some addresses may connect + +```hcl +security_policy = { + create = true + source_ip_allowlist = ["198.51.100.0/24"] # for example your VPN's egress range +} +``` + +Add `waf = { enabled = false }` and `rate_limits = []` for a policy that is the allowlist and nothing else. + +### Behind Cloudflare + +```hcl +security_policy = { + create = true + cloudflare = { enabled = true } +} +origin_secret_header = { name = "x-origin-auth", values = [var.origin_secret] } # openssl rand -hex 32 +``` + +- `cloudflare.enabled` admits only [Cloudflare's published ranges](https://www.cloudflare.com/ips/) as connecting addresses (the IPv4 list as read on 2026-11-18; `include_ipv6` adds the IPv6 list, which a load balancer with an IPv4 address never sees). Cloudflare announces changes to the list: override `cloudflare.ipv4_cidrs` when it changes. +- Cloudflare's ranges are shared by every Cloudflare customer, so the allowlist alone does not prove a request came through **your** zone. `origin_secret_header` does: add the header in a Cloudflare request header transform rule, and Cloud Armor denies any request without it. `values` takes up to three accepted values, so rotation has no outage: add the new value and apply, switch the Cloudflare rule to it, then remove the old one and apply. The values are kept in Terraform state and are readable in the policy by anyone with `compute.securityPolicies.get`. +- The client address is read from `CF-Connecting-IP` (Cloud Armor `origin.user_ip`), so rate limits count per client and not per Cloudflare address. +- `client_ip_rules` decide which clients may reach which paths, by the client address in `CF-Connecting-IP`. They trust that header, so they require `origin_secret_header`. See the next section. + +- There is now one more proxy in front of the application, so `relay-control.auth.trustedProxies` goes up by one: 3 behind Cloudflare (2 was measured for the Google load balancer alone; 3 is expected and not yet measured). `docs/gcp-install/3-EDGE-HARDENING.md` shows how to check it. With the wrong number, sign-in rate limits and lockout are shared by every user. + +### Which clients may reach which paths + +`client_ip_rules` are allowlists of client addresses, each scoped to path prefixes and optionally to hosts. They are evaluated in list order, and **the first rule whose paths (and hosts) match a request decides it**: admitted if the client address is in `cidrs`, denied 403 otherwise. A request that matches no rule falls through to the rules below. + +A Cloud Armor rule that admits a request ends evaluation. So a request admitted by a client rule meets no later rule: not a later client rule, not `rate_limits`, not the allowlist of connecting addresses (the secret header rule, which runs first, has already proved it came through the proxy). Two consequences, both enforced by the module: + +- **Every rule states its own limit**: `rate_limit = { count, interval_sec }` (per client address, 429 beyond it) or `unthrottled = true`. +- **A rule for every path (`["/"]` without `hosts`) makes `rate_limits` unreachable**, so set `rate_limits = []` and express the sign-in limit as a narrower rule ahead of it, as below. + +This example lets two vendors' published ranges reach only the session upload paths on the history host and the hosted MCP paths on the API host, limits sign-in attempts, lets the organisation's own ranges reach everything, and closes the rest. Check the path lists against the release you run: + +```hcl +security_policy = { + create = true + cloudflare = { enabled = true } + rate_limits = [] + client_ip_rules = [ + { + name = "vendor-uploads" # session uploads, on the history host + hosts = ["history.relay.example.com"] + path_prefixes = ["/v1/delivery", "/v1/turns", "/v1/auth/token/refresh"] + cidrs = ["192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24"] # vendor ranges, then your own + rate_limit = { count = 600, interval_sec = 60 } + }, + { + name = "vendor-mcp" # hosted MCP and its OAuth endpoints, on the API host + hosts = ["api.relay.example.com"] + path_prefixes = ["/cloud/api/v1/mcp", "/mcp", "/cloud/mcp", "/.well-known"] + cidrs = ["192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24"] + rate_limit = { count = 600, interval_sec = 60 } + }, + { + name = "sign-in" + path_prefixes = ["/login", "/api/v1/auth", "/cloud/api/v1/auth"] + cidrs = ["203.0.113.0/24"] + rate_limit = { count = 60, interval_sec = 60 } + }, + { + name = "staff" + path_prefixes = ["/"] # every remaining path, on every host + cidrs = ["203.0.113.0/24"] + unthrottled = true + }, + ] +} +origin_secret_header = { name = "x-origin-auth", values = [var.origin_secret] } +``` + +- Put narrower rules first. Repeat your own ranges (for example your VPN's egress range) in a vendor rule: your users' desktop uploads and MCP clients use those paths too, and a rule denies everyone it does not list. +- Without the closing `["/"]` rule, paths that no rule names stay open to every client that comes through the proxy (and `rate_limits` still apply to them). +- Each rule costs one advanced rule per four client ranges (three with `hosts`) plus one. A vendor that publishes hundreds of ranges will exceed the project's Cloud Armor quota for advanced rules; ask Google for a higher quota, or enforce that list at the proxy instead (Cloudflare WAF custom rules see the same client address and have no such limit). The limit per rule here is 294 ranges. +- No rule in this module can contain `0.0.0.0/0` or `::/0`: validation refuses them in every list. The only "all addresses" match is Cloud Armor's mandatory default rule, which is deny whenever an allowlist is set. + +## Chart side + +`gkeIngress.securityPolicy` puts the policy on the BackendConfig of every backend Service. The dashboard Service (and Relayfile with `gkeIngress.hosts.files`) needs its `cloud.google.com/backend-config` annotation; `helm template` fails with the exact value until it is set. See the chart README, "GKE edge hardening". + +Advanced rules (`expr`) count against the project's Cloud Armor quotas: the defaults use three, and seven with `cloudflare.enabled`. + + +### Requirements + +| Name | Version | +| ---- | ------- | +| terraform | >= 1.6 | +| google | >= 7.0, < 8.0 | + +### Providers + +| Name | Version | +| ---- | ------- | +| google | >= 7.0, < 8.0 | + +### Resources + +| Name | Type | +| ---- | ---- | +| [google_compute_security_policy.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_security_policy) | resource | +| [google_compute_security_policy_rule.allowlist](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_security_policy_rule) | resource | +| [google_compute_security_policy_rule.client_ip](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_security_policy_rule) | resource | +| [google_compute_security_policy_rule.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_security_policy_rule) | resource | +| [google_compute_security_policy_rule.origin_secret](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_security_policy_rule) | resource | +| [google_compute_security_policy_rule.rate_limit](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_security_policy_rule) | resource | +| [google_compute_security_policy_rule.waf](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_security_policy_rule) | resource | +| [google_compute_ssl_policy.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_ssl_policy) | resource | + +### Inputs + +| Name | Description | Type | Default | Required | +| ---- | ----------- | ---- | ------- | :------: | +| name | Name prefix: the SSL policy is -tls and the Cloud Armor policy -edge. | `string` | n/a | yes | +| project\_id | Project of the GKE cluster. GKE Ingress resolves gkeIngress.sslPolicy and gkeIngress.securityPolicy by name in this project. | `string` | n/a | yes | +| labels | Labels for every resource here that takes labels (the Cloud Armor policy; Compute SSL policies have none in the API). managed-by=terraform is always added. | `map(string)` | `{}` | no | +| origin\_secret\_header | Optional shared-secret request header a fronting proxy adds on every request (for Cloudflare: a request header transform rule). Requests without the exact value are denied 403 before any other rule, which proves the request came through the proxy and makes its client-address header trustworthy. values lists every accepted value (1 to 3), so a rotation has no outage: add the new value here and apply, switch the proxy to it, then remove the old value and apply. The values are stored in Terraform state and readable in the Cloud Armor policy by anyone with compute.securityPolicies.get: treat both as secret stores. | ```object({ name = string values = list(string) })``` | `null` | no | +| security\_policy | Cloud Armor backend security policy (chart value gkeIngress.securityPolicy), off unless create = true. - waf: preconfigured OWASP rules (deny 403) at the given sensitivity (1 = fewest false positives). preview = true (the default) only logs matches: session transcripts and messages carry code, SQL and shell text, so read the logs before enforcing. - rate\_limits: per-client throttle (429) on the listed path regexes; the default covers password sign-in, setup-key and login paths. - source\_ip\_allowlist: when non-empty (or cloudflare.enabled), the default rule becomes deny 403 and only these connecting addresses are admitted: a pure source-IP allowlist. - cloudflare: admit Cloudflare's published ranges as the connecting addresses (origin lockdown) and read the client address from CF-Connecting-IP. - client\_ip\_header: header an upstream proxy sets with the real client address (Cloud Armor origin.user\_ip). Only trustworthy when nothing but that proxy can reach the load balancer: combine with an allowlist of the proxy's ranges, or origin\_secret\_header. - client\_ip\_rules: per-client allowlists scoped to paths (and optionally hosts), evaluated in list order. The first rule whose path\_prefixes (and hosts, when given) match a request decides it: admitted if the client address (origin.user\_ip) is in cidrs, denied 403 otherwise. A last rule with path\_prefixes = ["/"] therefore closes every other path to everyone but its cidrs. Requires origin\_secret\_header. An admitted request meets no later rule, including rate\_limits, so every rule states its own limit: rate\_limit (requests per interval per client address), or unthrottled = true. With a rule for "/" set rate\_limits = [] and express sign-in limits as an earlier, narrower rule. - adaptive\_protection: Layer 7 DDoS detection (alerts; automatic deployment needs Cloud Armor Enterprise). | ```object({ create = optional(bool, false) waf = optional(object({ enabled = optional(bool, true) preview = optional(bool, true) sensitivity = optional(number, 1) rules = optional(list(string), [ "sqli-v33-stable", "xss-v33-stable", "lfi-v33-stable", "rfi-v33-stable", "rce-v33-stable", "scannerdetection-v33-stable", "protocolattack-v33-stable", "sessionfixation-v33-stable", "cve-canary", ]) }), {}) rate_limits = optional(list(object({ name = string path_regex = string count = number interval_sec = number ban_duration_sec = optional(number, 0) })), [{ name = "sign-in" path_regex = "^/(login|(cloud/)?api/v1/auth/)" count = 60 interval_sec = 60 }]) source_ip_allowlist = optional(list(string), []) cloudflare = optional(object({ enabled = optional(bool, false) include_ipv6 = optional(bool, false) # https://www.cloudflare.com/ips/ as read on 2026-11-18. Cloudflare # announces changes; compare the list with that page before each apply. ipv4_cidrs = optional(list(string), [ "173.245.48.0/20", "103.21.244.0/22", "103.22.200.0/22", "103.31.4.0/22", "141.101.64.0/18", "108.162.192.0/18", "190.93.240.0/20", "188.114.96.0/20", "197.234.240.0/22", "198.41.128.0/17", "162.158.0.0/15", "104.16.0.0/13", "104.24.0.0/14", "172.64.0.0/13", "131.0.72.0/22", ]) ipv6_cidrs = optional(list(string), [ "2400:cb00::/32", "2606:4700::/32", "2803:f800::/32", "2405:b500::/32", "2405:8100::/32", "2a06:98c0::/29", "2c0f:f248::/32", ]) }), {}) client_ip_header = optional(string) client_ip_rules = optional(list(object({ name = string path_prefixes = list(string) hosts = optional(list(string), []) cidrs = list(string) rate_limit = optional(object({ count = number, interval_sec = number })) unthrottled = optional(bool, false) })), []) adaptive_protection = optional(bool, false) })``` | `{}` | no | +| ssl\_policy | Compute SSL policy for the load balancer's HTTPS listener (chart value gkeIngress.sslPolicy). MODERN with TLS 1.2 serves current browsers, the desktop app and CLIs; RESTRICTED narrows the cipher list further (TLS 1.2 minimum is required by it). | ```object({ create = optional(bool, true) profile = optional(string, "MODERN") min_tls_version = optional(string, "TLS_1_2") })``` | `{}` | no | + +### Outputs + +| Name | Description | +| ---- | ----------- | +| chart\_values | The gkeIngress keys to merge into the agent-relay values. With securityPolicy, also set the backend-config annotation the chart names on the relay-webapp (and relayfile) Service. | +| client\_ip\_header | Header Cloud Armor reads the client address from (null: the connecting address). When set there is one more proxy in front of the load balancer: relay-control.auth.trustedProxies is one more than for the Google load balancer alone (2), so 3 (docs/gcp-install/3-EDGE-HARDENING.md). | +| security\_policy\_name | Chart value gkeIngress.securityPolicy. Null when security\_policy.create is false. | +| ssl\_policy\_name | Chart value gkeIngress.sslPolicy. Null when ssl\_policy.create is false. | + diff --git a/terraform/modules/gcp/edge-security/main.tf b/terraform/modules/gcp/edge-security/main.tf new file mode 100644 index 00000000..3d885bc1 --- /dev/null +++ b/terraform/modules/gcp/edge-security/main.tf @@ -0,0 +1,222 @@ +# Edge controls for the GKE Ingress of the agent-relay chart: the TLS policy of +# the HTTPS listener and an optional Cloud Armor policy for every backend. Both +# are global Compute resources the chart refers to by name +# (gkeIngress.sslPolicy, gkeIngress.securityPolicy); GKE attaches them. + +locals { + labels = merge(var.labels, { managed-by = "terraform" }) + sp = var.security_policy + armor = local.sp.create + # Whether the header is configured is not secret; its value is. + secret_header = nonsensitive(var.origin_secret_header != null) + + # Addresses allowed to connect to the load balancer. Any entry turns the + # default rule into deny. + cloudflare_cidrs = local.sp.cloudflare.enabled ? concat(local.sp.cloudflare.ipv4_cidrs, local.sp.cloudflare.include_ipv6 ? local.sp.cloudflare.ipv6_cidrs : []) : [] + allowlist = distinct(concat(local.sp.source_ip_allowlist, local.cloudflare_cidrs)) + locked = length(local.allowlist) > 0 + # Basic rules take 10 ranges each; expressions take five subexpressions, so + # an allowlist conjoined with a path match is cut into groups of four. + allow_rules = chunklist(local.allowlist, 10) + allow_groups = chunklist(local.allowlist, 4) + + client_ip_header = local.sp.client_ip_header != null ? local.sp.client_ip_header : (local.sp.cloudflare.enabled ? "CF-Connecting-IP" : null) + + waf_groups = local.sp.waf.enabled ? chunklist(local.sp.waf.rules, 5) : [] + + # A backslash in path_regex is doubled: the pattern sits inside a quoted + # string of the expression, whose own reader consumes one level of escaping. + # One throttle rule per rate limit, or one per allowlist group when the + # edge is locked: a throttle rule admits conforming requests, so it must + # itself require an allowed connecting address. + rate_rules = merge([for i, r in local.sp.rate_limits : { + for j, group in(local.locked ? local.allow_groups : [[]]) : "${r.name}-${j}" => { + priority = 3000 + i * 100 + j + limit = r + expression = join(" && ", compact(["request.path.matches('${replace(r.path_regex, "\\", "\\\\")}')", length(group) > 0 ? "(${join(" || ", [for c in group : "inIpRange(origin.ip, '${c}')"])})" : ""])) + } + }]...) + + # client_ip_rules: the first rule whose paths (and hosts) match decides. + # Its clients are admitted, by a throttle when the rule has a rate_limit, + # and everyone else on those paths is denied. A Cloud Armor rule that admits + # ends evaluation, so admitted requests meet no later rule: that is why each + # rule states its own limit and why the secret header rule must come first. + client_groups = { for r in local.sp.client_ip_rules : r.name => chunklist(r.cidrs, length(r.hosts) > 0 ? 3 : 4) } + client_rules = merge([for i, r in local.sp.client_ip_rules : merge({ + for j, group in local.client_groups[r.name] : "${r.name}-allow-${j}" => { + priority = 2000 + i * 100 + j + action = r.rate_limit == null ? "allow" : "throttle" + rate = { count = try(r.rate_limit.count, 0), interval_sec = try(r.rate_limit.interval_sec, 0) } + description = "${r.name}: listed client addresses on ${join(", ", concat(r.hosts, r.path_prefixes))}" + expression = "${local.path_match[r.name]} && (${join(" || ", [for c in group : "inIpRange(origin.user_ip, '${c}')"])})" + } + }, { "${r.name}-deny" = { + priority = 2000 + i * 100 + 99 + action = "deny(403)" + rate = { count = 0, interval_sec = 0 } + description = "${r.name}: any other client on ${join(", ", concat(r.hosts, r.path_prefixes))}" + expression = local.path_match[r.name] + } })]...) + # The path (one subexpression) and, with hosts, the Host header (a second), + # which leaves three or four subexpressions for client ranges. + path_match = { for r in local.sp.client_ip_rules : r.name => join(" && ", compact([ + length(r.hosts) > 0 ? "request.headers['host'].lower().matches('^(${join("|", [for h in r.hosts : replace(h, ".", "\\\\.")])})(:[0-9]+)?$')" : "", + "request.path.matches('^(${join("|", [for p in r.path_prefixes : replace(p, ".", "\\\\.")])})')", + ])) } +} + +resource "google_compute_ssl_policy" "this" { + #checkov:skip=CKV_GCP_4:The scanner cannot resolve the profile and minimum version from variables. var.ssl_policy only validates MODERN or RESTRICTED with TLS 1.2 or 1.3, and the module tests assert the applied values. + count = var.ssl_policy.create ? 1 : 0 + project = var.project_id + name = "${var.name}-tls" + description = "agent-relay GKE Ingress: ${var.ssl_policy.profile}, minimum ${var.ssl_policy.min_tls_version}" + profile = var.ssl_policy.profile + min_tls_version = var.ssl_policy.min_tls_version +} + +resource "google_compute_security_policy" "this" { + #checkov:skip=CKV_GCP_73:The check looks for an inline cve-canary rule. Rules here are google_compute_security_policy_rule resources, and the default waf.rules include cve-canary (asserted in the module tests). + count = local.armor ? 1 : 0 + project = var.project_id + name = "${var.name}-edge" + description = "agent-relay GKE Ingress backends" + type = "CLOUD_ARMOR" + labels = local.labels + + advanced_options_config { + json_parsing = "STANDARD" + log_level = "NORMAL" + user_ip_request_headers = local.client_ip_header != null ? [local.client_ip_header] : [] + } + dynamic "adaptive_protection_config" { + for_each = local.sp.adaptive_protection ? [1] : [] + content { + layer_7_ddos_defense_config { enable = true } + } + } + + lifecycle { + precondition { + condition = length(local.sp.client_ip_rules) == 0 || (local.secret_header && local.client_ip_header != null) + error_message = "client_ip_rules trust a client-address header, so they need origin_secret_header (proof the request came through the proxy) and client_ip_header or cloudflare.enabled." + } + precondition { + condition = alltrue([for r in local.sp.client_ip_rules : r.rate_limit != null || r.unthrottled]) + error_message = "Each client_ip_rules entry admits its clients, and admitted requests skip rate_limits. Give the rule its own rate_limit, or set unthrottled = true to accept that it has none." + } + precondition { + condition = length(local.sp.rate_limits) == 0 || !anytrue([for r in local.sp.client_ip_rules : length(r.hosts) == 0 && contains(r.path_prefixes, "/")]) + error_message = "A client_ip_rules entry for every path (\"/\" without hosts) decides every request, so rate_limits would never apply. Set rate_limits = [] and put the limits you want in client_ip_rules, for example a sign-in rule ahead of the closing rule." + } + precondition { + condition = local.client_ip_header == null || local.locked || local.secret_header + error_message = "client_ip_header is only trustworthy when direct connections are refused: set cloudflare.enabled, source_ip_allowlist (the proxy's ranges) or origin_secret_header." + } + } +} + +resource "google_compute_security_policy_rule" "origin_secret" { + count = local.armor && local.secret_header ? 1 : 0 + project = var.project_id + security_policy = google_compute_security_policy.this[0].name + priority = 100 + action = "deny(403)" + description = "Requests without the fronting proxy's shared-secret header" + match { + expr { + # One subexpression for the missing header and one per accepted value. + expression = "!has(request.headers['${var.origin_secret_header.name}']) || !(${join(" || ", [for v in var.origin_secret_header.values : "request.headers['${var.origin_secret_header.name}'] == '${v}'"])})" + } + } +} + +resource "google_compute_security_policy_rule" "waf" { + for_each = local.armor ? { for i, group in local.waf_groups : tostring(i) => group } : {} + project = var.project_id + security_policy = google_compute_security_policy.this[0].name + priority = 1000 + tonumber(each.key) + action = "deny(403)" + preview = local.sp.waf.preview + description = "OWASP preconfigured rules, sensitivity ${local.sp.waf.sensitivity}: ${join(", ", each.value)}" + match { + expr { + expression = join(" || ", [for r in each.value : "evaluatePreconfiguredWaf('${r}', {'sensitivity': ${local.sp.waf.sensitivity}})"]) + } + } +} + +resource "google_compute_security_policy_rule" "client_ip" { + for_each = local.armor ? local.client_rules : {} + project = var.project_id + security_policy = google_compute_security_policy.this[0].name + priority = each.value.priority + action = each.value.action + description = each.value.description + match { + expr { expression = each.value.expression } + } + dynamic "rate_limit_options" { + for_each = each.value.rate.count > 0 ? [each.value.rate] : [] + content { + conform_action = "allow" + exceed_action = "deny(429)" + enforce_on_key = "USER_IP" + rate_limit_threshold { + count = rate_limit_options.value.count + interval_sec = rate_limit_options.value.interval_sec + } + } + } + depends_on = [google_compute_security_policy_rule.origin_secret] +} + +resource "google_compute_security_policy_rule" "rate_limit" { + for_each = local.armor ? local.rate_rules : {} + project = var.project_id + security_policy = google_compute_security_policy.this[0].name + priority = each.value.priority + action = each.value.limit.ban_duration_sec > 0 ? "rate_based_ban" : "throttle" + description = "Per-client rate limit ${each.value.limit.name}: ${each.value.limit.count} requests per ${each.value.limit.interval_sec}s" + match { + expr { expression = each.value.expression } + } + rate_limit_options { + conform_action = "allow" + exceed_action = "deny(429)" + enforce_on_key = local.client_ip_header != null ? "USER_IP" : "IP" + ban_duration_sec = each.value.limit.ban_duration_sec > 0 ? each.value.limit.ban_duration_sec : null + rate_limit_threshold { + count = each.value.limit.count + interval_sec = each.value.limit.interval_sec + } + } +} + +resource "google_compute_security_policy_rule" "allowlist" { + for_each = local.armor ? { for i, group in local.allow_rules : tostring(i) => group } : {} + project = var.project_id + security_policy = google_compute_security_policy.this[0].name + priority = 9000 + tonumber(each.key) + action = "allow" + description = "Allowed connecting addresses (${local.sp.cloudflare.enabled ? "Cloudflare and " : ""}source_ip_allowlist)" + match { + versioned_expr = "SRC_IPS_V1" + config { src_ip_ranges = each.value } + } +} + +# The rule Cloud Armor evaluates last. Deny when an allowlist is in force. +resource "google_compute_security_policy_rule" "default" { + count = local.armor ? 1 : 0 + project = var.project_id + security_policy = google_compute_security_policy.this[0].name + priority = 2147483647 + action = local.locked ? "deny(403)" : "allow" + description = local.locked ? "Default deny: only allowlisted connecting addresses are admitted" : "Default allow" + match { + versioned_expr = "SRC_IPS_V1" + config { src_ip_ranges = ["*"] } + } +} diff --git a/terraform/modules/gcp/edge-security/outputs.tf b/terraform/modules/gcp/edge-security/outputs.tf new file mode 100644 index 00000000..d58c3d27 --- /dev/null +++ b/terraform/modules/gcp/edge-security/outputs.tf @@ -0,0 +1,22 @@ +output "ssl_policy_name" { + description = "Chart value gkeIngress.sslPolicy. Null when ssl_policy.create is false." + value = try(google_compute_ssl_policy.this[0].name, null) +} + +output "security_policy_name" { + description = "Chart value gkeIngress.securityPolicy. Null when security_policy.create is false." + value = try(google_compute_security_policy.this[0].name, null) +} + +output "chart_values" { + description = "The gkeIngress keys to merge into the agent-relay values. With securityPolicy, also set the backend-config annotation the chart names on the relay-webapp (and relayfile) Service." + value = { gkeIngress = merge( + var.ssl_policy.create ? { sslPolicy = google_compute_ssl_policy.this[0].name } : {}, + local.armor ? { securityPolicy = google_compute_security_policy.this[0].name } : {}, + ) } +} + +output "client_ip_header" { + description = "Header Cloud Armor reads the client address from (null: the connecting address). When set there is one more proxy in front of the load balancer: relay-control.auth.trustedProxies is one more than for the Google load balancer alone (2), so 3 (docs/gcp-install/3-EDGE-HARDENING.md)." + value = local.armor ? local.client_ip_header : null +} diff --git a/terraform/modules/gcp/edge-security/tests/edge_security.tftest.hcl b/terraform/modules/gcp/edge-security/tests/edge_security.tftest.hcl new file mode 100644 index 00000000..ad09f92e --- /dev/null +++ b/terraform/modules/gcp/edge-security/tests/edge_security.tftest.hcl @@ -0,0 +1,335 @@ +mock_provider "google" {} + +variables { + project_id = "proj" + name = "relay" +} + +run "defaults_create_only_the_tls_policy" { + command = apply + + assert { + condition = google_compute_ssl_policy.this[0].name == "relay-tls" && google_compute_ssl_policy.this[0].profile == "MODERN" && google_compute_ssl_policy.this[0].min_tls_version == "TLS_1_2" + error_message = "The default SSL policy is MODERN with a TLS 1.2 minimum." + } + assert { + condition = length(google_compute_security_policy.this) == 0 && length(google_compute_security_policy_rule.default) == 0 && length(google_compute_security_policy_rule.waf) == 0 + error_message = "Cloud Armor is off by default." + } + assert { + condition = length(output.chart_values.gkeIngress) == 1 && output.chart_values.gkeIngress.sslPolicy == "relay-tls" && output.security_policy_name == null && output.client_ip_header == null + error_message = "chart_values carries only the SSL policy by default." + } +} + +run "tls_1_0_is_refused" { + command = plan + variables { + ssl_policy = { profile = "COMPATIBLE", min_tls_version = "TLS_1_0" } + } + expect_failures = [var.ssl_policy] +} + +run "tls_1_3_needs_restricted" { + command = plan + variables { + ssl_policy = { min_tls_version = "TLS_1_3" } + } + expect_failures = [var.ssl_policy] +} + +run "waf_and_rate_limit_defaults" { + command = apply + variables { + labels = { team = "platform", managed-by = "someone-else" } + security_policy = { create = true } + } + + assert { + condition = google_compute_security_policy.this[0].name == "relay-edge" && google_compute_security_policy.this[0].type == "CLOUD_ARMOR" + error_message = "The policy is the backend policy -edge." + } + assert { + condition = google_compute_security_policy.this[0].labels == tomap({ team = "platform", managed-by = "terraform" }) + error_message = "Labels reach the policy and managed-by is reserved." + } + assert { + condition = google_compute_security_policy_rule.default[0].action == "allow" && google_compute_security_policy_rule.default[0].priority == 2147483647 + error_message = "Without an allowlist the default rule allows." + } + assert { + condition = length(google_compute_security_policy_rule.waf) == 2 && alltrue([for r in google_compute_security_policy_rule.waf : r.action == "deny(403)" && r.preview]) + error_message = "Nine OWASP rules in two deny rules, in preview by default." + } + assert { + condition = google_compute_security_policy_rule.waf["0"].match[0].expr[0].expression == "evaluatePreconfiguredWaf('sqli-v33-stable', {'sensitivity': 1}) || evaluatePreconfiguredWaf('xss-v33-stable', {'sensitivity': 1}) || evaluatePreconfiguredWaf('lfi-v33-stable', {'sensitivity': 1}) || evaluatePreconfiguredWaf('rfi-v33-stable', {'sensitivity': 1}) || evaluatePreconfiguredWaf('rce-v33-stable', {'sensitivity': 1})" + error_message = "WAF expression: five preconfigured rules at sensitivity 1." + } + assert { + condition = strcontains(google_compute_security_policy_rule.waf["1"].match[0].expr[0].expression, "evaluatePreconfiguredWaf('cve-canary', {'sensitivity': 1})") + error_message = "The default rules include cve-canary (Log4Shell)." + } + assert { + condition = keys(google_compute_security_policy_rule.rate_limit) == ["sign-in-0"] && google_compute_security_policy_rule.rate_limit["sign-in-0"].action == "throttle" && google_compute_security_policy_rule.rate_limit["sign-in-0"].match[0].expr[0].expression == "request.path.matches('^/(login|(cloud/)?api/v1/auth/)')" + error_message = "One throttle rule on the sign-in paths." + } + assert { + condition = google_compute_security_policy_rule.rate_limit["sign-in-0"].rate_limit_options[0].enforce_on_key == "IP" && google_compute_security_policy_rule.rate_limit["sign-in-0"].rate_limit_options[0].exceed_action == "deny(429)" && google_compute_security_policy_rule.rate_limit["sign-in-0"].rate_limit_options[0].rate_limit_threshold[0].count == 60 + error_message = "Throttle per connecting address, 60 per minute, 429 beyond." + } + assert { + condition = length(google_compute_security_policy_rule.allowlist) == 0 && length(google_compute_security_policy_rule.origin_secret) == 0 && length(google_compute_security_policy_rule.client_ip) == 0 + error_message = "No allowlist, secret-header or client rules unless asked for." + } + assert { + condition = length(output.chart_values.gkeIngress) == 2 && output.chart_values.gkeIngress.sslPolicy == "relay-tls" && output.chart_values.gkeIngress.securityPolicy == "relay-edge" + error_message = "chart_values names both policies." + } +} + +run "pure_source_ip_allowlist" { + command = apply + variables { + security_policy = { + create = true + waf = { enabled = false } + rate_limits = [] + source_ip_allowlist = ["198.51.100.0/24", "203.0.113.7/32"] + } + } + + assert { + condition = google_compute_security_policy_rule.default[0].action == "deny(403)" + error_message = "An allowlist makes the default rule deny." + } + assert { + condition = length(google_compute_security_policy_rule.allowlist) == 1 && google_compute_security_policy_rule.allowlist["0"].action == "allow" && google_compute_security_policy_rule.allowlist["0"].match[0].config[0].src_ip_ranges == toset(["198.51.100.0/24", "203.0.113.7/32"]) || tolist(google_compute_security_policy_rule.allowlist["0"].match[0].config[0].src_ip_ranges) == tolist(["198.51.100.0/24", "203.0.113.7/32"]) + error_message = "The allowlist is one basic allow rule." + } + assert { + condition = length(google_compute_security_policy_rule.waf) + length(google_compute_security_policy_rule.rate_limit) == 0 + error_message = "WAF and rate limits can be switched off: a pure allowlist." + } +} + +run "cloudflare_origin_lockdown" { + command = apply + variables { + security_policy = { + create = true + cloudflare = { enabled = true } + client_ip_rules = [{ + name = "uploads" + path_prefixes = ["/v1/delivery", "/v1/turns"] + cidrs = ["192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24", "192.0.2.128/25", "198.18.0.0/15"] + unthrottled = true + }] + } + origin_secret_header = { name = "x-origin-auth", values = ["0123456789abcdef0123456789abcdef"] } + } + + assert { + condition = google_compute_security_policy_rule.default[0].action == "deny(403)" && length(google_compute_security_policy_rule.allowlist) == 2 && length(flatten([for r in google_compute_security_policy_rule.allowlist : tolist(r.match[0].config[0].src_ip_ranges)])) == 15 + error_message = "Cloudflare's 15 IPv4 ranges in two basic allow rules, default deny." + } + assert { + condition = google_compute_security_policy.this[0].advanced_options_config[0].user_ip_request_headers == toset(["CF-Connecting-IP"]) && output.client_ip_header == "CF-Connecting-IP" + error_message = "The client address comes from CF-Connecting-IP." + } + assert { + condition = google_compute_security_policy_rule.origin_secret[0].priority == 100 && google_compute_security_policy_rule.origin_secret[0].action == "deny(403)" && nonsensitive(google_compute_security_policy_rule.origin_secret[0].match[0].expr[0].expression) == "!has(request.headers['x-origin-auth']) || !(request.headers['x-origin-auth'] == '0123456789abcdef0123456789abcdef')" + error_message = "The secret-header rule is the first rule and denies a missing or wrong value." + } + # Throttle admits conforming requests, so each throttle rule also requires a Cloudflare address. + assert { + condition = length(google_compute_security_policy_rule.rate_limit) == 4 && alltrue([for r in google_compute_security_policy_rule.rate_limit : strcontains(r.match[0].expr[0].expression, "&& (inIpRange(origin.ip, '") && r.rate_limit_options[0].enforce_on_key == "USER_IP"]) + error_message = "Rate limits stay inside the allowlist and count per client address." + } + assert { + condition = google_compute_security_policy_rule.rate_limit["sign-in-0"].match[0].expr[0].expression == "request.path.matches('^/(login|(cloud/)?api/v1/auth/)') && (inIpRange(origin.ip, '173.245.48.0/20') || inIpRange(origin.ip, '103.21.244.0/22') || inIpRange(origin.ip, '103.22.200.0/22') || inIpRange(origin.ip, '103.31.4.0/22'))" + error_message = "Five subexpressions at most: the path and four ranges." + } + assert { + condition = keys(google_compute_security_policy_rule.client_ip) == ["uploads-allow-0", "uploads-allow-1", "uploads-deny"] + error_message = "Five client ranges: two allow rules and the path deny." + } + assert { + condition = google_compute_security_policy_rule.client_ip["uploads-allow-1"].match[0].expr[0].expression == "request.path.matches('^(/v1/delivery|/v1/turns)') && (inIpRange(origin.user_ip, '198.18.0.0/15'))" && google_compute_security_policy_rule.client_ip["uploads-deny"].action == "deny(403)" && google_compute_security_policy_rule.client_ip["uploads-deny"].match[0].expr[0].expression == "request.path.matches('^(/v1/delivery|/v1/turns)')" + error_message = "Client rules match the path and origin.user_ip, then deny the path." + } + # Order: secret header, WAF, client rules, rate limits, allowlist, default. + assert { + condition = max([for r in google_compute_security_policy_rule.waf : r.priority]...) < min([for r in google_compute_security_policy_rule.client_ip : r.priority]...) && max([for r in google_compute_security_policy_rule.client_ip : r.priority]...) < min([for r in google_compute_security_policy_rule.rate_limit : r.priority]...) && max([for r in google_compute_security_policy_rule.rate_limit : r.priority]...) < min([for r in google_compute_security_policy_rule.allowlist : r.priority]...) + error_message = "Rule priorities keep deny rules ahead of every allow." + } +} + +run "client_rules_need_the_secret_header" { + command = plan + variables { + security_policy = { + create = true + cloudflare = { enabled = true } + client_ip_rules = [{ name = "uploads", path_prefixes = ["/v1/delivery"], cidrs = ["192.0.2.0/24"] }] + } + } + expect_failures = [google_compute_security_policy.this] +} + +run "client_header_needs_a_locked_edge" { + command = plan + variables { + security_policy = { create = true, client_ip_header = "True-Client-IP" } + } + expect_failures = [google_compute_security_policy.this] +} + +run "open_ranges_are_refused" { + command = plan + variables { + security_policy = { create = true, source_ip_allowlist = ["0.0.0.0/0"] } + } + expect_failures = [var.security_policy] +} + +run "weak_secret_is_refused" { + command = plan + variables { + security_policy = { create = true } + origin_secret_header = { name = "x-origin-auth", values = ["short"] } + } + expect_failures = [var.origin_secret_header] +} + +# Rotation without an outage: the current and the next value are both accepted. +run "secret_rotation_accepts_two_values" { + command = apply + variables { + security_policy = { create = true } + origin_secret_header = { name = "x-origin-auth", values = ["0123456789abcdef0123456789abcdef", "fedcba9876543210fedcba9876543210"] } + } + + assert { + condition = nonsensitive(google_compute_security_policy_rule.origin_secret[0].match[0].expr[0].expression) == "!has(request.headers['x-origin-auth']) || !(request.headers['x-origin-auth'] == '0123456789abcdef0123456789abcdef' || request.headers['x-origin-auth'] == 'fedcba9876543210fedcba9876543210')" + error_message = "A request carrying either value passes; anything else is denied." + } +} + +run "four_secret_values_are_refused" { + command = plan + variables { + security_policy = { create = true } + origin_secret_header = { name = "x-origin-auth", values = ["0123456789abcdef0123456789abcde1", "0123456789abcdef0123456789abcde2", "0123456789abcdef0123456789abcde3", "0123456789abcdef0123456789abcde4"] } + } + expect_failures = [var.origin_secret_header] +} + +# The documented shape behind a proxy: vendor ranges reach only the upload and +# MCP paths, the organisation's own ranges reach everything, nobody else +# reaches anything. +run "vendor_paths_and_a_closed_remainder" { + command = apply + variables { + security_policy = { + create = true + cloudflare = { enabled = true } + rate_limits = [] + client_ip_rules = [ + { + name = "vendors" + hosts = ["history.relay.example.com"] + path_prefixes = ["/v1/delivery", "/v1/turns", "/v1/auth/token/refresh"] + cidrs = ["192.0.2.0/24", "198.51.100.0/24"] + rate_limit = { count = 600, interval_sec = 60 } + }, + { + name = "sign-in" + path_prefixes = ["/login", "/api/v1/auth", "/cloud/api/v1/auth"] + cidrs = ["203.0.113.0/24"] + rate_limit = { count = 60, interval_sec = 60 } + }, + { + name = "staff" + path_prefixes = ["/"] + cidrs = ["203.0.113.0/24", "192.0.2.128/25", "198.18.0.0/15", "198.51.100.128/25", "203.0.113.128/25"] + unthrottled = true + }, + ] + } + origin_secret_header = { name = "x-origin-auth", values = ["0123456789abcdef0123456789abcdef"] } + } + + assert { + condition = keys(google_compute_security_policy_rule.client_ip) == ["sign-in-allow-0", "sign-in-deny", "staff-allow-0", "staff-allow-1", "staff-deny", "vendors-allow-0", "vendors-deny"] + error_message = "Every rule admits its clients (four ranges per rule, three with hosts) and then denies its paths." + } + assert { + condition = google_compute_security_policy_rule.client_ip["vendors-allow-0"].match[0].expr[0].expression == "request.headers['host'].lower().matches('^(history\\\\.relay\\\\.example\\\\.com)(:[0-9]+)?$') && request.path.matches('^(/v1/delivery|/v1/turns|/v1/auth/token/refresh)') && (inIpRange(origin.user_ip, '192.0.2.0/24') || inIpRange(origin.user_ip, '198.51.100.0/24'))" + error_message = "Host, path and the client ranges, within five subexpressions." + } + # A short vendor list ahead of a staff-only "/" rule: the vendor is admitted + # by its own rule, which ends evaluation, so the later staff rule cannot + # deny it. + assert { + condition = google_compute_security_policy_rule.client_ip["vendors-allow-0"].action == "throttle" && google_compute_security_policy_rule.client_ip["vendors-allow-0"].priority < google_compute_security_policy_rule.client_ip["vendors-deny"].priority && google_compute_security_policy_rule.client_ip["vendors-deny"].priority < google_compute_security_policy_rule.client_ip["sign-in-allow-0"].priority && google_compute_security_policy_rule.client_ip["sign-in-deny"].priority < google_compute_security_policy_rule.client_ip["staff-allow-0"].priority + error_message = "Rules apply in list order: the first rule that matches the path decides." + } + # A rule's own limit applies whatever the length of its list. + assert { + condition = google_compute_security_policy_rule.client_ip["vendors-allow-0"].rate_limit_options[0].rate_limit_threshold[0].count == 600 && google_compute_security_policy_rule.client_ip["sign-in-allow-0"].action == "throttle" && google_compute_security_policy_rule.client_ip["sign-in-allow-0"].rate_limit_options[0].rate_limit_threshold[0].count == 60 && google_compute_security_policy_rule.client_ip["sign-in-allow-0"].rate_limit_options[0].enforce_on_key == "USER_IP" && google_compute_security_policy_rule.client_ip["sign-in-allow-0"].rate_limit_options[0].exceed_action == "deny(429)" + error_message = "A one-range rule with a rate_limit throttles per client address." + } + assert { + condition = google_compute_security_policy_rule.client_ip["staff-allow-0"].action == "allow" && length(google_compute_security_policy_rule.client_ip["staff-allow-0"].rate_limit_options) == 0 && google_compute_security_policy_rule.client_ip["staff-deny"].action == "deny(403)" && google_compute_security_policy_rule.client_ip["staff-deny"].match[0].expr[0].expression == "request.path.matches('^(/)')" + error_message = "An unthrottled rule admits with allow; the closing rule denies every remaining path." + } + assert { + condition = alltrue([for r in google_compute_security_policy_rule.client_ip : r.priority > google_compute_security_policy_rule.origin_secret[0].priority]) && length(google_compute_security_policy_rule.rate_limit) == 0 + error_message = "The secret-header rule runs before any client rule." + } +} + +# Admitted requests skip rate_limits, so a rule must say how it is limited. +run "client_rule_must_choose_a_limit" { + command = plan + variables { + security_policy = { + create = true + cloudflare = { enabled = true } + client_ip_rules = [{ name = "uploads", path_prefixes = ["/v1/delivery"], cidrs = ["192.0.2.0/24"] }] + } + origin_secret_header = { name = "x-origin-auth", values = ["0123456789abcdef0123456789abcdef"] } + } + expect_failures = [google_compute_security_policy.this] +} + +# A rule for every path decides every request, so path rate_limits would be dead. +run "catch_all_client_rule_refuses_dead_rate_limits" { + command = plan + variables { + security_policy = { + create = true + cloudflare = { enabled = true } + client_ip_rules = [{ name = "staff", path_prefixes = ["/"], cidrs = ["203.0.113.0/24"], unthrottled = true }] + } + origin_secret_header = { name = "x-origin-auth", values = ["0123456789abcdef0123456789abcdef"] } + } + expect_failures = [google_compute_security_policy.this] +} + +# A regex escape in path_regex survives the expression's string literal. +run "rate_limit_regex_with_an_escape" { + command = apply + variables { + security_policy = { + create = true + waf = { enabled = false } + rate_limits = [{ name = "v1", path_regex = "^/v1\\.0/", count = 10, interval_sec = 60 }] + } + } + + assert { + condition = google_compute_security_policy_rule.rate_limit["v1-0"].match[0].expr[0].expression == "request.path.matches('^/v1\\\\.0/')" + error_message = "The backslash is doubled inside the quoted expression string." + } +} diff --git a/terraform/modules/gcp/edge-security/variables.tf b/terraform/modules/gcp/edge-security/variables.tf new file mode 100644 index 00000000..63175805 --- /dev/null +++ b/terraform/modules/gcp/edge-security/variables.tf @@ -0,0 +1,149 @@ +variable "project_id" { + description = "Project of the GKE cluster. GKE Ingress resolves gkeIngress.sslPolicy and gkeIngress.securityPolicy by name in this project." + type = string +} + +variable "name" { + description = "Name prefix: the SSL policy is -tls and the Cloud Armor policy -edge." + type = string + validation { + condition = can(regex("^[a-z]([-a-z0-9]{0,56}[a-z0-9])?$", var.name)) + error_message = "name must be 1-58 lowercase letters, digits or hyphens, beginning with a letter." + } +} + +variable "labels" { + description = "Labels for every resource here that takes labels (the Cloud Armor policy; Compute SSL policies have none in the API). managed-by=terraform is always added." + type = map(string) + default = {} +} + +variable "ssl_policy" { + description = "Compute SSL policy for the load balancer's HTTPS listener (chart value gkeIngress.sslPolicy). MODERN with TLS 1.2 serves current browsers, the desktop app and CLIs; RESTRICTED narrows the cipher list further (TLS 1.2 minimum is required by it)." + type = object({ + create = optional(bool, true) + profile = optional(string, "MODERN") + min_tls_version = optional(string, "TLS_1_2") + }) + default = {} + validation { + condition = contains(["MODERN", "RESTRICTED"], var.ssl_policy.profile) && contains(["TLS_1_2", "TLS_1_3"], var.ssl_policy.min_tls_version) + error_message = "ssl_policy.profile must be MODERN or RESTRICTED and min_tls_version TLS_1_2 or TLS_1_3 (COMPATIBLE and TLS 1.0/1.1 are what this policy exists to switch off)." + } + validation { + condition = var.ssl_policy.min_tls_version != "TLS_1_3" || var.ssl_policy.profile == "RESTRICTED" + error_message = "min_tls_version TLS_1_3 requires profile RESTRICTED." + } +} + +variable "security_policy" { + description = <<-EOT + Cloud Armor backend security policy (chart value gkeIngress.securityPolicy), off unless create = true. + + - waf: preconfigured OWASP rules (deny 403) at the given sensitivity (1 = fewest false positives). preview = true (the default) only logs matches: session transcripts and messages carry code, SQL and shell text, so read the logs before enforcing. + - rate_limits: per-client throttle (429) on the listed path regexes; the default covers password sign-in, setup-key and login paths. + - source_ip_allowlist: when non-empty (or cloudflare.enabled), the default rule becomes deny 403 and only these connecting addresses are admitted: a pure source-IP allowlist. + - cloudflare: admit Cloudflare's published ranges as the connecting addresses (origin lockdown) and read the client address from CF-Connecting-IP. + - client_ip_header: header an upstream proxy sets with the real client address (Cloud Armor origin.user_ip). Only trustworthy when nothing but that proxy can reach the load balancer: combine with an allowlist of the proxy's ranges, or origin_secret_header. + - client_ip_rules: per-client allowlists scoped to paths (and optionally hosts), evaluated in list order. The first rule whose path_prefixes (and hosts, when given) match a request decides it: admitted if the client address (origin.user_ip) is in cidrs, denied 403 otherwise. A last rule with path_prefixes = ["/"] therefore closes every other path to everyone but its cidrs. Requires origin_secret_header. An admitted request meets no later rule, including rate_limits, so every rule states its own limit: rate_limit (requests per interval per client address), or unthrottled = true. With a rule for "/" set rate_limits = [] and express sign-in limits as an earlier, narrower rule. + - adaptive_protection: Layer 7 DDoS detection (alerts; automatic deployment needs Cloud Armor Enterprise). + EOT + type = object({ + create = optional(bool, false) + waf = optional(object({ + enabled = optional(bool, true) + preview = optional(bool, true) + sensitivity = optional(number, 1) + rules = optional(list(string), [ + "sqli-v33-stable", "xss-v33-stable", "lfi-v33-stable", "rfi-v33-stable", "rce-v33-stable", + "scannerdetection-v33-stable", "protocolattack-v33-stable", "sessionfixation-v33-stable", "cve-canary", + ]) + }), {}) + rate_limits = optional(list(object({ + name = string + path_regex = string + count = number + interval_sec = number + ban_duration_sec = optional(number, 0) + })), [{ + name = "sign-in" + path_regex = "^/(login|(cloud/)?api/v1/auth/)" + count = 60 + interval_sec = 60 + }]) + source_ip_allowlist = optional(list(string), []) + cloudflare = optional(object({ + enabled = optional(bool, false) + include_ipv6 = optional(bool, false) + # https://www.cloudflare.com/ips/ as read on 2026-11-18. Cloudflare + # announces changes; compare the list with that page before each apply. + ipv4_cidrs = optional(list(string), [ + "173.245.48.0/20", "103.21.244.0/22", "103.22.200.0/22", "103.31.4.0/22", "141.101.64.0/18", + "108.162.192.0/18", "190.93.240.0/20", "188.114.96.0/20", "197.234.240.0/22", "198.41.128.0/17", + "162.158.0.0/15", "104.16.0.0/13", "104.24.0.0/14", "172.64.0.0/13", "131.0.72.0/22", + ]) + ipv6_cidrs = optional(list(string), [ + "2400:cb00::/32", "2606:4700::/32", "2803:f800::/32", "2405:b500::/32", "2405:8100::/32", + "2a06:98c0::/29", "2c0f:f248::/32", + ]) + }), {}) + client_ip_header = optional(string) + client_ip_rules = optional(list(object({ + name = string + path_prefixes = list(string) + hosts = optional(list(string), []) + cidrs = list(string) + rate_limit = optional(object({ count = number, interval_sec = number })) + unthrottled = optional(bool, false) + })), []) + adaptive_protection = optional(bool, false) + }) + default = {} + + validation { + condition = contains([0, 1, 2, 3, 4], var.security_policy.waf.sensitivity) && alltrue([for r in var.security_policy.waf.rules : can(regex("^[a-z0-9-]+$", r))]) + error_message = "waf.sensitivity is 0-4 and waf.rules are Cloud Armor preconfigured WAF rule names (for example sqli-v33-stable)." + } + validation { + condition = alltrue([for c in concat( + var.security_policy.source_ip_allowlist, var.security_policy.cloudflare.ipv4_cidrs, var.security_policy.cloudflare.ipv6_cidrs, + flatten([for r in var.security_policy.client_ip_rules : r.cidrs]) + ) : can(cidrhost(c, 0)) && !contains(["0.0.0.0/0", "::/0"], c)]) + error_message = "Every allowlist entry must be a CIDR range, and never 0.0.0.0/0 or ::/0." + } + validation { + condition = alltrue([for r in var.security_policy.rate_limits : + can(regex("^[a-z0-9]([-a-z0-9]{0,30}[a-z0-9])?$", r.name)) && !strcontains(r.path_regex, "'") && length(r.path_regex) <= 512 && + r.count >= 1 && contains([10, 30, 60, 120, 180, 240, 300, 600, 900, 1200, 1800, 2700, 3600], r.interval_sec) && r.ban_duration_sec >= 0 + ]) && length(var.security_policy.rate_limits) <= 9 && length(distinct(var.security_policy.rate_limits[*].name)) == length(var.security_policy.rate_limits) + error_message = "At most 9 rate_limits with unique lowercase names, a path_regex without single quotes, count >= 1 and an interval_sec Cloud Armor accepts (10, 30, 60, 120, 180, 240, 300, 600, 900, 1200, 1800, 2700 or 3600)." + } + validation { + condition = alltrue([for r in var.security_policy.client_ip_rules : + can(regex("^[a-z0-9]([-a-z0-9]{0,30}[a-z0-9])?$", r.name)) && length(r.cidrs) >= 1 && length(r.cidrs) <= 294 && + length(r.path_prefixes) >= 1 && alltrue([for p in r.path_prefixes : can(regex("^/[A-Za-z0-9/._~-]*$", p))]) && + alltrue([for h in r.hosts : can(regex("^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$", h))]) && + (r.rate_limit == null ? true : r.rate_limit.count >= 1 && contains([10, 30, 60, 120, 180, 240, 300, 600, 900, 1200, 1800, 2700, 3600], r.rate_limit.interval_sec)) + ]) && length(var.security_policy.client_ip_rules) <= 9 && length(distinct(var.security_policy.client_ip_rules[*].name)) == length(var.security_policy.client_ip_rules) + error_message = "At most 9 client_ip_rules with unique lowercase names, 1-294 cidrs, path_prefixes that start with / and hold only unreserved URL characters, lowercase DNS hosts, and a rate_limit with count >= 1 and an interval_sec Cloud Armor accepts." + } + validation { + condition = var.security_policy.client_ip_header == null || can(regex("^[A-Za-z][A-Za-z0-9-]{0,63}$", var.security_policy.client_ip_header)) + error_message = "client_ip_header is an HTTP header name, for example CF-Connecting-IP or True-Client-IP." + } +} + +variable "origin_secret_header" { + description = "Optional shared-secret request header a fronting proxy adds on every request (for Cloudflare: a request header transform rule). Requests without the exact value are denied 403 before any other rule, which proves the request came through the proxy and makes its client-address header trustworthy. values lists every accepted value (1 to 3), so a rotation has no outage: add the new value here and apply, switch the proxy to it, then remove the old value and apply. The values are stored in Terraform state and readable in the Cloud Armor policy by anyone with compute.securityPolicies.get: treat both as secret stores." + type = object({ + name = string + values = list(string) + }) + default = null + sensitive = true + validation { + # A conditional, not ||: Terraform before 1.9 evaluates both sides of ||, and the attributes of null cannot be read. + condition = var.origin_secret_header == null ? true : (can(regex("^[a-z][a-z0-9-]{0,63}$", var.origin_secret_header.name)) && length(var.origin_secret_header.values) >= 1 && length(var.origin_secret_header.values) <= 3 && length(distinct(var.origin_secret_header.values)) == length(var.origin_secret_header.values) && alltrue([for v in var.origin_secret_header.values : can(regex("^[A-Za-z0-9_-]{32,256}$", v))])) + error_message = "origin_secret_header.name is a lowercase header name; values holds 1-3 distinct values of 32-256 characters from A-Z a-z 0-9 _ - (for example: openssl rand -hex 32)." + } +} diff --git a/terraform/modules/gcp/edge-security/versions.tf b/terraform/modules/gcp/edge-security/versions.tf new file mode 100644 index 00000000..2ef82a5a --- /dev/null +++ b/terraform/modules/gcp/edge-security/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.6" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 7.0, < 8.0" + } + } +}