diff --git a/charts/agent-relay/Chart.yaml b/charts/agent-relay/Chart.yaml index b9b126bc..1077cad1 100644 --- a/charts/agent-relay/Chart.yaml +++ b/charts/agent-relay/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: agent-relay description: Self-hosted Agent Relay messaging and sessions suite type: application -version: 0.7.59 +version: 0.7.60 appVersion: "0.1.0" kubeVersion: ">=1.28.0-0" annotations: diff --git a/charts/agent-relay/README.md b/charts/agent-relay/README.md index b421da75..ce73d247 100644 --- a/charts/agent-relay/README.md +++ b/charts/agent-relay/README.md @@ -295,6 +295,49 @@ Behind a proxy that fronts the load balancer (Cloudflare), also raise `relay-control.auth.trustedProxies` by one (the Google load balancer alone needs 2; the guide shows how to check). +## Internal-only GKE edge + +For an install that must not be reachable from the internet, set +`gkeIngress.mode: internal` (start from `values-gcp-internal.yaml`). GKE then +creates a regional **internal** Application Load Balancer: a private address, +HTTPS only, reachable from the VPC and from the networks routed to it (VPN, +Interconnect). It is the GKE counterpart of an internal-scheme ALB on EKS. +Hosts, routes, backends and timeouts are the same as on the external edge. + +| | External (default) | Internal | +| --- | --- | --- | +| Ingress class | `gce` | `gce-internal` | +| `gkeIngress.staticIpName` | a global external address | a **regional internal** address | +| Certificate | ManagedCertificate, or `tlsSecretName` | `tlsSecretName`, or `internal.preSharedCerts` (regional self-managed certificates). A Google-managed certificate is not possible | +| HTTP | redirected to HTTPS | closed (`allow-http: "false"`) | +| `sslPolicy`, `securityPolicy` | optional | refused: GKE Ingress supports neither FrontendConfig nor Cloud Armor on an internal load balancer | +| Sources the pods see | Google front ends | the VPC's **proxy-only subnet** | + +Before the install, the VPC needs a proxy-only subnet in the cluster's region, +a reserved regional internal address, and a firewall rule from that subnet to +the nodes; +[`terraform/modules/gcp/internal-edge`](https://github.com/AgentWorkforce/infra/blob/main/terraform/modules/gcp/internal-edge) +creates the three. Then: + +- `gkeIngress.internal.proxyOnlySubnetCidr` is the subnet's range, and every + backend's `networkPolicy` must allow it next to the two health-check ranges. + The render fails, naming the backend, when one does not. + `values-gcp-internal.yaml` restates all five lists. +- DNS: A records for every host, pointing at the private address, in a zone + your VPN clients resolve (a private zone, or your corporate DNS). The public + URLs in the values stay `https://`. +- Certificate: with `tlsSecretName`, cert-manager with a DNS-01 challenge + issues and renews a publicly trusted certificate without the hosts being + reachable from the internet. If the DNS zone is in another Google Cloud + project, grant cert-manager's Workload Identity `roles/dns.admin` on that + zone's project. A private CA works if every desktop, CLI and agent trusts it. +- Agents and desktops outside the VPN cannot upload sessions or connect. Plan + for that before choosing this mode. +- Pass the overlay on every `helm upgrade`, or merge it into your site values. + An upgrade without it renders the external edge again. + +Rendered and tested in CI; not yet proven on a live cluster. + ## Customer-neutral EKS with RDS native History `values-eks-rds-native.yaml` is the reusable EKS overlay for the supported diff --git a/charts/agent-relay/templates/durable-actors-networkpolicy.yaml b/charts/agent-relay/templates/durable-actors-networkpolicy.yaml index 3800f0be..f3f038df 100644 --- a/charts/agent-relay/templates/durable-actors-networkpolicy.yaml +++ b/charts/agent-relay/templates/durable-actors-networkpolicy.yaml @@ -37,6 +37,10 @@ spec: {{- if dig "hosts" "actors" "" (.Values.gkeIngress | default dict) }} - ipBlock: { cidr: 35.191.0.0/16 } - ipBlock: { cidr: 130.211.0.0/22 } + {{- if eq (dig "mode" "external" (.Values.gkeIngress | default dict) | toString) "internal" }} + # The internal load balancer's proxies (gkeIngress.internal.proxyOnlySubnetCidr). + - ipBlock: { cidr: {{ dig "internal" "proxyOnlySubnetCidr" "" (.Values.gkeIngress | default dict) | quote }} } + {{- end }} {{- end }} {{- range dig "extraIngressFrom" (list) $policy }} - {{- toYaml . | nindent 10 }} diff --git a/charts/agent-relay/templates/gke-ingress.yaml b/charts/agent-relay/templates/gke-ingress.yaml index 6194d04e..6264a14e 100644 --- a/charts/agent-relay/templates/gke-ingress.yaml +++ b/charts/agent-relay/templates/gke-ingress.yaml @@ -20,6 +20,9 @@ creates them): {{- if and $securityPolicy (not (regexMatch $computeName $securityPolicy)) }}{{ fail "gkeIngress.securityPolicy must be the name of a Cloud Armor security policy in the cluster's project (lowercase letters, digits and hyphens, at most 63 characters), not a URL" }}{{ end -}} {{- $tlsSecretName := dig "tlsSecretName" "" $gke | toString -}} {{- if and $tlsSecretName (not (regexMatch "^[a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?$" $tlsSecretName)) }}{{ fail "gkeIngress.tlsSecretName must be the name of a kubernetes.io/tls Secret in the release namespace" }}{{ end -}} +{{- $gkeInternal := eq (dig "mode" "external" $gke | toString) "internal" -}} +{{- $gkeProxyCidr := dig "internal" "proxyOnlySubnetCidr" "" $gke | toString -}} +{{- if $gkeInternal }}{{- if not (regexMatch "^(10(\\.(25[0-5]|2[0-4][0-9]|1?[0-9]?[0-9])){3}|172\\.(1[6-9]|2[0-9]|3[01])(\\.(25[0-5]|2[0-4][0-9]|1?[0-9]?[0-9])){2}|192\\.168(\\.(25[0-5]|2[0-4][0-9]|1?[0-9]?[0-9])){2})/(8|9|1[0-9]|2[0-6])$" $gkeProxyCidr) }}{{ fail "gkeIngress.mode=internal requires gkeIngress.internal.proxyOnlySubnetCidr: the RFC 1918 range (/26 or larger) of the VPC's proxy-only subnet (purpose REGIONAL_MANAGED_PROXY) in the cluster's region; the load balancer's proxies connect to the pods from it" }}{{ end -}}{{ end -}} {{- $components := .Values.components | default dict -}} {{- $relaycast := .Values.relaycast | default dict -}} {{- $cloudApi := index .Values "relay-control" | default dict -}} @@ -53,10 +56,13 @@ shape (hosts.files without components.relayfile fails further down). */ -}} {{- range $name, $peers := ternary (dict) (dict "sessionReadApi.networkPolicy.publicFrom" (dig "publicFrom" (list) $readerPolicy) "ingestEdge.networkPolicy.ingressFrom" (dig "ingressFrom" (list) $ingestPolicy)) (not (empty $gateOnly)) }} {{- $hasGfePrimary := false -}} {{- $hasGfeSecondary := false -}} +{{- $hasProxyOnly := false -}} {{- range $peer := $peers }} {{- if and (eq (len $peer) 1) (eq (dig "ipBlock" "cidr" "" $peer) "35.191.0.0/16") (empty (dig "ipBlock" "except" (list) $peer)) }}{{ $hasGfePrimary = true }}{{ end -}} {{- if and (eq (len $peer) 1) (eq (dig "ipBlock" "cidr" "" $peer) "130.211.0.0/22") (empty (dig "ipBlock" "except" (list) $peer)) }}{{ $hasGfeSecondary = true }}{{ end -}} +{{- if and $gkeInternal (eq (len $peer) 1) (eq (dig "ipBlock" "cidr" "" $peer) $gkeProxyCidr) (empty (dig "ipBlock" "except" (list) $peer)) }}{{ $hasProxyOnly = true }}{{ end -}} {{- end }} +{{- if and $gkeInternal (not $hasProxyOnly) }}{{ fail (printf "gkeIngress.mode=internal requires sessions-pipeline.%s to allow the proxy-only subnet %s (gkeIngress.internal.proxyOnlySubnetCidr) without exclusions" $name $gkeProxyCidr) }}{{ end -}} {{- if not $hasGfePrimary }}{{ fail (printf "native gkeIngress requires sessions-pipeline.%s to allow GKE range 35.191.0.0/16 without exclusions" $name) }}{{ end -}} {{- if not $hasGfeSecondary }}{{ fail (printf "native gkeIngress requires sessions-pipeline.%s to allow GKE range 130.211.0.0/22 without exclusions" $name) }}{{ end -}} {{- end }} @@ -65,12 +71,15 @@ shape (hosts.files without components.relayfile fails further down). */ -}} {{- $hasSameNamespace := false -}} {{- $hasGfePrimary := false -}} {{- $hasGfeSecondary := false -}} +{{- $hasProxyOnly := false -}} {{- range $peer := dig "networkPolicy" "ingressFrom" (list) $backendValues }} +{{- if and $gkeInternal (eq (len $peer) 1) (eq (dig "ipBlock" "cidr" "" $peer) $gkeProxyCidr) (empty (dig "ipBlock" "except" (list) $peer)) }}{{ $hasProxyOnly = true }}{{ end -}} {{- if and (eq (len $peer) 1) (hasKey $peer "podSelector") (empty (get $peer "podSelector")) }}{{ $hasSameNamespace = true }}{{ end -}} {{- if and (eq (len $peer) 1) (eq (dig "ipBlock" "cidr" "" $peer) "35.191.0.0/16") (empty (dig "ipBlock" "except" (list) $peer)) }}{{ $hasGfePrimary = true }}{{ end -}} {{- if and (eq (len $peer) 1) (eq (dig "ipBlock" "cidr" "" $peer) "130.211.0.0/22") (empty (dig "ipBlock" "except" (list) $peer)) }}{{ $hasGfeSecondary = true }}{{ end -}} {{- end }} {{- if not $hasSameNamespace }}{{ fail (printf "gkeIngress requires %s.networkPolicy.ingressFrom to retain same-namespace pod access" $name) }}{{ end -}} +{{- if and $gkeInternal (not $hasProxyOnly) }}{{ fail (printf "gkeIngress.mode=internal requires %s.networkPolicy.ingressFrom to allow the proxy-only subnet %s (gkeIngress.internal.proxyOnlySubnetCidr) without exclusions" $name $gkeProxyCidr) }}{{ end -}} {{- if not $hasGfePrimary }}{{ fail (printf "gkeIngress requires %s.networkPolicy.ingressFrom to allow GKE health checker/proxy range 35.191.0.0/16 without exclusions" $name) }}{{ end -}} {{- if not $hasGfeSecondary }}{{ fail (printf "gkeIngress requires %s.networkPolicy.ingressFrom to allow GKE health checker/proxy range 130.211.0.0/22 without exclusions" $name) }}{{ end -}} {{- end }} @@ -79,8 +88,36 @@ shape (hosts.files without components.relayfile fails further down). */ -}} {{- $nativeIngestEdge := printf "%s-history-ingest" ($gkeName | trunc 48 | trimSuffix "-") -}} {{- $staticIpName := required "gkeIngress.staticIpName is required when gkeIngress.enabled=true" (dig "staticIpName" "" $gke) -}} {{- $managedCertificateName := dig "managedCertificateName" "" $gke -}} -{{- if not (or $tlsSecretName $managedCertificateName) }}{{ fail "gkeIngress.managedCertificateName is required when gkeIngress.enabled=true (or set gkeIngress.tlsSecretName to serve a certificate from a Secret)" }}{{ end -}} -{{- $frontendConfigName := required "gkeIngress.frontendConfigName is required when gkeIngress.enabled=true" (dig "frontendConfigName" "" $gke) -}} +{{- /* mode external (default): a global external Application Load Balancer +(Ingress class gce), a global static address, a ManagedCertificate or TLS +Secret, and a FrontendConfig. mode internal: a regional internal Application +Load Balancer (class gce-internal) that answers only inside the VPC and the +networks routed to it (VPN, Interconnect). GKE Ingress supports neither +ManagedCertificate nor FrontendConfig nor Cloud Armor on an internal load +balancer, so the certificate is a TLS Secret or pre-shared regional +certificates, port 80 is closed instead of redirected, and sslPolicy and +securityPolicy are refused. Its proxies connect to the pods from the VPC's +proxy-only subnet, which every backend NetworkPolicy must admit. */ -}} +{{- $mode := dig "mode" "external" $gke | toString -}} +{{- if not (has $mode (list "external" "internal")) }}{{ fail (printf "gkeIngress.mode must be external or internal, got %q" $mode) }}{{ end -}} +{{- $isInternal := eq $mode "internal" -}} +{{- $internal := dig "internal" (dict) $gke -}} +{{- if not (kindIs "map" $internal) }}{{ fail "gkeIngress.internal must be a map" }}{{ end -}} +{{- $preSharedCerts := dig "preSharedCerts" (list) $internal -}} +{{- if not (kindIs "slice" $preSharedCerts) }}{{ fail "gkeIngress.internal.preSharedCerts must be a list of regional SSL certificate names" }}{{ end -}} +{{- $proxyCidr := dig "proxyOnlySubnetCidr" "" $internal | toString -}} +{{- if $isInternal }} +{{- if and (empty $preSharedCerts) (not $tlsSecretName) }}{{ fail "gkeIngress.mode=internal requires a certificate: gkeIngress.tlsSecretName (a kubernetes.io/tls Secret) or gkeIngress.internal.preSharedCerts (regional SSL certificate names); ManagedCertificate is not supported on an internal load balancer" }}{{ end -}} +{{- if and $preSharedCerts $tlsSecretName }}{{ fail "set only one of gkeIngress.tlsSecretName and gkeIngress.internal.preSharedCerts" }}{{ end -}} +{{- range $cert := $preSharedCerts }} +{{- if not (and (kindIs "string" $cert) (regexMatch $computeName $cert)) }}{{ fail (printf "gkeIngress.internal.preSharedCerts: %v is not a Compute Engine SSL certificate name" $cert) }}{{ end -}} +{{- end }} +{{- if $securityPolicy }}{{ fail "gkeIngress.securityPolicy cannot be used with gkeIngress.mode=internal: GKE Ingress does not attach Cloud Armor to an internal load balancer. Restrict who can reach it with VPC firewall rules and the routes of your VPN or Interconnect" }}{{ end -}} +{{- if $sslPolicy }}{{ fail "gkeIngress.sslPolicy cannot be used with gkeIngress.mode=internal: it is applied through a FrontendConfig, which GKE supports on external Ingresses only" }}{{ end -}} +{{- else if or $preSharedCerts $proxyCidr }}{{ fail "gkeIngress.internal.* is set but gkeIngress.mode is external; set gkeIngress.mode=internal or clear gkeIngress.internal" }}{{ end -}} +{{- if not (or $isInternal $tlsSecretName $managedCertificateName) }}{{ fail "gkeIngress.managedCertificateName is required when gkeIngress.enabled=true (or set gkeIngress.tlsSecretName to serve a certificate from a Secret)" }}{{ end -}} +{{- $frontendConfigName := dig "frontendConfigName" "" $gke -}} +{{- if not (or $isInternal $frontendConfigName) }}{{ fail "gkeIngress.frontendConfigName is required when gkeIngress.enabled=true" }}{{ end -}} {{- $hostRoot := dig "hosts" "root" "" $gke -}} {{- $hostApi := dig "hosts" "api" "" $gke -}} {{- $hostCast := dig "hosts" "cast" "" $gke -}} @@ -317,7 +354,7 @@ spec: - { name: http, port: 8080, targetPort: http, protocol: TCP } --- {{- end }} -{{- if not $tlsSecretName }} +{{- if not (or $tlsSecretName $isInternal) }} apiVersion: networking.gke.io/v1 kind: ManagedCertificate metadata: @@ -339,6 +376,7 @@ spec: {{- end }} --- {{- end }} +{{- if not $isInternal }} apiVersion: networking.gke.io/v1beta1 kind: FrontendConfig metadata: @@ -355,6 +393,7 @@ spec: enabled: true responseCodeName: MOVED_PERMANENTLY_DEFAULT --- +{{- end }} # Deliberately has no selector. Sensitive Relaycast paths, and /internal on # the api host, are routed here so the edge returns an error instead of # forwarding them to the public backend. @@ -517,12 +556,23 @@ metadata: app.kubernetes.io/name: agent-relay app.kubernetes.io/instance: {{ .Release.Name }} annotations: + {{- if $isInternal }} + # Regional internal Application Load Balancer: reachable only from the + # VPC and the networks routed to it. HTTPS only; port 80 is not opened. + kubernetes.io/ingress.class: gce-internal + kubernetes.io/ingress.regional-static-ip-name: {{ $staticIpName | quote }} + kubernetes.io/ingress.allow-http: "false" + {{- if $preSharedCerts }} + ingress.gcp.kubernetes.io/pre-shared-cert: {{ join "," $preSharedCerts | quote }} + {{- end }} + {{- else }} kubernetes.io/ingress.class: gce kubernetes.io/ingress.global-static-ip-name: {{ $staticIpName | quote }} {{- if not $tlsSecretName }} networking.gke.io/managed-certificates: {{ $managedCertificateName | quote }} {{- end }} networking.gke.io/v1beta1.FrontendConfig: {{ $frontendConfigName | quote }} + {{- end }} spec: {{- if $tlsSecretName }} tls: diff --git a/charts/agent-relay/values-gcp-internal.yaml b/charts/agent-relay/values-gcp-internal.yaml new file mode 100644 index 00000000..f9436099 --- /dev/null +++ b/charts/agent-relay/values-gcp-internal.yaml @@ -0,0 +1,78 @@ +# Internal-only GKE edge: apply after values-gcp.yaml (and your site values), +# on EVERY helm upgrade of the release, or merge it into your site values. +# Helm keeps nothing from the previous command: an upgrade without these +# settings switches the install back to a public load balancer. +# +# helm upgrade --install ... -f values-gcp.yaml -f site.yaml -f values-gcp-internal.yaml +# +# The Ingress becomes a regional internal Application Load Balancer (class +# gce-internal). It has a private address and answers only inside the VPC and +# the networks routed to it (VPN, Interconnect); nothing is reachable from +# the internet. README: Internal-only GKE edge. +# +# Before the install, in the cluster's VPC and region (the Terraform module +# terraform/modules/gcp/internal-edge creates all three): +# - a proxy-only subnet (purpose REGIONAL_MANAGED_PROXY); +# - a reserved regional internal address, named in gkeIngress.staticIpName; +# - a firewall rule that lets the proxy-only subnet reach the nodes. +gkeIngress: + mode: internal + # GKE cannot apply an SSL policy or Cloud Armor to an internal load + # balancer, and the chart refuses them in this mode. Cleared here so a value + # from an earlier file (for example managed-values.json) does not stop the + # install. + sslPolicy: "" + securityPolicy: "" + # REQUIRED OVERRIDE: the name of the reserved regional INTERNAL address (not + # the global address of an external install). + staticIpName: example-agent-relay-internal + # The certificate. A Google-managed certificate cannot be used on an + # internal load balancer. Either a kubernetes.io/tls Secret: + # REQUIRED OVERRIDE (or use internal.preSharedCerts below instead). + tlsSecretName: example-agent-relay-tls + internal: + # REQUIRED OVERRIDE: the proxy-only subnet's range. The load balancer's + # proxies connect to the pods from it, so it is also in every list below. + proxyOnlySubnetCidr: 10.129.0.0/23 + # ...or regional self-managed SSL certificates + # (gcloud compute ssl-certificates create NAME --region REGION ...); + # then leave tlsSecretName empty. + preSharedCerts: [] + +# Helm replaces lists, so each backend's allowed sources are restated here +# with the proxy-only subnet added. Keep the two Google health-check ranges. +# REQUIRED OVERRIDE: replace 10.129.0.0/23 in all five lists with your range. +relaycast: + networkPolicy: + ingressFrom: + - podSelector: {} + - ipBlock: { cidr: 35.191.0.0/16 } + - ipBlock: { cidr: 130.211.0.0/22 } + - ipBlock: { cidr: 10.129.0.0/23 } +relay-control: + networkPolicy: + ingressFrom: + - podSelector: {} + - ipBlock: { cidr: 35.191.0.0/16 } + - ipBlock: { cidr: 130.211.0.0/22 } + - ipBlock: { cidr: 10.129.0.0/23 } +relay-webapp: + networkPolicy: + ingressFrom: + - podSelector: {} + - ipBlock: { cidr: 35.191.0.0/16 } + - ipBlock: { cidr: 130.211.0.0/22 } + - ipBlock: { cidr: 10.129.0.0/23 } +sessions-pipeline: + sessionReadApi: + networkPolicy: + publicFrom: + - ipBlock: { cidr: 35.191.0.0/16 } + - ipBlock: { cidr: 130.211.0.0/22 } + - ipBlock: { cidr: 10.129.0.0/23 } + ingestEdge: + networkPolicy: + ingressFrom: + - ipBlock: { cidr: 35.191.0.0/16 } + - ipBlock: { cidr: 130.211.0.0/22 } + - ipBlock: { cidr: 10.129.0.0/23 } diff --git a/charts/agent-relay/values.yaml b/charts/agent-relay/values.yaml index d8e9657c..0dc70c50 100644 --- a/charts/agent-relay/values.yaml +++ b/charts/agent-relay/values.yaml @@ -149,6 +149,22 @@ gkeIngress: # front of the load balancer, where Google cannot issue a managed # certificate. The certificate must cover every host below. tlsSecretName: "" + # external (default): a global external Application Load Balancer. + # internal: a regional internal one (Ingress class gce-internal) that + # answers only inside the VPC and the networks routed to it. Start from + # values-gcp-internal.yaml; README: Internal-only GKE edge. In internal + # mode staticIpName names a regional internal address, the certificate is + # tlsSecretName or internal.preSharedCerts, managedCertificateName and + # frontendConfigName are unused, and sslPolicy and securityPolicy are + # refused (GKE supports neither on an internal Ingress). + mode: external + internal: + # Range of the VPC's proxy-only subnet (purpose REGIONAL_MANAGED_PROXY) in + # the cluster's region. Required in internal mode, and every backend's + # networkPolicy must allow it. + proxyOnlySubnetCidr: "" + # Regional self-managed SSL certificate names, instead of tlsSecretName. + preSharedCerts: [] hosts: root: "" api: "" diff --git a/ci/agent-relay/render.sh b/ci/agent-relay/render.sh index 1d9a0a44..99dd770e 100755 --- a/ci/agent-relay/render.sh +++ b/ci/agent-relay/render.sh @@ -964,6 +964,69 @@ if edge_err=$(helm template agent-relay charts/agent-relay -n agent-relay --no-h --set gkeIngress.securityPolicy=relay-edge --set-json "$edge_dashboard" 2>&1); then edge_fail "securityPolicy rendered without the relayfile backend-config annotation"; fi grep -Fq "gkeIngress.securityPolicy requires relayfile.service.annotations[cloud.google.com/backend-config]" <<<"$edge_err" || edge_fail "unexpected error without the relayfile annotation: $edge_err" +# Internal-only GKE edge (gkeIngress.mode=internal, values-gcp-internal.yaml): +# a regional internal load balancer, HTTPS only, no ManagedCertificate or +# FrontendConfig, and every backend policy admits the proxy-only subnet. +int_fail() { echo "gke internal edge: $*" >&2; exit 1; } +int_args=(-n agent-relay --no-hooks -f charts/agent-relay/values-gcp-proof.yaml -f charts/agent-relay/ci/gcp-proof-values.yaml -f charts/agent-relay/values-gcp-internal.yaml) +int_out=$(helm template proof charts/agent-relay "${int_args[@]}") +test "$(yq 'select(.kind == "Ingress") | .metadata.annotations | [.["kubernetes.io/ingress.class"], .["kubernetes.io/ingress.regional-static-ip-name"], .["kubernetes.io/ingress.allow-http"]] | join(" ")' <<<"$int_out" | sed '/^---$/d;/^$/d')" = "gce-internal example-agent-relay-internal false" \ + || int_fail "Ingress must be class gce-internal with the regional address and HTTP closed" +test "$(yq 'select(.kind == "Ingress") | .metadata.annotations | [has("kubernetes.io/ingress.global-static-ip-name"), has("networking.gke.io/managed-certificates"), has("networking.gke.io/v1beta1.FrontendConfig"), has("ingress.gcp.kubernetes.io/pre-shared-cert")] | join(" ")' <<<"$int_out" | sed '/^---$/d;/^$/d')" = "false false false false" \ + || int_fail "Ingress must carry no external-only annotation" +test "$(yq 'select(.kind == "Ingress") | .spec.tls[0].secretName' <<<"$int_out")" = example-agent-relay-tls || int_fail "Ingress must serve the TLS Secret" +test -z "$(yq 'select(.kind == "ManagedCertificate" or .kind == "FrontendConfig") | .kind' <<<"$int_out")" || int_fail "ManagedCertificate and FrontendConfig must not render" +# The routes are the same as on the external edge. +int_rules=$(yq -o=json -I=0 'select(.kind == "Ingress") | .spec.rules' <<<"$int_out" | sed '/^---$/d;/^$/d;/^null$/d') +test "$(grep -c '"host"' <<<"$int_rules")" = 1 || int_fail "expected one Ingress with rules" +# Every host, path, pathType, backend Service and port is identical to the external edge. +test "$int_rules" = "$(yq -o=json -I=0 'select(.kind == "Ingress") | .spec.rules' <<<"$edge_default" | sed '/^---$/d;/^$/d;/^null$/d')" \ + || int_fail "internal mode changed the Ingress rules" +# Every NetworkPolicy that admits the Google health-check range also admits the proxy-only subnet. +int_gaps=$(yq 'select(.kind == "NetworkPolicy") | select([.spec.ingress[]?.from[]?.ipBlock.cidr] | contains(["35.191.0.0/16"])) | select([.spec.ingress[]?.from[]?.ipBlock.cidr] | contains(["10.129.0.0/23"]) | not) | .metadata.name' <<<"$int_out" | sed '/^---$/d;/^$/d') +test -z "$int_gaps" || int_fail "NetworkPolicy without the proxy-only subnet: $int_gaps" +test "$(yq 'select(.kind == "NetworkPolicy") | select([.spec.ingress[]?.from[]?.ipBlock.cidr] | contains(["10.129.0.0/23"])) | .metadata.name' <<<"$int_out" | grep -c '[a-z]')" -ge 5 || int_fail "expected at least five backend policies to admit the proxy-only subnet" +# Pre-shared regional certificates instead of a Secret. +int_psc=$(helm template proof charts/agent-relay "${int_args[@]}" --set gkeIngress.tlsSecretName= --set 'gkeIngress.internal.preSharedCerts={relay-cert-a,relay-cert-b}') +test "$(yq 'select(.kind == "Ingress") | .metadata.annotations["ingress.gcp.kubernetes.io/pre-shared-cert"] + " " + ((.spec | has("tls")) | tostring)' <<<"$int_psc")" = "relay-cert-a,relay-cert-b false" \ + || int_fail "preSharedCerts must render the pre-shared-cert annotation and no spec.tls" +expect_internal_failure() { # expected-error helm-args... + local expected=$1 output; shift + if output=$(helm template proof charts/agent-relay "${int_args[@]}" "$@" 2>&1); then int_fail "rendered despite expected failure: $expected"; fi + grep -Fq "$expected" <<<"$output" || { echo "$output" >&2; int_fail "failed without expected error: $expected"; } +} +expect_internal_failure "gkeIngress.mode must be external or internal" --set gkeIngress.mode=private +expect_internal_failure "requires gkeIngress.internal.proxyOnlySubnetCidr" --set gkeIngress.internal.proxyOnlySubnetCidr= +expect_internal_failure "requires gkeIngress.internal.proxyOnlySubnetCidr" --set gkeIngress.internal.proxyOnlySubnetCidr=203.0.113.0/24 +for bad in 10.129.999.0/23 172.16.0.0.1/24 10.129.0/23 10.129.0.0/27 10.129.0.0 172.32.0.0/16; do + expect_internal_failure "requires gkeIngress.internal.proxyOnlySubnetCidr" --set "gkeIngress.internal.proxyOnlySubnetCidr=$bad" +done +expect_internal_failure "gkeIngress.mode=internal requires a certificate" --set gkeIngress.tlsSecretName= +expect_internal_failure "set only one of gkeIngress.tlsSecretName and gkeIngress.internal.preSharedCerts" --set 'gkeIngress.internal.preSharedCerts={relay-cert}' +expect_internal_failure "is not a Compute Engine SSL certificate name" --set gkeIngress.tlsSecretName= --set 'gkeIngress.internal.preSharedCerts={Bad_Cert}' +expect_internal_failure "gkeIngress.securityPolicy cannot be used with gkeIngress.mode=internal" --set gkeIngress.securityPolicy=relay-edge --set-json "$edge_dashboard" +expect_internal_failure "gkeIngress.sslPolicy cannot be used with gkeIngress.mode=internal" --set gkeIngress.sslPolicy=relay-tls +# The overlay is applied last and clears an sslPolicy that an earlier file set +# (the GKE guide's managed-values.json names one). +int_managed=$(mktemp) +printf 'gkeIngress:\n sslPolicy: relay-tls\n' >"$int_managed" +helm template proof charts/agent-relay -n agent-relay --no-hooks -f charts/agent-relay/values-gcp-proof.yaml -f charts/agent-relay/ci/gcp-proof-values.yaml \ + -f "$int_managed" -f charts/agent-relay/values-gcp-internal.yaml >/dev/null || int_fail "the overlay must clear an sslPolicy set by an earlier values file" +rm -f "$int_managed" +# A backend whose policy does not admit the proxy-only subnet is refused, by name. +expect_internal_failure "gkeIngress.mode=internal requires relay-webapp.networkPolicy.ingressFrom to allow the proxy-only subnet 10.129.0.0/23" \ + --set-json 'relay-webapp.networkPolicy.ingressFrom=[{"podSelector":{}},{"ipBlock":{"cidr":"35.191.0.0/16"}},{"ipBlock":{"cidr":"130.211.0.0/22"}}]' +expect_internal_failure "gkeIngress.mode=internal requires sessions-pipeline.ingestEdge.networkPolicy.ingressFrom to allow the proxy-only subnet 10.129.0.0/23" \ + --set-json 'sessions-pipeline.ingestEdge.networkPolicy.ingressFrom=[{"ipBlock":{"cidr":"35.191.0.0/16"}},{"ipBlock":{"cidr":"130.211.0.0/22"}}]' +expect_internal_failure "to allow the proxy-only subnet 10.129.0.0/23 (gkeIngress.internal.proxyOnlySubnetCidr) without exclusions" \ + --set-json 'relaycast.networkPolicy.ingressFrom=[{"podSelector":{}},{"ipBlock":{"cidr":"35.191.0.0/16"}},{"ipBlock":{"cidr":"130.211.0.0/22"}},{"ipBlock":{"cidr":"10.129.0.0/23","except":["10.129.0.0/24"]}}]' +# internal.* without the mode is refused, so a half-applied overlay cannot pass silently. +expect_umbrella_failure "gkeIngress.internal.* is set but gkeIngress.mode is external" --set gkeIngress.internal.proxyOnlySubnetCidr=10.129.0.0/23 +# The values-gcp.yaml profile renders with the overlay too. +helm template proof charts/agent-relay -n agent-relay --no-hooks -f charts/agent-relay/values-gcp.yaml -f charts/agent-relay/values-gcp-native.yaml \ + -f charts/agent-relay/ci/gcp-native-values.yaml -f charts/agent-relay/values-gcp-internal.yaml >/dev/null 2>/tmp/int-gcp.err \ + || { cat /tmp/int-gcp.err >&2; int_fail "values-gcp.yaml with values-gcp-internal.yaml did not render"; } + # With single-host mode off, every default (subdomain) render is unchanged. ci/agent-relay/subdomain-golden.sh check diff --git a/docs/gcp-install/3-EDGE-HARDENING.md b/docs/gcp-install/3-EDGE-HARDENING.md index b47732b2..05cad492 100644 --- a/docs/gcp-install/3-EDGE-HARDENING.md +++ b/docs/gcp-install/3-EDGE-HARDENING.md @@ -8,6 +8,7 @@ after step 14 of [0-START-HERE.md](0-START-HERE.md), in any order. | [TLS policy](#1-tls-policy) | Refuses TLS 1.0 and 1.1 | **On** when this release's Terraform (step 4) created your resources | | [Cloud Armor](#2-cloud-armor) | OWASP rules, a sign-in rate limit, and optionally an allowlist of source addresses | Off | | [A proxy in front (Cloudflare)](#3-cloudflare-in-front) | Only the proxy may reach the load balancer | Off | +| [Internal only (VPN)](#4-internal-only-behind-your-vpn) | No public address at all | Off | **Which release.** These controls need the `agent-relay` chart 0.7.59 or later and the Terraform from the same release. The release that @@ -263,6 +264,78 @@ curl -sk -o /dev/null -w "direct %{http_code}\n" --resolve "api.$DOMAIN:443:$(gc `terraform apply`, compare with `cloudflare.ipv4_cidrs` in the module (the list there is dated). +## 4. Internal only, behind your VPN + +Needs the `agent-relay` chart 0.7.60 or later (an earlier chart ignores +`gkeIngress.mode` and would create a public load balancer). + +Instead of a public load balancer, the install can have a private address +that only your VPC and the networks routed to it (VPN, Interconnect) can +reach. Decide this before step 6 of the guide: the address, DNS and +certificate all differ. Agents and desktops that are not on the VPN cannot +connect or upload sessions. + +1. **Network.** Use `terraform/modules/gcp/internal-edge` (its README has the + call). It creates a proxy-only subnet (pick a free /23 in the VPC), a + reserved private address, and the firewall rule that lets the load + balancer's proxies reach the nodes. This replaces step 6's + `gcloud compute addresses create ... --global`. + +2. **DNS.** Create A records for ``, `api.`, `cast.` + and `history.` pointing at the module's `address` output, in a zone + your VPN clients resolve. + +3. **Certificate.** Google cannot issue a managed certificate for a private + address. Create a `kubernetes.io/tls` Secret for the four names (for + example with cert-manager and a DNS-01 challenge, which needs no inbound + access), or upload a regional certificate: + + ```sh + gcloud compute ssl-certificates create agent-relay-internal --region $REGION \ + --certificate fullchain.pem --private-key privkey.pem + ``` + +4. **Values.** Copy `values-gcp-internal.yaml` from the chart, replace + `10.129.0.0/23` with your proxy-only range in all six places, set + `staticIpName` and the certificate, then merge it into `values.yaml`: + + ```sh + cp "$CHART/values-gcp-internal.yaml" internal.yaml + # edit internal.yaml, then: + yq -i '. *= load("internal.yaml")' values.yaml + yq '.gkeIngress.mode, .gkeIngress.internal.proxyOnlySubnetCidr' values.yaml + ``` + + The last command prints `internal` and your range. The settings go into + `values.yaml`, not a separate `-f` file, because every `helm upgrade` in + this guide (steps 11, 13, 16, 17 and each later upgrade) reads + `values.yaml`. Helm keeps nothing from the previous command: an upgrade + without these settings would switch the install back to a public load + balancer and drop the proxy-only range from the NetworkPolicies. + + The file also clears `gkeIngress.sslPolicy` and `securityPolicy`: GKE + cannot apply them to an internal load balancer, so sections 1 and 2 above + do not apply to this mode. + +**Done when**, from a machine on the VPN: + +```sh +kubectl -n $NAMESPACE get ingress agent-relay -o jsonpath='{.metadata.annotations.kubernetes\.io/ingress\.class} {.status.loadBalancer.ingress[0].ip}{"\n"}' +curl -fsS "https://api.$DOMAIN/readyz" && echo " api ok" +``` + +prints `gce-internal` with a private address, then `api ok`; and from a +machine that is **not** on the VPN the same `curl` cannot connect. Step 14's +other checks apply unchanged (skip the `managedcertificate` one). + +- **Requests hang, then 504:** the firewall rule from the proxy-only subnet + is missing, or a backend's `networkPolicy` does not list the range. +- **`Error syncing to GCP ... proxy-only subnetwork`** in + `kubectl describe ingress`: the VPC has no active proxy-only subnet in the + cluster's region. +- Sign-in rate limits: check `trustedProxies` as in section 3, item 4. The + internal load balancer's `X-Forwarded-For` has not been measured. + ## Removing these **Cloud Armor.** Clearing `gkeIngress.securityPolicy` alone does not detach diff --git a/docs/self-hosted-requirements.md b/docs/self-hosted-requirements.md index e16fa64a..dad595af 100644 --- a/docs/self-hosted-requirements.md +++ b/docs/self-hosted-requirements.md @@ -14,7 +14,7 @@ registered, every shipped entry must exist in the charts, planned entries must not be referenced by any template, and the compatibility matrix must equal the pinned digests. -Source of truth: [`self-hosted/requirements.yaml`](../self-hosted/requirements.yaml). 200 entries: 55 requirements (55 shipped, 0 planned), 145 options (144 shipped, 1 planned). +Source of truth: [`self-hosted/requirements.yaml`](../self-hosted/requirements.yaml). 201 entries: 55 requirements (55 shipped, 0 planned), 146 options (145 shipped, 1 planned). ## Contents @@ -167,7 +167,8 @@ Services image (sessions-pipeline-worker, alleyoop and the privileged SQL Jobs): | relayauth-0.1.2 | shipped | relayauth 0.1.2 | relayauth: `ghcr.io/agentworkforce/relayauth@sha256:c23c92d25fdf…` | | | License-annotation patch bump of 0.1.1. Superseded by 0.1.3. | | relayauth-0.1.1 | shipped | relayauth 0.1.1 | relayauth: `ghcr.io/agentworkforce/relayauth@sha256:c23c92d25fdf…` | | | relayauth 784582ef (machine-bound History assertion contract, relayauth #113). Superseded by 0.1.2. | | relayauth-0.1.0 | shipped | relayauth 0.1.0 | relayauth: `ghcr.io/agentworkforce/relayauth@sha256:eb71daa28e30…` | | | superseded by 0.1.1. | -| agent-relay-0.7.59 | unreleased | agent-relay 0.7.59
relay-control 0.4.22
relay-webapp 0.2.7
(deps: relaycast 0.3.16, relayauth 0.1.5, sessions-pipeline 0.8.49, relay-control 0.4.22, relay-webapp 0.2.7, relayflows 0.2.5, durable-actors 0.14.6-awf.4, relayfile 0.2.2, telemetry-collector 0.1.7, relay-chief 0.1.0, slack-relay 0.3.11, relaygate 0.1.10) | relaycast: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
relaycast-postgres: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
sessions-pipeline-worker: `ghcr.io/agentworkforce/sessions-pipeline@sha256:77022c5b6a1d…`
relay-control: `ghcr.io/agentworkforce/relay-control@sha256:d1b6ca95fab2…`
relay-webapp: `ghcr.io/agentworkforce/relay-webapp@sha256:bd295ccb7d13…` | | | Carries 0.7.57 (0.7.58 is taken by another open change). GKE edge hardening, all opt-in and empty by default: gkeIngress.sslPolicy names a Compute SSL policy that the FrontendConfig applies to the HTTPS listener (without one Google's default policy still accepts TLS 1.0 and 1.1); gkeIngress.securityPolicy names a Cloud Armor backend security policy set on the BackendConfig of every backend behind the Ingress (api, cast, both history edges, the actors and relaygate edges, and new policy-only BackendConfigs -dashboard, -edge-deny and -files; the relay-webapp and relayfile Services need their backend-config annotation and the render fails with the exact value otherwise); gkeIngress.tlsSecretName serves a kubernetes.io/tls Secret instead of the ManagedCertificate, for an edge whose DNS points at a proxy or CDN. The resources come from terraform/modules/gcp/edge-security; terraform/examples/gcp now creates the TLS 1.2 SSL policy by default and exports it in chart_values. Default renders differ only in the helm.sh/chart labels (subdomain golden unchanged); images unchanged. | +| agent-relay-0.7.60 | unreleased | agent-relay 0.7.60
relay-control 0.4.22
relay-webapp 0.2.7
(deps: relaycast 0.3.16, relayauth 0.1.5, sessions-pipeline 0.8.49, relay-control 0.4.22, relay-webapp 0.2.7, relayflows 0.2.5, durable-actors 0.14.6-awf.4, relayfile 0.2.2, telemetry-collector 0.1.7, relay-chief 0.1.0, slack-relay 0.3.11, relaygate 0.1.10) | relaycast: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
relaycast-postgres: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
sessions-pipeline-worker: `ghcr.io/agentworkforce/sessions-pipeline@sha256:77022c5b6a1d…`
relay-control: `ghcr.io/agentworkforce/relay-control@sha256:d1b6ca95fab2…`
relay-webapp: `ghcr.io/agentworkforce/relay-webapp@sha256:bd295ccb7d13…` | | | Carries 0.7.59. Internal-only GKE edge, opt-in: gkeIngress.mode=internal (default external) renders the Ingress with class gce-internal, a regional static internal address (gkeIngress.staticIpName) and HTTP closed, for a regional internal Application Load Balancer that answers only inside the VPC and the networks routed to it. No ManagedCertificate or FrontendConfig is rendered (GKE Ingress supports neither internally): the certificate is gkeIngress.tlsSecretName or gkeIngress.internal.preSharedCerts (regional certificates). gkeIngress.internal.proxyOnlySubnetCidr is required and every backend NetworkPolicy (and the durable-actors policy) must admit it; the render refuses otherwise, and also refuses sslPolicy or securityPolicy in internal mode and internal.* in external mode. values-gcp-internal.yaml is the overlay; terraform/modules/gcp/internal-edge creates the proxy-only subnet, the address and the firewall rule. Default renders differ only in the helm.sh/chart labels; images unchanged. | +| agent-relay-0.7.59 | **planned** | agent-relay 0.7.59
relay-control 0.4.22
relay-webapp 0.2.7
(deps: relaycast 0.3.16, relayauth 0.1.5, sessions-pipeline 0.8.49, relay-control 0.4.22, relay-webapp 0.2.7, relayflows 0.2.5, durable-actors 0.14.6-awf.4, relayfile 0.2.2, telemetry-collector 0.1.7, relay-chief 0.1.0, slack-relay 0.3.11, relaygate 0.1.10) | relaycast: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
relaycast-postgres: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
sessions-pipeline-worker: `ghcr.io/agentworkforce/sessions-pipeline@sha256:77022c5b6a1d…`
relay-control: `ghcr.io/agentworkforce/relay-control@sha256:d1b6ca95fab2…`
relay-webapp: `ghcr.io/agentworkforce/relay-webapp@sha256:bd295ccb7d13…` | | | Carries 0.7.57 (0.7.58 is taken by another open change). GKE edge hardening, all opt-in and empty by default: gkeIngress.sslPolicy names a Compute SSL policy that the FrontendConfig applies to the HTTPS listener (without one Google's default policy still accepts TLS 1.0 and 1.1); gkeIngress.securityPolicy names a Cloud Armor backend security policy set on the BackendConfig of every backend behind the Ingress (api, cast, both history edges, the actors and relaygate edges, and new policy-only BackendConfigs -dashboard, -edge-deny and -files; the relay-webapp and relayfile Services need their backend-config annotation and the render fails with the exact value otherwise); gkeIngress.tlsSecretName serves a kubernetes.io/tls Secret instead of the ManagedCertificate, for an edge whose DNS points at a proxy or CDN. The resources come from terraform/modules/gcp/edge-security; terraform/examples/gcp now creates the TLS 1.2 SSL policy by default and exports it in chart_values. Default renders differ only in the helm.sh/chart labels (subdomain golden unchanged); images unchanged. | | agent-relay-0.7.57 | **planned** | agent-relay 0.7.57
relay-control 0.4.22
relay-webapp 0.2.7
(deps: relaycast 0.3.16, relayauth 0.1.5, sessions-pipeline 0.8.49, relay-control 0.4.22, relay-webapp 0.2.7, relayflows 0.2.5, durable-actors 0.14.6-awf.4, relayfile 0.2.2, telemetry-collector 0.1.7, relay-chief 0.1.0, slack-relay 0.3.11, relaygate 0.1.10) | relaycast: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
relaycast-postgres: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
sessions-pipeline-worker: `ghcr.io/agentworkforce/sessions-pipeline@sha256:77022c5b6a1d…`
relay-control: `ghcr.io/agentworkforce/relay-control@sha256:d1b6ca95fab2…`
relay-webapp: `ghcr.io/agentworkforce/relay-webapp@sha256:bd295ccb7d13…` | | | Carries 0.7.56. Opt-in externalSecrets (default off): renders one External Secrets Operator ExternalSecret (external-secrets.io/v1) per externalSecrets.secrets entry, targeting the Kubernetes Secret of that name, so install Secrets can come from AWS SSM Parameter Store, AWS Secrets Manager or another ESO provider without chart edits; store.create renders a namespaced SecretStore from store.provider. Default renders differ only in the helm.sh/chart labels. | | agent-relay-0.7.56 | **planned** | agent-relay 0.7.56
relay-control 0.4.22
relay-webapp 0.2.7
(deps: relaycast 0.3.16, relayauth 0.1.5, sessions-pipeline 0.8.49, relay-control 0.4.22, relay-webapp 0.2.7, relayflows 0.2.5, durable-actors 0.14.6-awf.4, relayfile 0.2.2, telemetry-collector 0.1.7, relay-chief 0.1.0, slack-relay 0.3.11, relaygate 0.1.10) | relaycast: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
relaycast-postgres: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
sessions-pipeline-worker: `ghcr.io/agentworkforce/sessions-pipeline@sha256:77022c5b6a1d…`
relay-control: `ghcr.io/agentworkforce/relay-control@sha256:d1b6ca95fab2…`
relay-webapp: `ghcr.io/agentworkforce/relay-webapp@sha256:bd295ccb7d13…` | | | Carries sessions-pipeline 0.8.49 (searchIndexer default memory sized for the streaming reconcile: request 1Gi, limit 3Gi). | | agent-relay-0.7.55 | **planned** | agent-relay 0.7.55
relay-control 0.4.22
relay-webapp 0.2.7
(deps: relaycast 0.3.16, relayauth 0.1.5, sessions-pipeline 0.8.48, relay-control 0.4.22, relay-webapp 0.2.7, relayflows 0.2.5, durable-actors 0.14.6-awf.4, relayfile 0.2.2, telemetry-collector 0.1.7, relay-chief 0.1.0, slack-relay 0.3.11, relaygate 0.1.10) | relaycast: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
relaycast-postgres: `ghcr.io/agentworkforce/relaycast-self-host@sha256:ec062ef901af…`
sessions-pipeline-worker: `ghcr.io/agentworkforce/sessions-pipeline@sha256:77022c5b6a1d…`
relay-control: `ghcr.io/agentworkforce/relay-control@sha256:d1b6ca95fab2…`
relay-webapp: `ghcr.io/agentworkforce/relay-webapp@sha256:bd295ccb7d13…` | | | Carries 0.7.54 and sessions-pipeline 0.8.48: opt-in ingestEdge.admission.spread, ingestEdge.admission.spreadGraceMs and ingestEdge.livenessTimeoutSeconds. Default renders differ only in the helm.sh/chart labels. | @@ -1204,6 +1205,7 @@ chart `agent-relay` · values [`values-gcp-proof.yaml`](../charts/agent-relay/va | shipped | [Single-host mode for the dashboard and the control API](#opt-agent-relay-single-host) | `disabled`, `enabled` | `disabled` | values `global.singleHost.host`, `relay-control:global.singleHost.host`, `relay-control:publicUrl`, `relay-webapp:global.singleHost.host`, `relay-webapp:runtime.publicUrl`, `relay-webapp:ingress.host`; env `SELFHOST_PUBLIC_URL`, `DASHBOARD_PUBLIC_URL` | | shipped | [Single GKE ingress for the proof hosts](#opt-agent-relay-gke-ingress) | `enabled`, `disabled` | `disabled` | values `gkeIngress.enabled`, `gkeIngress.staticIpName`, `gkeIngress.managedCertificateName`, `gkeIngress.frontendConfigName`, `gkeIngress.hosts.*`, `gkeIngress.backends.*`, `relayauth.jwksIngress.enabled` | | shipped | [GKE edge TLS policy, Cloud Armor and certificate from a Secret](#opt-agent-relay-gke-edge-hardening) | `none`, `sslPolicy`, `securityPolicy`, `tlsSecretName` | `none` | values `gkeIngress.sslPolicy`, `gkeIngress.securityPolicy`, `gkeIngress.tlsSecretName` | +| shipped | [Internal-only GKE edge (regional internal load balancer)](#opt-agent-relay-gke-internal-edge) | `external`, `internal` | `external` | values `gkeIngress.mode`, `gkeIngress.internal.proxyOnlySubnetCidr`, `gkeIngress.internal.preSharedCerts`, `gkeIngress.tlsSecretName` | | shipped | [Optional Relayflows component in the suite](#opt-agent-relay-relayflows) | `enabled`, `disabled` | `disabled` | values `components.relayflows`, `relayflows.*` | | shipped | [Control legacy RelayAuth client environment](#opt-control-legacy-relayauth) | `true`, `false` | `true` | values `services.relayauthEnabled`; env `RELAYAUTH_URL`, `RELAYAUTH_API_KEY_FILE` | | shipped | [Native pipeline collector adapters](#opt-native-pipeline-analytics) | `false`, `true` | `false` | values `ingestEdge.analytics.*`, `nativeConsumer.analytics.*`, `session-read-api:analytics.*`; env `ANALYTICS_SINK`, `ANALYTICS_DEPLOYMENT_ID`, `ANALYTICS_COLLECTOR_URL`, `ANALYTICS_COLLECTOR_TOKEN`, `NODE_EXTRA_CA_CERTS`; Secret `` keys `token`; Secret `` keys `token`; Secret `` keys `token` | @@ -3569,6 +3571,36 @@ allowlist, Cloudflare origin lockdown). - Verify: render with the values set shows spec.sslPolicy on the FrontendConfig, spec.securityPolicy.name on every BackendConfig, every Ingress backend Service bound to one of them, and spec.tls without a ManagedCertificate - Sources: [`charts/agent-relay/templates/gke-ingress.yaml`](../charts/agent-relay/templates/gke-ingress.yaml), [`charts/agent-relay/values.yaml`](../charts/agent-relay/values.yaml), [`ci/agent-relay/render.sh`](../ci/agent-relay/render.sh), [`terraform/modules/gcp/edge-security/main.tf`](../terraform/modules/gcp/edge-security/main.tf) +#### opt.agent-relay.gke-internal-edge + + + +**Internal-only GKE edge (regional internal load balancer)** · shipped · chart `agent-relay` · profiles agent-relay + +gkeIngress.mode=internal renders the umbrella Ingress for a regional +internal Application Load Balancer (Ingress class gce-internal): a +private address, reachable only from the VPC and the networks routed to +it (VPN, Interconnect), HTTPS only. It is the GKE counterpart of an +internal-scheme ALB on EKS. gkeIngress.staticIpName then names a regional +internal address. GKE Ingress supports neither ManagedCertificate, +FrontendConfig nor Cloud Armor on an internal load balancer, so neither +is rendered, the certificate comes from gkeIngress.tlsSecretName or +gkeIngress.internal.preSharedCerts (regional self-managed certificates), +and gkeIngress.sslPolicy and gkeIngress.securityPolicy are refused. The +load balancer's proxies connect to the pods from the VPC's proxy-only +subnet: gkeIngress.internal.proxyOnlySubnetCidr is required, and every +backend's networkPolicy must admit that range next to the two Google +health-check ranges (values-gcp-internal.yaml restates the lists). +terraform/modules/gcp/internal-edge creates the proxy-only subnet, the +address and the firewall rule from the subnet to the nodes. + +- Default: `external` +- Choices: `external`: global external Application Load Balancer; `internal`: regional internal Application Load Balancer; values-gcp-internal.yaml +- Provided by: operator +- Values: `gkeIngress.mode`, `gkeIngress.internal.proxyOnlySubnetCidr`, `gkeIngress.internal.preSharedCerts`, `gkeIngress.tlsSecretName` +- Verify: render with values-gcp-internal.yaml shows one Ingress with class gce-internal, the regional-static-ip-name annotation and allow-http false, no ManagedCertificate or FrontendConfig, and the proxy-only subnet in every backend NetworkPolicy +- Sources: [`charts/agent-relay/templates/gke-ingress.yaml`](../charts/agent-relay/templates/gke-ingress.yaml), [`charts/agent-relay/values-gcp-internal.yaml`](../charts/agent-relay/values-gcp-internal.yaml), [`ci/agent-relay/render.sh`](../ci/agent-relay/render.sh), [`terraform/modules/gcp/internal-edge/main.tf`](../terraform/modules/gcp/internal-edge/main.tf) + #### opt.agent-relay.relayflows diff --git a/self-hosted/requirements.yaml b/self-hosted/requirements.yaml index 2d5bf01d..b9e20f45 100644 --- a/self-hosted/requirements.yaml +++ b/self-hosted/requirements.yaml @@ -1956,9 +1956,22 @@ compatibility: images: relayauth: { repository: ghcr.io/agentworkforce/relayauth, digest: sha256:eb71daa28e30eb7305f666d0bccc5a4002e53e8a574e3c4c32163d907a1b71df } notes: superseded by 0.1.1. - - id: agent-relay-0.7.59 + - id: agent-relay-0.7.60 status: unreleased current: true + charts: { agent-relay: 0.7.60, relay-control: 0.4.22, relay-webapp: 0.2.7 } + app_versions: { agent-relay: 0.1.0, relay-control: 0.1.0, relay-webapp: 0.1.0 } + depends: + agent-relay: { relaycast: 0.3.16, relayauth: 0.1.5, sessions-pipeline: 0.8.49, relay-control: 0.4.22, relay-webapp: 0.2.7, relayflows: 0.2.5, durable-actors: 0.14.6-awf.4, relayfile: 0.2.2, telemetry-collector: 0.1.7, relay-chief: 0.1.0, slack-relay: 0.3.11, relaygate: 0.1.10 } + images: + relaycast: { repository: ghcr.io/agentworkforce/relaycast-self-host, digest: sha256:ec062ef901af15cf575b53a71d7eb09f2e6ab6e7c0d5be2da35339a275a5c6cc } + relaycast-postgres: { repository: ghcr.io/agentworkforce/relaycast-self-host, digest: sha256:ec062ef901af15cf575b53a71d7eb09f2e6ab6e7c0d5be2da35339a275a5c6cc } + sessions-pipeline-worker: { repository: ghcr.io/agentworkforce/sessions-pipeline, digest: sha256:77022c5b6a1d30f96f921d703eb27a1acada8ec273ad84aebdde24f8eddce305 } + relay-control: { repository: ghcr.io/agentworkforce/relay-control, digest: sha256:d1b6ca95fab240a9d41208d83de3b1f4245408b0765038762f1c419ee60f43bc } + relay-webapp: { repository: ghcr.io/agentworkforce/relay-webapp, digest: sha256:bd295ccb7d1323965dce890f19d0b7f857d706c00d7756f719c713763fdb49f3 } + notes: 'Carries 0.7.59. Internal-only GKE edge, opt-in: gkeIngress.mode=internal (default external) renders the Ingress with class gce-internal, a regional static internal address (gkeIngress.staticIpName) and HTTP closed, for a regional internal Application Load Balancer that answers only inside the VPC and the networks routed to it. No ManagedCertificate or FrontendConfig is rendered (GKE Ingress supports neither internally): the certificate is gkeIngress.tlsSecretName or gkeIngress.internal.preSharedCerts (regional certificates). gkeIngress.internal.proxyOnlySubnetCidr is required and every backend NetworkPolicy (and the durable-actors policy) must admit it; the render refuses otherwise, and also refuses sslPolicy or securityPolicy in internal mode and internal.* in external mode. values-gcp-internal.yaml is the overlay; terraform/modules/gcp/internal-edge creates the proxy-only subnet, the address and the firewall rule. Default renders differ only in the helm.sh/chart labels; images unchanged.' + - id: agent-relay-0.7.59 + status: planned charts: { agent-relay: 0.7.59, relay-control: 0.4.22, relay-webapp: 0.2.7 } app_versions: { agent-relay: 0.1.0, relay-control: 0.1.0, relay-webapp: 0.1.0 } depends: @@ -5887,6 +5900,46 @@ entries: - "ci/agent-relay/render.sh" - "terraform/modules/gcp/edge-security/main.tf" + - id: opt.agent-relay.gke-internal-edge + kind: option + title: Internal-only GKE edge (regional internal load balancer) + description: | + gkeIngress.mode=internal renders the umbrella Ingress for a regional + internal Application Load Balancer (Ingress class gce-internal): a + private address, reachable only from the VPC and the networks routed to + it (VPN, Interconnect), HTTPS only. It is the GKE counterpart of an + internal-scheme ALB on EKS. gkeIngress.staticIpName then names a regional + internal address. GKE Ingress supports neither ManagedCertificate, + FrontendConfig nor Cloud Armor on an internal load balancer, so neither + is rendered, the certificate comes from gkeIngress.tlsSecretName or + gkeIngress.internal.preSharedCerts (regional self-managed certificates), + and gkeIngress.sslPolicy and gkeIngress.securityPolicy are refused. The + load balancer's proxies connect to the pods from the VPC's proxy-only + subnet: gkeIngress.internal.proxyOnlySubnetCidr is required, and every + backend's networkPolicy must admit that range next to the two Google + health-check ranges (values-gcp-internal.yaml restates the lists). + terraform/modules/gcp/internal-edge creates the proxy-only subnet, the + address and the firewall rule from the subnet to the nodes. + chart: agent-relay + values: + - gkeIngress.mode + - gkeIngress.internal.proxyOnlySubnetCidr + - gkeIngress.internal.preSharedCerts + - gkeIngress.tlsSecretName + profiles: [agent-relay] + status: shipped + provided_by: operator + default: external + choices: + - { value: external, note: "global external Application Load Balancer" } + - { value: internal, note: "regional internal Application Load Balancer; values-gcp-internal.yaml" } + verify: render with values-gcp-internal.yaml shows one Ingress with class gce-internal, the regional-static-ip-name annotation and allow-http false, no ManagedCertificate or FrontendConfig, and the proxy-only subnet in every backend NetworkPolicy + sources: + - "charts/agent-relay/templates/gke-ingress.yaml" + - "charts/agent-relay/values-gcp-internal.yaml" + - "ci/agent-relay/render.sh" + - "terraform/modules/gcp/internal-edge/main.tf" + - id: opt.agent-relay.relayflows kind: option title: Optional Relayflows component in the suite diff --git a/terraform/README.md b/terraform/README.md index 62462191..0d81a84c 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -16,6 +16,7 @@ terraform/ modules/aws/relay-chief-slots # EC2 hosts for Relay Chief modules/gcp/{foundation,postgres,object-storage,search,functions} modules/gcp/edge-security # GKE Ingress TLS policy and Cloud Armor + modules/gcp/internal-edge # prerequisites for the internal-only GKE Ingress examples/{aws,gcp} # thin roots: four capabilities together examples/{aws,gcp}-foundation # network and Kubernetes foundation roots examples/aws-environments # environment-scoped AWS data composition diff --git a/terraform/modules/gcp/internal-edge/README.md b/terraform/modules/gcp/internal-edge/README.md new file mode 100644 index 00000000..1a4391ad --- /dev/null +++ b/terraform/modules/gcp/internal-edge/README.md @@ -0,0 +1,78 @@ +# gcp/internal-edge + +Network prerequisites for the `agent-relay` chart's internal-only GKE edge (`gkeIngress.mode: internal`): a regional internal Application Load Balancer that GKE creates from the Ingress. The install then has a private address and is reachable only from the VPC and the networks routed to it (VPN, Interconnect). + +| Resource | Why | Chart value | +| --- | --- | --- | +| Proxy-only subnet `-proxy-only` (`REGIONAL_MANAGED_PROXY`) | The load balancer's Envoy proxies take their addresses from it. A VPC has one active proxy-only subnet per region; set `proxy_only_subnet.create = false` to use an existing one | `gkeIngress.internal.proxyOnlySubnetCidr`, and every backend's `networkPolicy` | +| Regional internal address `-internal` | The load balancer's stable private address; DNS points at it | `gkeIngress.staticIpName` | +| Firewall rule `-allow-proxy-only` | GKE opens the firewall for Google's health checks but not for the proxy-only subnet. The rule admits only that subnet, only to instances running as the node service account, only on the backend ports | none | + +This module opens no path from outside the VPC and creates no external address. Reaching the address from offices or laptops is your VPN's or Interconnect's routing, plus DNS records for the install's hosts in a zone those clients resolve. + +**Not created here:** the certificate. A Google-managed certificate cannot be used on an internal load balancer. Use a `kubernetes.io/tls` Secret (`gkeIngress.tlsSecretName`), for example issued by cert-manager with a DNS-01 challenge, or a regional self-managed certificate (`gcloud compute ssl-certificates create NAME --region REGION --certificate ... --private-key ...`, then `gkeIngress.internal.preSharedCerts`). A Terraform-managed certificate would put the private key in state. + +`labels` reach the address. Subnets and firewall rules have no labels in the API. + +```hcl +module "internal_edge" { + source = "../../modules/gcp/internal-edge" + + project_id = var.project_id + region = var.region + name = "agent-relay" + network = module.foundation.network.id + subnetwork = module.foundation.network.subnet_self_link + node_service_account = module.foundation.gke.node_service_account + proxy_only_subnet = { cidr = "10.129.0.0/23" } +} +``` + +Tested with a mock provider only; not yet applied to a live project. + + +### Requirements + +| Name | Version | +| ---- | ------- | +| terraform | >= 1.6 | +| google | >= 7.0, < 8.0 | + +### Providers + +| Name | Version | +| ---- | ------- | +| google | >= 7.0, < 8.0 | + +### Resources + +| Name | Type | +| ---- | ---- | +| [google_compute_address.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_firewall.proxy_only](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource | +| [google_compute_subnetwork.proxy_only](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_subnetwork) | resource | + +### Inputs + +| Name | Description | Type | Default | Required | +| ---- | ----------- | ---- | ------- | :------: | +| name | Name prefix: the address is -internal, the proxy-only subnet -proxy-only and the firewall rule -allow-proxy-only. | `string` | n/a | yes | +| network | Self link or ID of the cluster's VPC network. | `string` | n/a | yes | +| node\_service\_account | Email of the service account the GKE nodes run as. The firewall rule lets the proxy-only subnet reach only instances running as it. | `string` | n/a | yes | +| project\_id | Project of the GKE cluster and its VPC. | `string` | n/a | yes | +| proxy\_only\_subnet | The VPC's proxy-only subnet for regional Envoy load balancers in this region. A VPC has one active proxy-only subnet per region: create = true makes it (cidr is a free RFC 1918 range, /26 or larger, /23 recommended); create = false uses one that already exists, and cidr is its range. | ```object({ create = optional(bool, true) cidr = string })``` | n/a | yes | +| region | Region of the GKE cluster. The internal load balancer, its address and the proxy-only subnet are regional. | `string` | n/a | yes | +| subnetwork | Self link or ID of the subnet the load balancer's address is taken from, normally the cluster's node subnet. | `string` | n/a | yes | +| address | A specific unused address in subnetwork for the load balancer (null: Google picks one). DNS for every install host points at it. | `string` | `null` | no | +| backend\_ports | TCP ports of the pods behind the Ingress, which the load balancer's proxies connect to: 3000 (control API), 8080 (dashboard, session read API, Relayfile, relaygate), 8081 (session ingest), 8787 (Relaycast) and 7100 (durable actors). Change it if you changed a server port in the values. | `list(string)` | ```[ "3000", "7100", "8080", "8081", "8787" ]``` | no | +| labels | Labels for the reserved address (the only resource here that takes labels). managed-by=terraform is always added. | `map(string)` | `{}` | no | + +### Outputs + +| Name | Description | +| ---- | ----------- | +| address | The load balancer's private address. Point the DNS records of every install host at it, in a zone your VPN clients resolve. | +| chart\_values | The gkeIngress keys for internal mode. Add the certificate (gkeIngress.tlsSecretName or internal.preSharedCerts) and the NetworkPolicy lists from values-gcp-internal.yaml. | +| proxy\_only\_subnet\_cidr | Chart value gkeIngress.internal.proxyOnlySubnetCidr, and the range to add to every backend's networkPolicy (values-gcp-internal.yaml). | +| static\_ip\_name | Chart value gkeIngress.staticIpName. | + diff --git a/terraform/modules/gcp/internal-edge/main.tf b/terraform/modules/gcp/internal-edge/main.tf new file mode 100644 index 00000000..7d1b82f1 --- /dev/null +++ b/terraform/modules/gcp/internal-edge/main.tf @@ -0,0 +1,58 @@ +# Network prerequisites for the agent-relay chart's internal-only GKE edge +# (gkeIngress.mode=internal): a regional internal Application Load Balancer +# that GKE creates from the Ingress. Nothing here is reachable from the +# internet, and nothing here opens a path from outside the VPC: reaching the +# address from an office or laptop needs your VPN or Interconnect routes. + +locals { + labels = merge(var.labels, { managed-by = "terraform" }) +} + +# Envoy proxies of regional load balancers take their addresses here. They +# are the source of every request the pods receive. +resource "google_compute_subnetwork" "proxy_only" { + #checkov:skip=CKV_GCP_26:Flow logs are not supported on a proxy-only subnet (purpose REGIONAL_MANAGED_PROXY). + #checkov:skip=CKV_GCP_74:Private Google Access does not apply to a proxy-only subnet; no workload runs in it. + #checkov:skip=CKV_GCP_76:IPv4 only; no IPv6 access is configured on this subnet. + count = var.proxy_only_subnet.create ? 1 : 0 + project = var.project_id + region = var.region + name = "${var.name}-proxy-only" + network = var.network + ip_cidr_range = var.proxy_only_subnet.cidr + purpose = "REGIONAL_MANAGED_PROXY" + role = "ACTIVE" +} + +# The load balancer's private address (chart value gkeIngress.staticIpName). +resource "google_compute_address" "this" { + project = var.project_id + region = var.region + name = "${var.name}-internal" + address_type = "INTERNAL" + purpose = "SHARED_LOADBALANCER_VIP" + subnetwork = var.subnetwork + address = var.address + labels = local.labels +} + +# GKE opens the firewall for Google's health checks itself, but not for the +# proxy-only subnet. Without this rule every request times out at the proxy. +resource "google_compute_firewall" "proxy_only" { + project = var.project_id + name = "${var.name}-allow-proxy-only" + network = var.network + description = "Internal load balancer proxies (proxy-only subnet) to the agent-relay pods" + direction = "INGRESS" + priority = 1000 + source_ranges = [var.proxy_only_subnet.cidr] + target_service_accounts = [var.node_service_account] + allow { + protocol = "tcp" + ports = var.backend_ports + } + log_config { + metadata = "EXCLUDE_ALL_METADATA" + } + depends_on = [google_compute_subnetwork.proxy_only] +} diff --git a/terraform/modules/gcp/internal-edge/outputs.tf b/terraform/modules/gcp/internal-edge/outputs.tf new file mode 100644 index 00000000..202a6957 --- /dev/null +++ b/terraform/modules/gcp/internal-edge/outputs.tf @@ -0,0 +1,23 @@ +output "static_ip_name" { + description = "Chart value gkeIngress.staticIpName." + value = google_compute_address.this.name +} + +output "address" { + description = "The load balancer's private address. Point the DNS records of every install host at it, in a zone your VPN clients resolve." + value = google_compute_address.this.address +} + +output "proxy_only_subnet_cidr" { + description = "Chart value gkeIngress.internal.proxyOnlySubnetCidr, and the range to add to every backend's networkPolicy (values-gcp-internal.yaml)." + value = var.proxy_only_subnet.cidr +} + +output "chart_values" { + description = "The gkeIngress keys for internal mode. Add the certificate (gkeIngress.tlsSecretName or internal.preSharedCerts) and the NetworkPolicy lists from values-gcp-internal.yaml." + value = { gkeIngress = { + mode = "internal" + staticIpName = google_compute_address.this.name + internal = { proxyOnlySubnetCidr = var.proxy_only_subnet.cidr } + } } +} diff --git a/terraform/modules/gcp/internal-edge/tests/internal_edge.tftest.hcl b/terraform/modules/gcp/internal-edge/tests/internal_edge.tftest.hcl new file mode 100644 index 00000000..42975ff9 --- /dev/null +++ b/terraform/modules/gcp/internal-edge/tests/internal_edge.tftest.hcl @@ -0,0 +1,73 @@ +mock_provider "google" { + mock_resource "google_compute_address" { + defaults = { address = "10.10.0.50" } + } +} + +variables { + project_id = "proj" + region = "us-central1" + name = "relay" + network = "projects/proj/global/networks/vpc" + subnetwork = "projects/proj/regions/us-central1/subnetworks/nodes" + node_service_account = "relay-nodes@proj.iam.gserviceaccount.com" + proxy_only_subnet = { cidr = "10.129.0.0/23" } +} + +run "internal_edge_prerequisites" { + command = apply + + assert { + condition = google_compute_subnetwork.proxy_only[0].purpose == "REGIONAL_MANAGED_PROXY" && google_compute_subnetwork.proxy_only[0].role == "ACTIVE" && google_compute_subnetwork.proxy_only[0].ip_cidr_range == "10.129.0.0/23" && google_compute_subnetwork.proxy_only[0].region == "us-central1" + error_message = "The proxy-only subnet is a regional managed proxy subnet in the cluster's region." + } + assert { + condition = google_compute_address.this.address_type == "INTERNAL" && google_compute_address.this.purpose == "SHARED_LOADBALANCER_VIP" && google_compute_address.this.region == "us-central1" && google_compute_address.this.subnetwork == "projects/proj/regions/us-central1/subnetworks/nodes" + error_message = "The address is regional and internal: never a global or external address." + } + assert { + condition = google_compute_address.this.labels["managed-by"] == "terraform" + error_message = "The address carries the labels." + } + # The only allow rule: from the proxy-only subnet, to the nodes' identity, on the backend ports. + assert { + condition = google_compute_firewall.proxy_only.direction == "INGRESS" && google_compute_firewall.proxy_only.source_ranges == toset(["10.129.0.0/23"]) && google_compute_firewall.proxy_only.target_service_accounts == toset(["relay-nodes@proj.iam.gserviceaccount.com"]) + error_message = "The firewall rule admits only the proxy-only subnet, only to instances running as the node service account." + } + assert { + condition = one(google_compute_firewall.proxy_only.allow).protocol == "tcp" && toset(one(google_compute_firewall.proxy_only.allow).ports) == toset(["3000", "7100", "8080", "8081", "8787"]) + error_message = "Only the backend TCP ports are opened." + } + assert { + condition = output.chart_values.gkeIngress.mode == "internal" && output.chart_values.gkeIngress.staticIpName == "relay-internal" && output.chart_values.gkeIngress.internal.proxyOnlySubnetCidr == "10.129.0.0/23" && output.address == "10.10.0.50" + error_message = "Outputs feed the chart's internal mode." + } +} + +run "existing_proxy_only_subnet" { + command = apply + variables { + proxy_only_subnet = { create = false, cidr = "10.200.0.0/24" } + } + + assert { + condition = length(google_compute_subnetwork.proxy_only) == 0 && google_compute_firewall.proxy_only.source_ranges == toset(["10.200.0.0/24"]) && output.proxy_only_subnet_cidr == "10.200.0.0/24" + error_message = "An existing proxy-only subnet is used by its range, not created again." + } +} + +run "public_range_refused" { + command = plan + variables { + proxy_only_subnet = { cidr = "203.0.113.0/24" } + } + expect_failures = [var.proxy_only_subnet] +} + +run "too_small_range_refused" { + command = plan + variables { + proxy_only_subnet = { cidr = "10.129.0.0/28" } + } + expect_failures = [var.proxy_only_subnet] +} diff --git a/terraform/modules/gcp/internal-edge/variables.tf b/terraform/modules/gcp/internal-edge/variables.tf new file mode 100644 index 00000000..c3b8a7f9 --- /dev/null +++ b/terraform/modules/gcp/internal-edge/variables.tf @@ -0,0 +1,67 @@ +variable "project_id" { + description = "Project of the GKE cluster and its VPC." + type = string +} + +variable "region" { + description = "Region of the GKE cluster. The internal load balancer, its address and the proxy-only subnet are regional." + type = string +} + +variable "name" { + description = "Name prefix: the address is -internal, the proxy-only subnet -proxy-only and the firewall rule -allow-proxy-only." + type = string + validation { + condition = can(regex("^[a-z]([-a-z0-9]{0,40}[a-z0-9])?$", var.name)) + error_message = "name must be 1-42 lowercase letters, digits or hyphens, beginning with a letter." + } +} + +variable "network" { + description = "Self link or ID of the cluster's VPC network." + type = string +} + +variable "subnetwork" { + description = "Self link or ID of the subnet the load balancer's address is taken from, normally the cluster's node subnet." + type = string +} + +variable "address" { + description = "A specific unused address in subnetwork for the load balancer (null: Google picks one). DNS for every install host points at it." + type = string + default = null +} + +variable "proxy_only_subnet" { + description = "The VPC's proxy-only subnet for regional Envoy load balancers in this region. A VPC has one active proxy-only subnet per region: create = true makes it (cidr is a free RFC 1918 range, /26 or larger, /23 recommended); create = false uses one that already exists, and cidr is its range." + type = object({ + create = optional(bool, true) + cidr = string + }) + validation { + condition = can(cidrhost(var.proxy_only_subnet.cidr, 0)) && try(tonumber(split("/", var.proxy_only_subnet.cidr)[1]) <= 26, false) && can(regex("^(10\\.|172\\.(1[6-9]|2[0-9]|3[01])\\.|192\\.168\\.)", var.proxy_only_subnet.cidr)) + error_message = "proxy_only_subnet.cidr must be an RFC 1918 IPv4 range of /26 or larger." + } +} + +variable "node_service_account" { + description = "Email of the service account the GKE nodes run as. The firewall rule lets the proxy-only subnet reach only instances running as it." + type = string +} + +variable "backend_ports" { + description = "TCP ports of the pods behind the Ingress, which the load balancer's proxies connect to: 3000 (control API), 8080 (dashboard, session read API, Relayfile, relaygate), 8081 (session ingest), 8787 (Relaycast) and 7100 (durable actors). Change it if you changed a server port in the values." + type = list(string) + default = ["3000", "7100", "8080", "8081", "8787"] + validation { + condition = length(var.backend_ports) > 0 && alltrue([for p in var.backend_ports : can(regex("^[0-9]{1,5}(-[0-9]{1,5})?$", p))]) + error_message = "backend_ports are TCP ports or ranges, for example \"8080\" or \"8080-8081\"." + } +} + +variable "labels" { + description = "Labels for the reserved address (the only resource here that takes labels). managed-by=terraform is always added." + type = map(string) + default = {} +} diff --git a/terraform/modules/gcp/internal-edge/versions.tf b/terraform/modules/gcp/internal-edge/versions.tf new file mode 100644 index 00000000..2ef82a5a --- /dev/null +++ b/terraform/modules/gcp/internal-edge/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.6" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 7.0, < 8.0" + } + } +}