From aff4aa79ba70641ae6b07071f368ef95f3268260 Mon Sep 17 00:00:00 2001 From: jerryfane Date: Sun, 15 Nov 2026 17:09:28 +0100 Subject: [PATCH 1/2] chore(deploy): video runner on hetzner2 (#28) --- deploy/hetzner2-video/README.md | 41 +++++++++++++++++++ deploy/hetzner2-video/config.yaml | 19 +++++++++ deploy/hetzner2-video/stagehand-video | 4 ++ deploy/hetzner2-video/stagehand-video.service | 27 ++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 deploy/hetzner2-video/README.md create mode 100644 deploy/hetzner2-video/config.yaml create mode 100755 deploy/hetzner2-video/stagehand-video create mode 100644 deploy/hetzner2-video/stagehand-video.service diff --git a/deploy/hetzner2-video/README.md b/deploy/hetzner2-video/README.md new file mode 100644 index 0000000..aba79b7 --- /dev/null +++ b/deploy/hetzner2-video/README.md @@ -0,0 +1,41 @@ +# Video runner on hetzner2 + +Video pipelines run on hetzner2, next to their data, so the fleet host stays usable +(owner direction 2026-11-15; gitmoot/stagehand#28). + +| Item | Value | +|---|---| +| Unit | `stagehand-video.service` | +| Home | `/var/lib/stagehand-video` | +| Config | `/etc/stagehand-video/config.yaml` | +| Trigger key | `/etc/stagehand-video/trigger.key` (0600) | +| Status | `127.0.0.1:8807/status` | +| Trigger | `127.0.0.1:8808` | +| Agent stages | sandboxd instance `stagehandvid` (API 43297, guest port 43298) | +| Operator CLI | `stagehand-video ` (sets the home and the config) | + +## Install + +```sh +install -m 755 stagehand /usr/local/bin/stagehand +install -m 755 stagehand-video /usr/local/bin/stagehand-video +install -d -m 700 /etc/stagehand-video /var/lib/stagehand-video +install -m 600 config.yaml /etc/stagehand-video/config.yaml +(umask 077; openssl rand -hex 32 > /etc/stagehand-video/trigger.key) +install -m 644 stagehand-video.service /etc/systemd/system/ +systemctl daemon-reload && systemctl enable --now stagehand-video +``` + +## Moving a pipeline here + +Never run a pipeline on both runners at once. Do the cutover between runs: + +1. `stagehand disable ` on the fleet host. +2. Move its state. +3. `stagehand-video add --enable` here. + +## Model credentials + +`agent.model_gateway.upstream` points at `127.0.0.1:43299`, where nothing listens until the owner +approves a credential route. Until then, shell stages work and agent stages fail at their first +model call. diff --git a/deploy/hetzner2-video/config.yaml b/deploy/hetzner2-video/config.yaml new file mode 100644 index 0000000..d687458 --- /dev/null +++ b/deploy/hetzner2-video/config.yaml @@ -0,0 +1,19 @@ +# /etc/stagehand-video/config.yaml on hetzner2 (gitmoot/stagehand#28) +agent: + executor: sandboxd + sandboxd: + url: http://127.0.0.1:43297 # local sandboxd instance "stagehandvid" + api_key_file: /var/lib/sandboxd-stagehandvid/api-key # the instance's own key (0600 root); not copied + template: stagehand-omp-strict + guest_omp: /usr/local/bin/omp + max_ttl: 1h + model_gateway: + listen: 127.0.0.1:43298 # the instance's guest host port + guest_url: http://10.0.2.2:43298 + # Credential route for guest model calls. Nothing listens here until the + # owner approves one (a tunnel to the fleet host's stagehand-gateway-proxy, + # or a local gateway-proxy with its own token); until then agent stages + # fail at their first model call and shell stages are unaffected. + upstream: http://127.0.0.1:43299 +events: + secret_file: /etc/stagehand-video/trigger.key # HMAC key for the trigger endpoint, 0600 diff --git a/deploy/hetzner2-video/stagehand-video b/deploy/hetzner2-video/stagehand-video new file mode 100755 index 0000000..58d7c12 --- /dev/null +++ b/deploy/hetzner2-video/stagehand-video @@ -0,0 +1,4 @@ +#!/bin/sh +# Operator CLI for the video runner on hetzner2: plain `stagehand` there would +# use (and create) the default home /var/lib/stagehand instead. +STAGEHAND_HOME=/var/lib/stagehand-video STAGEHAND_CONFIG=/etc/stagehand-video/config.yaml exec /usr/local/bin/stagehand "$@" diff --git a/deploy/hetzner2-video/stagehand-video.service b/deploy/hetzner2-video/stagehand-video.service new file mode 100644 index 0000000..1749691 --- /dev/null +++ b/deploy/hetzner2-video/stagehand-video.service @@ -0,0 +1,27 @@ +# Video runner on hetzner2 (gitmoot/stagehand#28): video pipelines run next to +# their data there, not on the fleet host. Its own home, config, trigger key and +# loopback ports, so it never shares state with another runner. Agent stages use +# the local sandboxd instance "stagehandvid" (operated by the sandboxd seat). +[Unit] +Description=stagehand video pipeline runner +After=network-online.target sandboxd-stagehandvid.service +Wants=network-online.target sandboxd-stagehandvid.service + +[Service] +Type=simple +Environment=STAGEHAND_HOME=/var/lib/stagehand-video +Environment=STAGEHAND_CONFIG=/etc/stagehand-video/config.yaml +# systemd sets no HOME for a root service without User=; shell stages inherit +# this one (tools such as omp, herdr and gh find their credentials through it). +Environment=HOME=/root +ExecStart=/usr/local/bin/stagehand serve --status-addr 127.0.0.1:8807 --trigger-addr 127.0.0.1:8808 +Restart=on-failure +RestartSec=10 +# Stop sends SIGTERM: running stages are stopped and their runs parked as +# interrupted. control-group also kills anything a crashed runner left. +KillMode=control-group +TimeoutStopSec=60 +UMask=0077 + +[Install] +WantedBy=multi-user.target From 9e6e34240065160cc39774714199ee498787f54e Mon Sep 17 00:00:00 2001 From: jerryfane Date: Sun, 15 Nov 2026 17:24:51 +0100 Subject: [PATCH 2/2] chore(deploy): model route tunnel and memory budget for the hetzner2 video runner (#28) --- deploy/hetzner2-video/README.md | 27 +++++++++++++-- deploy/hetzner2-video/config.yaml | 7 ++-- .../stagehand-video-hetzner2-tunnel.service | 34 +++++++++++++++++++ deploy/hetzner2-video/stagehand-video.service | 10 ++++++ 4 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 deploy/hetzner2-video/fleet-host/stagehand-video-hetzner2-tunnel.service diff --git a/deploy/hetzner2-video/README.md b/deploy/hetzner2-video/README.md index aba79b7..d117c3b 100644 --- a/deploy/hetzner2-video/README.md +++ b/deploy/hetzner2-video/README.md @@ -26,6 +26,13 @@ install -m 644 stagehand-video.service /etc/systemd/system/ systemctl daemon-reload && systemctl enable --now stagehand-video ``` +On the fleet host: + +```sh +install -m 644 fleet-host/stagehand-video-hetzner2-tunnel.service /etc/systemd/system/ +systemctl daemon-reload && systemctl enable --now stagehand-video-hetzner2-tunnel +``` + ## Moving a pipeline here Never run a pipeline on both runners at once. Do the cutover between runs: @@ -36,6 +43,20 @@ Never run a pipeline on both runners at once. Do the cutover between runs: ## Model credentials -`agent.model_gateway.upstream` points at `127.0.0.1:43299`, where nothing listens until the owner -approves a credential route. Until then, shell stages work and agent stages fail at their first -model call. +Owner decision, 2026-11-15: use a tunnel. No model credential is stored on hetzner2. + +- `agent.model_gateway.upstream` is `127.0.0.1:43299`. The fleet host's + `stagehand-video-hetzner2-tunnel.service` (`fleet-host/` here) reverse-forwards that port to its + own `stagehand-gateway-proxy` on `127.0.0.1:43295`, which adds the credential. +- Guests never reach 43299. They hold only per-run tokens for the run gateway on 43298. +- On hetzner2 only root can open 43299. The firewall rejects loopback for the CI runner uids + (`h2ci_*`, chain `ci`) and for the sandboxd jail uids (chain `guest`). +- If the tunnel is down, agent stages fail at their first model call, and shell stages are + unaffected. + +## Resources + +The unit caps the runner and its shell stages at `MemoryMax=8G` and `MemoryHigh=7G`, with no +swap, `CPUQuota=600%`, and CPU and IO weights of 50. These come from the hetzner2 memory budget. +After the first week, check `memory.peak` in the unit's cgroup. If it stays under 5 GiB, lower +the cap to 6 GiB. diff --git a/deploy/hetzner2-video/config.yaml b/deploy/hetzner2-video/config.yaml index d687458..f601834 100644 --- a/deploy/hetzner2-video/config.yaml +++ b/deploy/hetzner2-video/config.yaml @@ -10,10 +10,9 @@ agent: model_gateway: listen: 127.0.0.1:43298 # the instance's guest host port guest_url: http://10.0.2.2:43298 - # Credential route for guest model calls. Nothing listens here until the - # owner approves one (a tunnel to the fleet host's stagehand-gateway-proxy, - # or a local gateway-proxy with its own token); until then agent stages - # fail at their first model call and shell stages are unaffected. + # Reverse forward from the fleet host (stagehand-video-hetzner2-tunnel) to + # its stagehand-gateway-proxy, which adds the model credential there. No + # model credential is stored on hetzner2 (owner decision 2026-11-15). upstream: http://127.0.0.1:43299 events: secret_file: /etc/stagehand-video/trigger.key # HMAC key for the trigger endpoint, 0600 diff --git a/deploy/hetzner2-video/fleet-host/stagehand-video-hetzner2-tunnel.service b/deploy/hetzner2-video/fleet-host/stagehand-video-hetzner2-tunnel.service new file mode 100644 index 0000000..8abf934 --- /dev/null +++ b/deploy/hetzner2-video/fleet-host/stagehand-video-hetzner2-tunnel.service @@ -0,0 +1,34 @@ +# Fleet host: gives the hetzner2 video runner's model gateway its upstream +# without putting a model credential on hetzner2 (owner decision 2026-11-15, +# gitmoot/stagehand#28). +# -R hetzner2 127.0.0.1:43299 -> 127.0.0.1:43295 stagehand-gateway-proxy here, +# which adds the gateway bearer +# Only root on hetzner2 can open 43299: its firewall rejects loopback for the +# CI runner and sandboxd jail uids. Modeled on stagehand-hetzner2-tunnel.service. +[Unit] +Description=stagehand video runner model route (hetzner2 43299 -> gateway-proxy 43295) +Wants=network-online.target tailscaled.service stagehand-gateway-proxy.service +After=network-online.target tailscaled.service stagehand-gateway-proxy.service + +[Service] +Type=simple +User=root +UMask=0077 +# After a network drop, hetzner2's sshd can briefly keep the dead session, and +# that session still holds 43299, so a reconnect fails with "remote port +# forwarding failed". Before connecting, end that session only if it is +# provably ours and dead. Leading "-": a failure here (host down) must not block +# the retry loop. +ExecStartPre=-/usr/local/bin/stagehand tunnel-clear-stale --ssh-host root@hetzner2 --remote-listen 127.0.0.1:43299 +ExecStart=/usr/bin/ssh -N -T -o BatchMode=yes -o StrictHostKeyChecking=yes -o UpdateHostKeys=no -o IdentityAgent=none -o ControlMaster=no -o ControlPath=none -o ExitOnForwardFailure=yes -o ConnectTimeout=10 -o ServerAliveInterval=15 -o ServerAliveCountMax=3 -R 127.0.0.1:43299:127.0.0.1:43295 root@hetzner2 +Restart=always +RestartSec=5 +KillMode=control-group +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/hetzner2-video/stagehand-video.service b/deploy/hetzner2-video/stagehand-video.service index 1749691..562d6d6 100644 --- a/deploy/hetzner2-video/stagehand-video.service +++ b/deploy/hetzner2-video/stagehand-video.service @@ -22,6 +22,16 @@ RestartSec=10 KillMode=control-group TimeoutStopSec=60 UMask=0077 +# hetzner2 memory budget (/root/fleet-tools/state/hetzner2-memory-budget.md, +# approved by jarvis msg 488023, confirmed by keephair): covers the runner and +# every shell stage it starts. Agent VMs are capped separately by sandboxd. +MemoryMax=8G +MemoryHigh=7G +MemorySwapMax=0 +CPUQuota=600% +CPUWeight=50 +IOWeight=50 +OOMScoreAdjust=300 [Install] WantedBy=multi-user.target