diff --git a/README.md b/README.md index ea8baa1..16c80d5 100644 --- a/README.md +++ b/README.md @@ -138,10 +138,14 @@ uid 1000), and the VM is destroyed afterwards. attempt. It accepts only that attempt's random token, removes it, and forwards the call. - **Credentials:** the real model credential is added further along, by `stagehand gateway-proxy`, on the host that holds the omp auth-gateway token. +- **Where things run:** the runner (and so the shell stages) runs next to the data its shell + stages need. The sandboxd instance can be on another host: one ssh tunnel forwards its API to + the runner (`-L`) and its guest host port back to the runner's run gateway (`-R`). ``` -guest omp ── token for this attempt ──▶ run gateway (stagehand serve, 127.0.0.1:43293) - ──▶ 127.0.0.1:43294 (ssh -R tunnel) ──▶ stagehand gateway-proxy (127.0.0.1:43295, adds bearer) +guest omp ── token for this attempt ──▶ 10.0.2.2:43293 (sandbox host's -fc-host-port) + ──▶ ssh -R ──▶ run gateway (stagehand serve on the runner host, 127.0.0.1:43293) + ──▶ stagehand gateway-proxy (127.0.0.1:43295, adds bearer) ──▶ omp auth-gateway (127.0.0.1:4011) ──▶ provider ``` @@ -149,15 +153,15 @@ guest omp ── token for this attempt ──▶ run gateway (stagehand serve, agent: executor: sandboxd sandboxd: - url: http://127.0.0.1:43292 # the instance's -listen - api_key_file: /var/lib/sandboxd-stagehand/api-key + url: http://127.0.0.1:43292 # the instance's -listen (via the tunnel's -L) + api_key_file: /etc/stagehand/sandboxd-api-key template: stagehand-omp-strict guest_omp: /usr/local/bin/omp # default max_ttl: 1h # the instance's -max-ttl model_gateway: - listen: 127.0.0.1:43293 # = the instance's -fc-host-port + listen: 127.0.0.1:43293 # reached from the instance's -fc-host-port (tunnel -R) guest_url: http://10.0.2.2:43293 - upstream: http://127.0.0.1:43294 # tunnel to gateway-proxy + upstream: http://127.0.0.1:43295 # stagehand gateway-proxy ``` - **Which models work:** the VM's omp gets only `ANTHROPIC_BASE_URL` and `ANTHROPIC_API_KEY` @@ -296,6 +300,25 @@ WantedBy=multi-user.target - **Crashes:** `KillMode=control-group` makes systemd kill any stage processes left behind if the runner crashes. +### Upgrading and rolling back + +1. Check that nothing is running: `stagehand status --json` shows `active_run_id: null` for every pipeline. +2. Put the new binary next to the old one, then `mv` it over `/usr/local/bin/stagehand`. Renaming + on the same filesystem is atomic and avoids "text file busy". +3. `systemctl restart stagehand`, then check `stagehand version` and `GET /status`. + +To roll back, do the same with the previous binary. The database only gets new migrations: an older +binary refuses a newer schema rather than misreading it (`database schema version N is newer than +this binary supports`). Rolling back across a schema change therefore needs the database file +backed up before the upgrade. + +The sandbox tunnel (`deploy/stagehand-hetzner2-tunnel.service`) reconnects on its own after a +network drop. Before each connection, `stagehand tunnel-clear-stale` ends a session on the far side +that still holds the forwarded port, but only one it can prove is this host's own dead +connection. That means an sshd-session on exactly that address and port, whose client is this host +and whose TCP connection no longer exists here. Live sessions and other hosts' sessions are left +alone. + ## Development ```sh diff --git a/deploy/config.example.yaml b/deploy/config.example.yaml index ca1ecb2..8e80801 100644 --- a/deploy/config.example.yaml +++ b/deploy/config.example.yaml @@ -1,13 +1,13 @@ -# /etc/stagehand/config.yaml on hetzner2 +# /etc/stagehand/config.yaml on the runner host agent: executor: sandboxd sandboxd: - url: http://127.0.0.1:43292 - api_key_file: /var/lib/sandboxd-stagehand/api-key + url: http://127.0.0.1:43292 # hetzner2's instance, via stagehand-hetzner2-tunnel (-L) + api_key_file: /etc/stagehand/sandboxd-api-key # copy of the instance's api-key, 0600 template: stagehand-omp-strict guest_omp: /usr/local/bin/omp max_ttl: 1h model_gateway: - listen: 127.0.0.1:43293 # = sandboxd -fc-host-port; the guest sees 10.0.2.2:43293 + listen: 127.0.0.1:43293 # reached from hetzner2's -fc-host-port via the tunnel (-R) guest_url: http://10.0.2.2:43293 - upstream: http://127.0.0.1:43294 # stagehand-model-tunnel -> gateway-proxy on the fleet host + upstream: http://127.0.0.1:43295 # stagehand gateway-proxy on this host diff --git a/deploy/stagehand-gateway-proxy.service b/deploy/stagehand-gateway-proxy.service index ef4b58a..b69052b 100644 --- a/deploy/stagehand-gateway-proxy.service +++ b/deploy/stagehand-gateway-proxy.service @@ -1,5 +1,5 @@ -# Fleet host (holds the omp auth-gateway token): adds the gateway bearer to -# model calls arriving from the hetzner2 tunnel. Loopback only. +# Runner host (holds the omp auth-gateway token): adds the gateway bearer to +# agents' model calls coming from the per-attempt run gateway. Loopback only. [Unit] Description=stagehand model credential proxy After=network-online.target diff --git a/deploy/stagehand-hetzner2-tunnel.service b/deploy/stagehand-hetzner2-tunnel.service new file mode 100644 index 0000000..3dc0137 --- /dev/null +++ b/deploy/stagehand-hetzner2-tunnel.service @@ -0,0 +1,36 @@ +# Runner host: links the stagehand runner to the sandboxd instance "stagehand" +# on hetzner2, where agent VMs run. +# -L 127.0.0.1:43292 -> hetzner2 127.0.0.1:43292 sandboxd control and guest API +# -R hetzner2 127.0.0.1:43293 -> 127.0.0.1:43293 the guests' only host port +# (= sandboxd -fc-host-port), served +# by the runner's per-attempt run gateway +# Modeled on phobos-sandbox-hetzner2-tunnel.service. +[Unit] +Description=stagehand tunnel to the hetzner2 sandboxd instance (API 43292, guest gateway 43293) +Wants=network-online.target tailscaled.service +After=network-online.target tailscaled.service + +[Service] +Type=simple +User=root +UMask=0077 +# After a network drop, hetzner2's sshd keeps the dead session (its +# ClientAliveInterval is 0), and it still holds 43293, so every reconnect +# fails with "remote port forwarding failed". Before connecting, end that +# session only if it is provably ours and dead: an sshd-session on exactly +# 127.0.0.1:43293 whose client is this host and whose connection no longer +# exists here. Live or foreign sessions are left alone. Leading "-": a +# failure here (host down) must not block the retry loop. +ExecStartPre=-/usr/local/bin/stagehand tunnel-clear-stale --ssh-host root@hetzner2 --remote-listen 127.0.0.1:43293 +ExecStart=/usr/bin/ssh -N -T -o BatchMode=yes -o StrictHostKeyChecking=yes -o UpdateHostKeys=no -o IdentityAgent=none -o ControlMaster=no -o ControlPath=none -o ExitOnForwardFailure=yes -o ConnectTimeout=10 -o ServerAliveInterval=15 -o ServerAliveCountMax=3 -L 127.0.0.1:43292:127.0.0.1:43292 -R 127.0.0.1:43293:127.0.0.1:43293 root@hetzner2 +Restart=always +RestartSec=5 +KillMode=control-group +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/stagehand-model-tunnel.service b/deploy/stagehand-model-tunnel.service deleted file mode 100644 index 6a26052..0000000 --- a/deploy/stagehand-model-tunnel.service +++ /dev/null @@ -1,24 +0,0 @@ -# Fleet host: exposes the credential proxy on hetzner2 as 127.0.0.1:43294, -# where the stagehand run gateway forwards agents' model calls. -# Modeled on phobos-sandbox-hetzner2-tunnel.service. -[Unit] -Description=stagehand model tunnel: hetzner2 127.0.0.1:43294 -> this host 127.0.0.1:43295 -Wants=network-online.target tailscaled.service stagehand-gateway-proxy.service -After=network-online.target tailscaled.service stagehand-gateway-proxy.service - -[Service] -Type=simple -User=root -UMask=0077 -ExecStart=/usr/bin/ssh -N -T -o BatchMode=yes -o StrictHostKeyChecking=yes -o UpdateHostKeys=no -o IdentityAgent=none -o ControlMaster=no -o ControlPath=none -o ExitOnForwardFailure=yes -o ConnectTimeout=10 -o ServerAliveInterval=15 -o ServerAliveCountMax=3 -R 127.0.0.1:43294:127.0.0.1:43295 root@hetzner2 -Restart=always -RestartSec=5 -KillMode=control-group -NoNewPrivileges=true -PrivateTmp=true -ProtectSystem=strict -ProtectHome=read-only -RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/stagehand.service b/deploy/stagehand.service index 84b8c1d..fb8438e 100644 --- a/deploy/stagehand.service +++ b/deploy/stagehand.service @@ -1,15 +1,15 @@ -# hetzner2: the runner. Agent stages run in the sandboxd instance "stagehand"; -# shell stages run here as root. +# Runner host (next to the data the shell stages use). Agent stages run in the +# sandboxd instance "stagehand" on hetzner2 via stagehand-hetzner2-tunnel. [Unit] Description=stagehand pipeline runner -After=network-online.target sandboxd-stagehand.service -Wants=network-online.target +After=network-online.target stagehand-hetzner2-tunnel.service stagehand-gateway-proxy.service +Wants=network-online.target stagehand-hetzner2-tunnel.service stagehand-gateway-proxy.service [Service] Type=simple Environment=STAGEHAND_HOME=/var/lib/stagehand Environment=STAGEHAND_CONFIG=/etc/stagehand/config.yaml -ExecStart=/usr/local/bin/stagehand serve --status-addr 127.0.0.1:8795 +ExecStart=/usr/local/bin/stagehand serve --status-addr 127.0.0.1:8797 Restart=on-failure RestartSec=10 # Stop sends SIGTERM: running stages are stopped and their runs parked as diff --git a/internal/cli/cli.go b/internal/cli/cli.go index f60af27..88b2a5f 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -55,6 +55,8 @@ Service: tick one pass: fire due schedules, run them to the end gateway-proxy [--listen ADDR] [--upstream URL] [--token-file FILE] model credential proxy for sandboxed agent stages + tunnel-clear-stale --ssh-host HOST --remote-listen ADDR:PORT + end this host's dead ssh session still holding a forward status [--json] last and next run per pipeline version @@ -95,6 +97,14 @@ func Main(args []string, stdout, stderr io.Writer) int { } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() + if cmd == "tunnel-clear-stale" { + // Runs before a tunnel reconnects; needs no home. + if err := tunnelClearStale(ctx, cmdArgs, stdout, stderr); err != nil { + fmt.Fprintln(stderr, "stagehand:", err) + return 1 + } + return 0 + } if cmd == "gateway-proxy" { // Runs on the host holding the omp auth-gateway token; needs no home. if err := gatewayProxy(ctx, cmdArgs, stderr); err != nil { diff --git a/internal/cli/gateway.go b/internal/cli/gateway.go index 8dfa036..c016f32 100644 --- a/internal/cli/gateway.go +++ b/internal/cli/gateway.go @@ -7,11 +7,15 @@ import ( "fmt" "io" "log/slog" + "net" + "net/netip" "net/url" "os" + "os/exec" "strings" "github.com/gitmoot/stagehand/internal/modelgw" + "github.com/gitmoot/stagehand/internal/tunnelclean" ) // gatewayProxy runs `stagehand gateway-proxy`: the credential side of the @@ -59,3 +63,67 @@ func readSecretFile(path string) (string, error) { } return s, nil } + +// tunnelClearStale runs `stagehand tunnel-clear-stale`: before a reverse +// tunnel reconnects, end this host's own dead sshd session that still holds +// the forwarded port on the far host (see package tunnelclean). +func tunnelClearStale(ctx context.Context, args []string, stdout, stderr io.Writer) error { + fs := flag.NewFlagSet("tunnel-clear-stale", flag.ContinueOnError) + host := fs.String("ssh-host", "", "far host, as given to ssh (e.g. root@hetzner2)") + listen := fs.String("remote-listen", "", "the forward's exact address on the far host, e.g. 127.0.0.1:43293") + usage := "tunnel-clear-stale --ssh-host HOST --remote-listen ADDR:PORT" + if _, err := parse(fs, args, 0, usage); err != nil { + return err + } + ap, err := netip.ParseAddrPort(*listen) + if err != nil || *host == "" { + return fmt.Errorf("usage: stagehand %s", usage) + } + sshArgs := []string{"-T", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", "UpdateHostKeys=no", + "-o", "IdentityAgent=none", "-o", "ControlMaster=no", "-o", "ControlPath=none", "-o", "ConnectTimeout=10", *host} + remote := func(cmd string) (string, error) { + out, err := exec.CommandContext(ctx, "ssh", append(sshArgs, cmd)...).Output() + return string(out), err + } + out, err := remote(tunnelclean.QueryCommand(ap)) + if err != nil { + return fmt.Errorf("query %s: %w", *host, err) + } + listeners, conns, _ := strings.Cut(out, tunnelclean.Separator) + holders := tunnelclean.Holders(listeners, conns, ap) + if len(holders) == 0 { + fmt.Fprintf(stdout, "%s on %s: no sshd session holds it\n", ap, *host) + return nil + } + local, err := exec.CommandContext(ctx, "ss", "-Htn", "state", "established").Output() + if err != nil { + return fmt.Errorf("local sockets: %w", err) + } + var ours []netip.Addr + ifaddrs, err := net.InterfaceAddrs() + if err != nil { + return err + } + for _, a := range ifaddrs { + if p, err := netip.ParsePrefix(a.String()); err == nil { + ours = append(ours, p.Addr()) + } + } + stale := tunnelclean.Stale(holders, ours, tunnelclean.LocalConnections(string(local))) + for _, h := range holders { + isStale := false + for _, s := range stale { + isStale = isStale || s.PID == h.PID + } + if !isStale { + fmt.Fprintf(stdout, "%s on %s: held by live or foreign session pid %d (client %s); left alone\n", ap, *host, h.PID, h.Client) + continue + } + if _, err := remote(tunnelclean.KillCommand(h.PID, ap)); err != nil { + fmt.Fprintf(stderr, "end stale session pid %d: %v\n", h.PID, err) + continue + } + fmt.Fprintf(stdout, "%s on %s: ended stale session pid %d (client %s, gone here)\n", ap, *host, h.PID, h.Client) + } + return nil +} diff --git a/internal/tunnelclean/tunnelclean.go b/internal/tunnelclean/tunnelclean.go new file mode 100644 index 0000000..686727b --- /dev/null +++ b/internal/tunnelclean/tunnelclean.go @@ -0,0 +1,160 @@ +// Package tunnelclean finds and ends a stale sshd session that still holds a +// reverse-forwarded port on the far host after a network drop, so the tunnel +// can bind it again. +// +// A holder is ended only when all of these hold: +// - it is an sshd-session listening on exactly the forward's address and port; +// - its client connection comes from one of THIS host's routable addresses +// (never loopback or link-local, which name the far host itself); +// - this host has no live TCP connection with that client address and port. +// +// So it never touches another host's tunnel, a live tunnel from this host, +// or a listener on a different address. +package tunnelclean + +import ( + "fmt" + "net/netip" + "regexp" + "strconv" + "strings" +) + +// Session is an sshd-session on the far host holding the forwarded port. +type Session struct { + PID int + // Server and Client are the ends of its ssh connection, as the far host + // sees them. + Server, Client netip.AddrPort +} + +var pidRE = regexp.MustCompile(`\("sshd-session",pid=(\d+),`) + +// fields splits an `ss -H` line into its two socket addresses and the +// sshd-session pids in its process list. +func fields(line string) (local, peer string, pids []int) { + var addrs []string + for _, f := range strings.Fields(line) { + if strings.HasPrefix(f, "users:") { + break + } + if strings.Contains(f, ":") { + addrs = append(addrs, f) + } + } + if len(addrs) >= 2 { + local, peer = addrs[len(addrs)-2], addrs[len(addrs)-1] + } + for _, m := range pidRE.FindAllStringSubmatch(line, -1) { + if n, err := strconv.Atoi(m[1]); err == nil { + pids = append(pids, n) + } + } + return local, peer, pids +} + +// Holders parses the far host's `ss -Hltnp` (listeners) and +// `ss -Htnp state established '( sport = :22 )'` (ssh connections) output +// and returns the sshd-sessions listening on exactly listen, with their +// client connection. A holder whose connection is not listed is left out: +// nothing proves whose it is. +func Holders(listeners, conns string, listen netip.AddrPort) []Session { + holding := map[int]bool{} + for _, line := range strings.Split(listeners, "\n") { + local, _, pids := fields(line) + ap, err := netip.ParseAddrPort(local) + if err != nil || ap != listen { + continue + } + for _, p := range pids { + holding[p] = true + } + } + var out []Session + seen := map[int]bool{} + for _, line := range strings.Split(conns, "\n") { + local, peer, pids := fields(line) + srv, err1 := netip.ParseAddrPort(local) + cli, err2 := netip.ParseAddrPort(peer) + if err1 != nil || err2 != nil { + continue + } + for _, p := range pids { + if holding[p] && !seen[p] { + seen[p] = true + out = append(out, Session{PID: p, Server: srv, Client: cli}) + } + } + } + return out +} + +// Stale returns the holders that are this host's dead connections: the +// client address is one of ours, and we have no live connection from that +// address and port to the server. +func Stale(holders []Session, ours []netip.Addr, live []Connection) []Session { + own := map[netip.Addr]bool{} + for _, a := range ours { + // Loopback and link-local addresses name the far host itself (or + // its link) when they appear in its socket table, never this host, + // so they prove nothing about ownership. + if a = a.Unmap(); a.IsLoopback() || a.IsLinkLocalUnicast() || a.IsUnspecified() { + continue + } + own[a] = true + } + alive := map[Connection]bool{} + for _, c := range live { + alive[Connection{Local: unmap(c.Local), Remote: unmap(c.Remote)}] = true + } + var out []Session + for _, h := range holders { + cli, srv := unmap(h.Client), unmap(h.Server) + if cli.Addr().IsLoopback() || !own[cli.Addr()] { + continue + } + if alive[Connection{Local: cli, Remote: srv}] { + continue + } + out = append(out, h) + } + return out +} + +// Connection is a local TCP connection. +type Connection struct { + Local, Remote netip.AddrPort +} + +// LocalConnections parses this host's `ss -Htn state established` output. +func LocalConnections(out string) []Connection { + var cs []Connection + for _, line := range strings.Split(out, "\n") { + local, peer, _ := fields(line) + l, err1 := netip.ParseAddrPort(local) + r, err2 := netip.ParseAddrPort(peer) + if err1 == nil && err2 == nil { + cs = append(cs, Connection{Local: l, Remote: r}) + } + } + return cs +} + +func unmap(ap netip.AddrPort) netip.AddrPort { + return netip.AddrPortFrom(ap.Addr().Unmap(), ap.Port()) +} + +// KillCommand is the far-host command that ends pid, but only if it still +// holds listen at that moment (guarding against pid reuse). +func KillCommand(pid int, listen netip.AddrPort) string { + return fmt.Sprintf(`ss -Hltnp 'src %s' | grep -q '("sshd-session",pid=%d,' && kill %d`, listen, pid, pid) +} + +// QueryCommand is the far-host command whose output Holders parses: the +// listeners section, a separator line, then the ssh connections. +func QueryCommand(listen netip.AddrPort) string { + return fmt.Sprintf(`ss -Hltnp 'src %s'; echo %s; ss -Htnp state established '( sport = :22 )'`, listen, Separator) +} + +// Separator splits QueryCommand's two sections. +const Separator = "--stagehand-separator--" diff --git a/internal/tunnelclean/tunnelclean_test.go b/internal/tunnelclean/tunnelclean_test.go new file mode 100644 index 0000000..3ee5d52 --- /dev/null +++ b/internal/tunnelclean/tunnelclean_test.go @@ -0,0 +1,78 @@ +package tunnelclean + +import ( + "net/netip" + "testing" +) + +// Real `ss` lines from hetzner2 (3 Nov), plus decoys. +const listeners = `LISTEN 0 128 127.0.0.1:43293 0.0.0.0:* users:(("sshd-session",pid=2890732,fd=12)) +LISTEN 0 128 127.0.0.2:43293 0.0.0.0:* users:(("sshd-session",pid=777,fd=12)) +LISTEN 0 128 127.0.0.1:43291 0.0.0.0:* users:(("sshd-session",pid=2886811,fd=12)) +LISTEN 0 4096 127.0.0.1:43292 0.0.0.0:* users:(("sandboxd",pid=2840640,fd=12))` + +const conns = `0 0 100.73.209.5:22 100.106.218.88:46184 users:(("sshd-session",pid=2890732,fd=9),("sshd-session",pid=2890686,fd=9)) +0 0 100.73.209.5:22 100.106.218.88:50000 users:(("sshd-session",pid=777,fd=9)) +0 0 100.73.209.5:22 100.106.218.88:40000 users:(("sshd-session",pid=2886811,fd=9))` + +var ( + listen = netip.MustParseAddrPort("127.0.0.1:43293") + us = []netip.Addr{netip.MustParseAddr("100.106.218.88"), netip.MustParseAddr("127.0.0.1")} +) + +func TestHoldersMatchExactAddressOnly(t *testing.T) { + h := Holders(listeners, conns, listen) + if len(h) != 1 || h[0].PID != 2890732 || h[0].Client.String() != "100.106.218.88:46184" { + t.Fatalf("holders %+v; want only pid 2890732 (127.0.0.2 and other ports excluded)", h) + } +} + +func TestLiveConnectionFromThisHostIsNotStale(t *testing.T) { + live := LocalConnections(`0 0 100.106.218.88:46184 100.73.209.5:22 users:(("ssh",pid=1,fd=3))`) + if s := Stale(Holders(listeners, conns, listen), us, live); len(s) != 0 { + t.Fatalf("killed a live tunnel: %+v", s) + } +} + +func TestDeadConnectionFromThisHostIsStale(t *testing.T) { + live := LocalConnections(`0 0 100.106.218.88:50066 100.73.209.5:22 users:(("ssh",pid=1,fd=3))`) + s := Stale(Holders(listeners, conns, listen), us, live) + if len(s) != 1 || s[0].PID != 2890732 { + t.Fatalf("stale %+v; want pid 2890732", s) + } +} + +func TestOtherHostsSessionIsNeverStale(t *testing.T) { + other := []netip.Addr{netip.MustParseAddr("100.64.0.9")} + if s := Stale(Holders(listeners, conns, listen), other, nil); len(s) != 0 { + t.Fatalf("killed another host's session: %+v", s) + } +} + +func TestHolderWithoutVisibleConnectionIsLeftAlone(t *testing.T) { + h := Holders(listeners, "", listen) + if len(h) != 0 { + t.Fatalf("holders %+v; a holder whose client cannot be identified must not be killed", h) + } +} + +// A session on the far host from its own loopback (ssh localhost -R ...) +// must never pass for this host's, whatever this host's loopback addresses. +func TestFarHostLoopbackSessionIsNeverStale(t *testing.T) { + for _, client := range []string{"127.0.0.1:51000", "[::1]:51000", "[::ffff:127.0.0.1]:51000"} { + server := "127.0.0.1:22" + if client[0] == '[' { + server = "[::1]:22" + } + conn := "0 0 " + server + " " + client + ` users:(("sshd-session",pid=4242,fd=9))` + lst := `LISTEN 0 128 127.0.0.1:43293 0.0.0.0:* users:(("sshd-session",pid=4242,fd=12))` + ours := []netip.Addr{netip.MustParseAddr("127.0.0.1"), netip.MustParseAddr("::1"), netip.MustParseAddr("100.106.218.88")} + h := Holders(lst, conn, listen) + if len(h) != 1 { + t.Fatalf("%s: holders %+v", client, h) + } + if s := Stale(h, ours, nil); len(s) != 0 { + t.Errorf("%s: a far-host loopback session was marked stale: %+v", client, s) + } + } +}