feat(tools): keep both ends of oversized tool output and spill the rest

.github/workflows/ci.yml

@@ -16,8 +16,11 @@ jobs:
         run: test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; }
       - name: vet
         run: go vet ./...
+      # -race, not bare go test: parallel tool calls, concurrent subagents, the
+      # process supervisor, and the TUI's signal handlers all share state, so a
+      # data race here is a lost keystroke or a hang, not a theoretical finding.
       - name: test
-        run: go test ./...
+        run: go test -race ./...
       # The end-to-end rig: the built binary, headless, against a scripted
       # endpoint. Catches assembled-product regressions unit tests cannot
       # (wire shape, gates, drive lifecycle, session persistence).

AGENTS.md

@@ -205,7 +205,10 @@ write results carry the diff they applied.
 ## Working in this repo
 
 - Verify before claiming done: `go build ./...`, `go vet ./...`, and
-  `go test ./...` must all pass. After changing a tool, exercise it with `bash`.
+  `go test -race ./...` must all pass. The race detector is part of the bar
+  because concurrency is part of the design: parallel tool calls, concurrent
+  subagents reporting usage, the process supervisor, and the TUI's signal
+  handlers all share state. After changing a tool, exercise it with `bash`.
 - **Tests earn their keep by failing.** Passing is not the bar; agents
   reliably write tests that pass. Before keeping any test, name the one-line
   code change that would make it fail. Cannot name one: it proves nothing,

README.md

@@ -195,7 +195,7 @@ The core exposes one extension surface (a `Hooks` struct plus tools-as-values) a
 ## Development
 
 ```sh
-gofmt -l . && go vet ./... && go test ./...
+gofmt -l . && go vet ./... && go test -race ./...
 ```
 
 CI enforces all three. Keep the core pure: if a change makes `agent/` import a provider, read input, or print, it is in the wrong package.

harness/continuity.go

@@ -83,11 +83,7 @@ func renderTranscript(turns []agent.Turn, maxResult int) string {
 			b.WriteByte('\n')
 		case "tool":
 			for _, res := range t.Results {
-				out := res.Content
-				if len(out) > maxResult {
-					out = out[:maxResult] + "..."
-				}
-				fmt.Fprintf(&b, "TOOL RESULT: %s\n", strings.ReplaceAll(out, "\n", " · "))
+				fmt.Fprintf(&b, "TOOL RESULT: %s\n", strings.ReplaceAll(elideText(res.Content, maxResult), "\n", " · "))
 			}
 			b.WriteByte('\n')
 		}
@@ -100,6 +96,26 @@ func renderTranscript(turns []agent.Turn, maxResult int) string {
 	return s
 }
 
+// elideText bounds one tool result inside the transcript, keeping its head AND
+// its tail around a marker, the same shape diff.go's elide gives a large diff
+// and the whole-transcript cut below gives a long transcript. It matters most
+// here: the judge rules done on this text, and a head-only cut of a failing
+// test run reads as an unbroken wall of PASS lines, which is worse than seeing
+// nothing at all. The head/tail split mirrors the transcript cut, so one ratio
+// governs both levels.
+func elideText(s string, max int) string {
+	if max <= 0 || len(s) <= max {
+		return s
+	}
+	const marker = " ... "
+	if max <= len(marker)+2 { // too small to keep both ends meaningfully
+		return s[:max]
+	}
+	budget := max - len(marker)
+	head := budget / 3
+	return s[:head] + marker + s[len(s)-(budget-head):]
+}
+
 // imageNote renders a one-line, byte-free summary of a user turn's images for
 // the transcript: enough for the brief writer to record that images existed and
 // their shape, never the data itself.

harness/continuity_test.go

@@ -56,11 +56,20 @@ func TestRenderTranscript(t *testing.T) {
 		{Role: "assistant", Text: "done"},
 	}
 	got := renderTranscript(h, 100)
-	for _, want := range []string{"USER: fix the bug", "ASSISTANT ran read", "TOOL RESULT: " + strings.Repeat("z", 100) + "...", "ASSISTANT: done"} {
+	for _, want := range []string{"USER: fix the bug", "ASSISTANT ran read", "ASSISTANT: done"} {
 		if !strings.Contains(got, want) {
 			t.Fatalf("transcript missing %q:\n%s", want, got)
 		}
 	}
+	// A tool result is elided from the MIDDLE: the head shows what the output
+	// was, the tail shows how it ended. Keeping only the head is what let a
+	// failing command reach the judge looking like a successful one.
+	if !strings.Contains(got, "TOOL RESULT: "+strings.Repeat("z", 30)) {
+		t.Fatalf("elided result lost its head:\n%s", got)
+	}
+	if !strings.Contains(got, "second line") {
+		t.Fatalf("elided result lost its tail:\n%s", got)
+	}
 	if strings.Contains(got, strings.Repeat("z", 101)) {
 		t.Fatal("tool results must be elided to the stub")
 	}

harness/doctor.go

@@ -218,6 +218,15 @@ func runDoctor() int {
 		pass("%s: writable, %d sessions", sessionsDir(), len(allSessions()))
 	}
 
+	// Spilled output is the one mount point that grows on its own, so doctor
+	// reports what is there and confirms the sweep has something to do.
+	if n, bytes := outputUsage(); n > 0 {
+		pass("%s: %d conversation(s), %s (swept after %d idle days)",
+			outputsDir(), n, humanBytes(bytes), tune.ResultKeepDays)
+	} else {
+		pass("%s: empty", outputsDir())
+	}
+
 	if ok {
 		fmt.Println("\nall checks passed")
 		return 0

harness/drive.go

@@ -215,7 +215,7 @@ func drive(r *repl, cfg driveConfig, firstTurns []agent.Turn) int {
 			// Escape pauses the drive during the judge phase too (not just during a
 			// streamed worker iteration).
 			jctx, jdone := turnCtx()
-			transcript := renderTranscript(iterTurns, 300)
+			transcript := renderTranscript(iterTurns, tune.TranscriptResult)
 			var jUsed agent.Usage
 			var jerr error
 			v, jUsed, jerr = judgeGoal(jctx, r.p, cfg.request, transcript)

harness/e2e_test.go

@@ -667,4 +667,93 @@ func TestE2E(t *testing.T) {
 			t.Fatal("a denied write must not touch disk")
 		}
 	})
+	// The defect this feature exists for, end to end through the real binary: a
+	// failing test run whose output exceeds the window budget must still reach
+	// the model with its verdict, and the elided middle must be recoverable with
+	// the read tool at the offset the pointer names.
+	t.Run("OversizedOutputKeepsVerdictAndSpills", func(t *testing.T) {
+		m, dir := newRig(t,
+			[]e2eStep{
+				eTool("bash", `{"command":"sh gen.sh"}`),
+				eText("the suite failed"),
+			},
+			[]e2eStep{verdictJSON("reported the failure")})
+		// A generator whose output brackets the budget: PASS noise around a
+		// diagnostic in the middle, and the verdict on the final line.
+		gen := "#!/bin/sh\n" +
+			"i=0; while [ $i -lt 900 ]; do echo \"=== RUN   TestAlpha$i\"; echo \"--- PASS: TestAlpha$i (0.00s)\"; i=$((i+1)); done\n" +
+			"echo '    a_test.go:403: boom: nil map write at pkg/store.go:88'\n" +
+			"i=0; while [ $i -lt 900 ]; do echo \"=== RUN   TestBeta$i\"; echo \"--- PASS: TestBeta$i (0.00s)\"; i=$((i+1)); done\n" +
+			"echo 'FAIL\tsigprobe/pkg\t0.013s'\n"
+		os.WriteFile(filepath.Join(dir, "gen.sh"), []byte(gen), 0o755)
+
+		m.run(t, dir, "run the suite and tell me whether it passed")
+
+		res := m.lastToolResult(t, 2)
+		// The judge rules done on the transcript, so the verdict must survive the
+		// per-result elision too: a head-only cut fed it a wall of PASS lines
+		// from a run that failed.
+		judged := false
+		m.mu.Lock()
+		for _, r := range m.reqs {
+			if r.Class != "judge" {
+				continue
+			}
+			msgs, _ := r.Body["messages"].([]any)
+			for _, mm := range msgs {
+				x, ok := mm.(map[string]any)
+				if !ok {
+					continue
+				}
+				if c, _ := x["content"].(string); strings.Contains(c, "FAIL\tsigprobe/pkg") {
+					judged = true
+				}
+			}
+		}
+		m.mu.Unlock()
+		if !judged {
+			t.Error("the judge must see the failing verdict in the transcript")
+		}
+		if len(res) > tune.ResultMaxChars {
+			t.Fatalf("result reached the model unbounded: %d bytes", len(res))
+		}
+		if !strings.Contains(res, "FAIL\tsigprobe/pkg") {
+			t.Fatalf("the verdict on the last line must survive:\n%s", tailOf(res, 400))
+		}
+		if !strings.Contains(res, "TestAlpha0") {
+			t.Fatalf("the head must survive:\n%s", headOf(res, 400))
+		}
+
+		// The pointer must name a real file, and it must hold what was elided.
+		i := strings.Index(res, "full output: ")
+		if i < 0 {
+			t.Fatalf("no spill pointer in the shaped result:\n%s", res[max(0, len(res)-400):])
+		}
+		path := res[i+len("full output: "):]
+		path = path[:strings.Index(path, " ")]
+		body, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatalf("spilled output unreadable at the path the model was given: %v", err)
+		}
+		if !strings.Contains(string(body), "boom: nil map write") {
+			t.Fatal("the spilled file must hold the elided diagnostic")
+		}
+		if !strings.Contains(string(body), "TestBeta450") {
+			t.Fatal("the spilled file must hold the full output, not just the shaped ends")
+		}
+	})
+}
+
+func headOf(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n]
+}
+
+func tailOf(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[len(s)-n:]
 }

harness/engines.go

@@ -20,15 +20,15 @@ func engineTools() ([]agent.Tool, []string, string) {
 	skill, n, ok := skillTool()
 	notes = append(notes, n...)
 	if ok {
-		tools = append(tools, skill)
+		tools = append(tools, shaped(skill))
 		if !tune.SkillNoteOff {
 			sysNote = "\n\n<skills>\nSkills are installed. Before starting a task, check the skill tool's manifest: when a line matches the task, load that skill first and follow its instructions.\n</skills>"
 		}
 	}
 	mcp, n, ok := mcpTool()
 	notes = append(notes, n...)
 	if ok {
-		tools = append(tools, mcp)
+		tools = append(tools, shaped(mcp))
 	}
 	return tools, notes, sysNote
 }

harness/harness.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -193,7 +194,11 @@ func Main() {
 	sweepDeadProcs(sess.ID) // reap processes a previously-crashed sesh left behind
 	pm := newProcManager(sess.ID)
 	os.Setenv("SESH_SESSION", sess.ID) // tool/gate/statusline mods can find this session's run dir
-	go gcBlobs()                       // sweep orphaned image blobs off the hot path; best-effort, never blocks startup
+	// Spilled tool output is keyed by the chain root, so it survives every
+	// handoff without rewriting the pointers already in the transcript.
+	spill = newOutStore(outputDir(sess))
+	go gcBlobs()                          // sweep orphaned image blobs off the hot path; best-effort, never blocks startup
+	go gcOutput(filepath.Base(spill.dir)) // same for spilled output of conversations long finished
 	tools := builtinTools(*unsafePaths, pm)
 	// The engines (skill, mcp) join only when their user-space content exists:
 	// an empty mount costs zero tokens. They are built-ins, so they claim

harness/help.go

@@ -97,8 +97,10 @@ FILES AND MODS (project .sesh/ overrides global ~/.sesh/)
                                seed_ledger_entries, task_depth, stuck_after,
                                recall_links, diff_lines, proc_promote_secs,
                                max_procs, proc_log_tail, update_check,
-                               input_max_rows, brief_provider, brief_model
-                               (state only what you change)
+                               input_max_rows, brief_provider, brief_model,
+                               result_max_chars, result_head_pct,
+                               result_spill_off, result_keep_days,
+                               transcript_result (state only what you change)
   tools/<name>                 executables that become agent tools (global
                                mount only): --schema describes, args JSON on
                                stdin, stdout is the result; mutating ones
@@ -108,6 +110,9 @@ FILES AND MODS (project .sesh/ overrides global ~/.sesh/)
                                line is the reason); broken mod fails closed
   statusline                   executable; JSON on stdin, first line shown
   sessions/, chains/           transcripts and chain ledgers (plain JSON/JSONL)
+  out/                         full text of tool output too large for the
+                               context window, keyed by conversation; the
+                               shaped result points here and read can page it
   run/                         background-process logs and crash records,
                                cleared when a session exits
 

harness/main_test.go

@@ -281,14 +281,18 @@ func TestBashOutputCap(t *testing.T) {
 	}
 }
 
+// TestCappedBuffer: the buffer keeps the TAIL, evicting from the head like the
+// process supervisor's ring. A command that outgrows the cap is almost always a
+// build or test run, and its verdict is on the last lines. Breaker: keep the
+// head instead and "efgh" no longer ends the buffer.
 func TestCappedBuffer(t *testing.T) {
 	c := &cappedBuffer{max: 5}
 	for _, chunk := range []string{"ab", "cd", "efgh"} {
 		if n, err := c.Write([]byte(chunk)); err != nil || n != len(chunk) {
 			t.Fatalf("write must report full consumption: n=%d err=%v", n, err)
 		}
 	}
-	if string(c.buf) != "abcde" || c.dropped != 3 {
+	if string(c.buf) != "defgh" || c.dropped != 3 {
 		t.Fatalf("buf=%q dropped=%d", c.buf, c.dropped)
 	}
 }

harness/mcp.go

@@ -622,8 +622,9 @@ func runMCP(ctx context.Context, pool *mcpPool, raw json.RawMessage) (string, bo
 
 // renderMCPResult flattens an MCP content array to the one string a sesh
 // tool returns: text blocks verbatim, other block types as markers, isError
-// surfacing as a tool error the model can act on. Oversize output is
-// truncated loudly, the same cap as bash.
+// surfacing as a tool error the model can act on. Bounding the size is the
+// shaper's job at assembly, which keeps both ends and spills the rest, so
+// there is no second cut here to lose the tail before it gets there.
 func renderMCPResult(res json.RawMessage) (string, bool) {
 	var r struct {
 		Content []struct {
@@ -647,8 +648,5 @@ func renderMCPResult(res json.RawMessage) (string, bool) {
 	if out == "" {
 		out = "(no output)"
 	}
-	if len(out) > maxBashOutput {
-		out = out[:maxBashOutput] + "\n[output truncated at 1MB]"
-	}
 	return out, r.IsError
 }

harness/mcp_test.go

@@ -129,6 +129,9 @@ func mcpCall(t *testing.T, tool string, args string) (string, bool) {
 	if !ok {
 		t.Fatal("mcp tool did not activate")
 	}
+	// shaped() is what engineTools applies at assembly, so bounding the result
+	// is exercised here the way the model actually meets it.
+	tl = shaped(tl)
 	raw := json.RawMessage(fmt.Sprintf(`{"server":"gauntlet","tool":%q,"args":%s}`, tool, args))
 	return tl.Run(context.Background(), raw)
 }
@@ -273,15 +276,15 @@ func TestMCPManifestCacheFeedsNextSession(t *testing.T) {
 	}
 }
 
-// Breaker: return server output unbounded.
+// Breaker: return server output unbounded, or bound it without saying so.
 func TestMCPBigOutputTruncatedLoudly(t *testing.T) {
 	mcpHome(t, map[string]mcpServerConf{"gauntlet": gauntletConf(nil)})
 	out, _ := mcpCall(t, "big", `{}`)
-	if len(out) > maxBashOutput+100 {
-		t.Fatalf("output not capped: %d bytes", len(out))
+	if len(out) > tune.ResultMaxChars {
+		t.Fatalf("output not bounded: %d bytes", len(out))
 	}
-	if !strings.Contains(out, "[output truncated") {
-		t.Fatal("truncation must be loud, not silent")
+	if !strings.Contains(out, "elided") {
+		t.Fatalf("bounding must be loud, not silent:\n%s", out[max(0, len(out)-300):])
 	}
 }
 

harness/repl.go

@@ -1061,7 +1061,7 @@ func (r *repl) handoff() bool {
 	} else {
 		emit("%s  writing handoff brief...%s\n", dim, reset)
 	}
-	brief, entry, used, err := writeBrief(context.Background(), bp, renderTranscript(r.history, 300))
+	brief, entry, used, err := writeBrief(context.Background(), bp, renderTranscript(r.history, tune.TranscriptResult))
 	if err != nil {
 		emit("%s  handoff brief failed: %v%s\n", red, err, reset)
 		return false

harness/scaffold/tuning.json.example

@@ -62,6 +62,18 @@
   // "proc_log_tail": 200,       // default lines a `proc logs` read returns
   // "proc_spill_off": false,    // true keeps logs in memory only (no run/ file)
 
+  // --- Tool output too large for the window ---
+  // An over-budget result keeps its head AND its tail around an elided middle,
+  // and the full text spills to ~/.sesh/out/<conversation>/ so the model can
+  // read what was elided. Both ends matter: a failing command's diagnostic can
+  // sit anywhere in the output, but its verdict is always on the last lines.
+  // "result_max_chars": 28000,  // budget for one shaped tool result
+  // "result_head_pct": 25,      // share of that budget given to the head
+  // "result_spill_off": false,  // true keeps the shaped result but writes no file
+  // "result_keep_days": 7,      // how long spilled output outlives its last write
+  // "transcript_result": 300,   // per-result budget in the transcript the judge
+  //                             // and the brief writer read (also head + tail)
+
   // --- Other dials (full reference: ~/.sesh/README.md and -help) ---
   // "task_depth": 3,            // subagent nesting cap
   // "stuck_after": 3,           // driven iterations w/o a mutation before "stuck"