diff --git a/README.md b/README.md index d55b439..ea8baa1 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,8 @@ agent: - **Which models work:** the VM's omp gets only `ANTHROPIC_BASE_URL` and `ANTHROPIC_API_KEY` (the token for that attempt), so `anthropic/*` models work through this path. - **One stage at a time:** each instance has one host port, so only one agent stage runs at a - time. A second one fails to start. + time. Agent stages that become ready meanwhile, in any pipeline, wait as `pending` and start + when the slot frees; their time limit starts only then. - **Crash recovery:** the sandbox job id is recorded before the VM is requested. Cleanup finds VMs by that id in the instance's inventory, so even a VM whose create response was lost gets destroyed. diff --git a/internal/agentexec/agentexec.go b/internal/agentexec/agentexec.go index 99f39a5..ad67d1b 100644 --- a/internal/agentexec/agentexec.go +++ b/internal/agentexec/agentexec.go @@ -174,6 +174,14 @@ func (x Executor) Execute(ctx context.Context, job stage.Job) stage.Result { return finish(stage.Succeeded, fmt.Sprintf("wrote %s (%d bytes)", st.Output, len(doc))) } +// MaxConcurrent forwards the backend's limit, if it has one. +func (x Executor) MaxConcurrent() int { + if l, ok := x.Backend.(stage.Limiter); ok { + return l.MaxConcurrent() + } + return 0 +} + // Cleanup forwards crash recovery to the backend, if it starts anything. func (x Executor) Cleanup(ctx context.Context, resource string) (bool, error) { if c, ok := x.Backend.(stage.Cleaner); ok { diff --git a/internal/agentexec/sandboxd.go b/internal/agentexec/sandboxd.go index a6012b7..a168595 100644 --- a/internal/agentexec/sandboxd.go +++ b/internal/agentexec/sandboxd.go @@ -179,6 +179,10 @@ func (s Sandboxd) Run(ctx context.Context, req Request, onLine func([]byte)) (in return code, err } +// MaxConcurrent is 1: the instance exposes one host port, which one run +// gateway at a time listens on. +func (Sandboxd) MaxConcurrent() int { return 1 } + // Cleanup destroys every VM created under a recorded job id. It reports done // once a listing shows none left AND the recorded settle time has passed, so // a create that sandboxd completes after the client gave up is still found. diff --git a/internal/engine/advance.go b/internal/engine/advance.go index 095893a..4b70021 100644 --- a/internal/engine/advance.go +++ b/internal/engine/advance.go @@ -115,6 +115,7 @@ func (e *Engine) advance(ctx context.Context, run store.Run) (bool, error) { } // Start every pending stage whose needs all succeeded. + waiting := false state := map[string]string{} for _, st := range stages { state[st.StageID] = st.State @@ -146,13 +147,23 @@ func (e *Engine) advance(ctx context.Context, run store.Run) (bool, error) { // next runner picks it up from here. return true, nil } - if err := e.start(ctx, run, sp, stg, st); err != nil { + if !e.reserve(stg.Kind()) { + // The executor is at its limit (e.g. one sandbox at a time): + // stay pending, and keep the run open until a slot frees. + waiting = true + continue + } + launched, err := e.start(ctx, run, sp, stg, st) + if !launched { + e.unreserve(stg.Kind()) + } + if err != nil { return true, err } state[st.StageID] = store.StageRunning running++ } - if running > 0 { + if running > 0 || waiting { return true, nil } @@ -209,8 +220,9 @@ func (e *Engine) foldStage(ctx context.Context, st *store.RunStage, state, summa return e.Store.UpdateStage(ctx, *st) } -// start records a new attempt for st and launches it. -func (e *Engine) start(ctx context.Context, run store.Run, sp spec.Spec, stg spec.Stage, st *store.RunStage) error { +// start records a new attempt for st and launches it. launched reports +// whether its goroutine is running (and now owns the execution slot). +func (e *Engine) start(ctx context.Context, run store.Run, sp spec.Spec, stg spec.Stage, st *store.RunStage) (launched bool, err error) { now := e.now() st.Attempts++ if st.StartedAt == nil { @@ -223,17 +235,17 @@ func (e *Engine) start(ctx context.Context, run store.Run, sp spec.Spec, stg spe a := store.Attempt{RunID: run.ID, StageID: stg.ID, Attempt: st.Attempts, State: store.AttemptRunning, LogPath: logPath, StartedAt: now} if err := e.Store.StartAttempt(ctx, *st, a); err != nil { st.Attempts-- - return err + return false, err } st.State = store.StageRunning - fail := func(summary string) error { + fail := func(summary string) (bool, error) { end := e.now() a.State, a.Summary, a.FinishedAt = store.AttemptFailed, summary, &end e.Log.Error("stage could not start", "run", run.ID, "stage", stg.ID, "err", summary) err := e.Store.FinishAttempt(ctx, a) e.signal() - return err + return false, err } for _, d := range []string{workDir, filepath.Dir(logPath), filepath.Dir(resultPath)} { if err := os.MkdirAll(d, 0o700); err != nil { @@ -318,13 +330,14 @@ func (e *Engine) start(ctx context.Context, run store.Run, sp spec.Spec, stg spe e.mu.Lock() delete(e.inflight, key) + e.running[stg.Kind()]-- for _, r := range tracked { delete(e.liveResources, r) } e.mu.Unlock() e.signal() }() - return nil + return true, nil } func exitString(code *int) string { diff --git a/internal/engine/engine.go b/internal/engine/engine.go index d9a8b29..a953204 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -52,6 +52,9 @@ type Engine struct { // liveResources are tracked by attempts still running in this process; // the cleanup sweep leaves them alone. liveResources map[string]bool + // running counts attempts in flight per stage kind, for executors with + // a concurrency limit (stage.Limiter). + running map[spec.Kind]int // stopping is set once Loop is shutting down; advance then only folds // results and never starts a stage. stopping atomic.Bool @@ -80,6 +83,7 @@ func New(st *store.Store, home string, executors map[spec.Kind]stage.Executor, b Poll: DefaultPoll, inflight: map[flightKey]context.CancelCauseFunc{}, liveResources: map[string]bool{}, + running: map[spec.Kind]int{}, wake: make(chan struct{}, 1), } } @@ -406,6 +410,28 @@ func (e *Engine) stopAll(only map[int64]bool) { } } +// reserve takes an execution slot for a stage of kind, if its executor's +// limit allows one more. Every successful reserve is paired with unreserve. +func (e *Engine) reserve(kind spec.Kind) bool { + limit := 0 + if l, ok := e.Executors[kind].(stage.Limiter); ok { + limit = l.MaxConcurrent() + } + e.mu.Lock() + defer e.mu.Unlock() + if limit > 0 && e.running[kind] >= limit { + return false + } + e.running[kind]++ + return true +} + +func (e *Engine) unreserve(kind spec.Kind) { + e.mu.Lock() + defer e.mu.Unlock() + e.running[kind]-- +} + func (e *Engine) isInflight(k flightKey) bool { e.mu.Lock() defer e.mu.Unlock() diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index cd2ce96..e548ba5 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -652,3 +652,56 @@ func TestScheduleJSONUsesDocumentedKeys(t *testing.T) { t.Errorf("schedule JSON %s", raw) } } + +// limitedExec allows one attempt at a time and fails any attempt that +// overlaps another, like a second run gateway on the one sandbox host port. +type limitedExec struct { + mu sync.Mutex + active int + overlap bool + ran int +} + +func (l *limitedExec) MaxConcurrent() int { return 1 } + +func (l *limitedExec) Execute(ctx context.Context, job stage.Job) stage.Result { + l.mu.Lock() + l.active++ + clash := l.active > 1 + l.overlap = l.overlap || clash + l.ran++ + l.mu.Unlock() + defer func() { l.mu.Lock(); l.active--; l.mu.Unlock() }() + if clash { + return stage.Result{Outcome: stage.Failed, Summary: "address already in use"} + } + time.Sleep(50 * time.Millisecond) + return stage.Result{Outcome: stage.Succeeded} +} + +// Agent stages of different pipelines wait for the executor's slot instead +// of failing; the waiting run is neither started early nor settled. +func TestLimitedExecutorQueuesStagesAcrossRuns(t *testing.T) { + h := newHarness(t, nil) + lim := &limitedExec{} + h.eng.Executors[spec.KindAgent] = lim + agent := func(name string) string { + return "name: " + name + "\nstages:\n - {id: r, agent: omp, prompt: x, output: o.json, max_cost: 1, timeout: 1m}\n" + } + h.add(agent("p1"), false) + h.add(agent("p2"), false) + r1, r2 := h.start("p1"), h.start("p2") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := h.eng.Loop(ctx, LoopOptions{Only: map[int64]bool{r1: true, r2: true}, UntilIdle: true}); err != nil { + t.Fatal(err) + } + for _, id := range []int64{r1, r2} { + if d, _ := h.eng.Run(context.Background(), id); d.State != store.RunSucceeded { + t.Errorf("run %d: %s (%s)", id, d.State, d.HaltReason) + } + } + if lim.overlap || lim.ran != 2 { + t.Errorf("overlap=%v ran=%d; want two sequential attempts", lim.overlap, lim.ran) + } +} diff --git a/internal/stage/stage.go b/internal/stage/stage.go index 093b353..9a06b2e 100644 --- a/internal/stage/stage.go +++ b/internal/stage/stage.go @@ -79,3 +79,10 @@ type Executor interface { type Cleaner interface { Cleanup(ctx context.Context, resource string) (done bool, err error) } + +// Limiter is implemented by executors that can run only a limited number of +// attempts at once (a sandboxd instance has one host port). The engine keeps +// further ready stages pending until a slot frees. Zero means no limit. +type Limiter interface { + MaxConcurrent() int +}