Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,8 @@ agent:
- **Which models work:** the VM's omp gets only `ANTHROPIC_BASE_URL` and `ANTHROPIC_API_KEY`
(the token for that attempt), so `anthropic/*` models work through this path.
- **One stage at a time:** each instance has one host port, so only one agent stage runs at a
time. A second one fails to start.
time. Agent stages that become ready meanwhile, in any pipeline, wait as `pending` and start
when the slot frees; their time limit starts only then.
- **Crash recovery:** the sandbox job id is recorded before the VM is requested. Cleanup finds
VMs by that id in the instance's inventory, so even a VM whose create response was lost gets
destroyed.
Expand Down
8 changes: 8 additions & 0 deletions internal/agentexec/agentexec.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,14 @@ func (x Executor) Execute(ctx context.Context, job stage.Job) stage.Result {
return finish(stage.Succeeded, fmt.Sprintf("wrote %s (%d bytes)", st.Output, len(doc)))
}

// MaxConcurrent forwards the backend's limit, if it has one.
func (x Executor) MaxConcurrent() int {
if l, ok := x.Backend.(stage.Limiter); ok {
return l.MaxConcurrent()
}
return 0
}

// Cleanup forwards crash recovery to the backend, if it starts anything.
func (x Executor) Cleanup(ctx context.Context, resource string) (bool, error) {
if c, ok := x.Backend.(stage.Cleaner); ok {
Expand Down
4 changes: 4 additions & 0 deletions internal/agentexec/sandboxd.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,10 @@ func (s Sandboxd) Run(ctx context.Context, req Request, onLine func([]byte)) (in
return code, err
}

// MaxConcurrent is 1: the instance exposes one host port, which one run
// gateway at a time listens on.
func (Sandboxd) MaxConcurrent() int { return 1 }

// Cleanup destroys every VM created under a recorded job id. It reports done
// once a listing shows none left AND the recorded settle time has passed, so
// a create that sandboxd completes after the client gave up is still found.
Expand Down
29 changes: 21 additions & 8 deletions internal/engine/advance.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ func (e *Engine) advance(ctx context.Context, run store.Run) (bool, error) {
}

// Start every pending stage whose needs all succeeded.
waiting := false
state := map[string]string{}
for _, st := range stages {
state[st.StageID] = st.State
Expand Down Expand Up @@ -146,13 +147,23 @@ func (e *Engine) advance(ctx context.Context, run store.Run) (bool, error) {
// next runner picks it up from here.
return true, nil
}
if err := e.start(ctx, run, sp, stg, st); err != nil {
if !e.reserve(stg.Kind()) {
// The executor is at its limit (e.g. one sandbox at a time):
// stay pending, and keep the run open until a slot frees.
waiting = true
continue
}
launched, err := e.start(ctx, run, sp, stg, st)
if !launched {
e.unreserve(stg.Kind())
}
if err != nil {
return true, err
}
state[st.StageID] = store.StageRunning
running++
}
if running > 0 {
if running > 0 || waiting {
return true, nil
}

Expand Down Expand Up @@ -209,8 +220,9 @@ func (e *Engine) foldStage(ctx context.Context, st *store.RunStage, state, summa
return e.Store.UpdateStage(ctx, *st)
}

// start records a new attempt for st and launches it.
func (e *Engine) start(ctx context.Context, run store.Run, sp spec.Spec, stg spec.Stage, st *store.RunStage) error {
// start records a new attempt for st and launches it. launched reports
// whether its goroutine is running (and now owns the execution slot).
func (e *Engine) start(ctx context.Context, run store.Run, sp spec.Spec, stg spec.Stage, st *store.RunStage) (launched bool, err error) {
now := e.now()
st.Attempts++
if st.StartedAt == nil {
Expand All @@ -223,17 +235,17 @@ func (e *Engine) start(ctx context.Context, run store.Run, sp spec.Spec, stg spe
a := store.Attempt{RunID: run.ID, StageID: stg.ID, Attempt: st.Attempts, State: store.AttemptRunning, LogPath: logPath, StartedAt: now}
if err := e.Store.StartAttempt(ctx, *st, a); err != nil {
st.Attempts--
return err
return false, err
}
st.State = store.StageRunning

fail := func(summary string) error {
fail := func(summary string) (bool, error) {
end := e.now()
a.State, a.Summary, a.FinishedAt = store.AttemptFailed, summary, &end
e.Log.Error("stage could not start", "run", run.ID, "stage", stg.ID, "err", summary)
err := e.Store.FinishAttempt(ctx, a)
e.signal()
return err
return false, err
}
for _, d := range []string{workDir, filepath.Dir(logPath), filepath.Dir(resultPath)} {
if err := os.MkdirAll(d, 0o700); err != nil {
Expand Down Expand Up @@ -318,13 +330,14 @@ func (e *Engine) start(ctx context.Context, run store.Run, sp spec.Spec, stg spe

e.mu.Lock()
delete(e.inflight, key)
e.running[stg.Kind()]--
for _, r := range tracked {
delete(e.liveResources, r)
}
e.mu.Unlock()
e.signal()
}()
return nil
return true, nil
}

func exitString(code *int) string {
Expand Down
26 changes: 26 additions & 0 deletions internal/engine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ type Engine struct {
// liveResources are tracked by attempts still running in this process;
// the cleanup sweep leaves them alone.
liveResources map[string]bool
// running counts attempts in flight per stage kind, for executors with
// a concurrency limit (stage.Limiter).
running map[spec.Kind]int
// stopping is set once Loop is shutting down; advance then only folds
// results and never starts a stage.
stopping atomic.Bool
Expand Down Expand Up @@ -80,6 +83,7 @@ func New(st *store.Store, home string, executors map[spec.Kind]stage.Executor, b
Poll: DefaultPoll,
inflight: map[flightKey]context.CancelCauseFunc{},
liveResources: map[string]bool{},
running: map[spec.Kind]int{},
wake: make(chan struct{}, 1),
}
}
Expand Down Expand Up @@ -406,6 +410,28 @@ func (e *Engine) stopAll(only map[int64]bool) {
}
}

// reserve takes an execution slot for a stage of kind, if its executor's
// limit allows one more. Every successful reserve is paired with unreserve.
func (e *Engine) reserve(kind spec.Kind) bool {
limit := 0
if l, ok := e.Executors[kind].(stage.Limiter); ok {
limit = l.MaxConcurrent()
}
e.mu.Lock()
defer e.mu.Unlock()
if limit > 0 && e.running[kind] >= limit {
return false
}
e.running[kind]++
return true
}

func (e *Engine) unreserve(kind spec.Kind) {
e.mu.Lock()
defer e.mu.Unlock()
e.running[kind]--
}

func (e *Engine) isInflight(k flightKey) bool {
e.mu.Lock()
defer e.mu.Unlock()
Expand Down
53 changes: 53 additions & 0 deletions internal/engine/engine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -652,3 +652,56 @@ func TestScheduleJSONUsesDocumentedKeys(t *testing.T) {
t.Errorf("schedule JSON %s", raw)
}
}

// limitedExec allows one attempt at a time and fails any attempt that
// overlaps another, like a second run gateway on the one sandbox host port.
type limitedExec struct {
mu sync.Mutex
active int
overlap bool
ran int
}

func (l *limitedExec) MaxConcurrent() int { return 1 }

func (l *limitedExec) Execute(ctx context.Context, job stage.Job) stage.Result {
l.mu.Lock()
l.active++
clash := l.active > 1
l.overlap = l.overlap || clash
l.ran++
l.mu.Unlock()
defer func() { l.mu.Lock(); l.active--; l.mu.Unlock() }()
if clash {
return stage.Result{Outcome: stage.Failed, Summary: "address already in use"}
}
time.Sleep(50 * time.Millisecond)
return stage.Result{Outcome: stage.Succeeded}
}

// Agent stages of different pipelines wait for the executor's slot instead
// of failing; the waiting run is neither started early nor settled.
func TestLimitedExecutorQueuesStagesAcrossRuns(t *testing.T) {
h := newHarness(t, nil)
lim := &limitedExec{}
h.eng.Executors[spec.KindAgent] = lim
agent := func(name string) string {
return "name: " + name + "\nstages:\n - {id: r, agent: omp, prompt: x, output: o.json, max_cost: 1, timeout: 1m}\n"
}
h.add(agent("p1"), false)
h.add(agent("p2"), false)
r1, r2 := h.start("p1"), h.start("p2")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := h.eng.Loop(ctx, LoopOptions{Only: map[int64]bool{r1: true, r2: true}, UntilIdle: true}); err != nil {
t.Fatal(err)
}
for _, id := range []int64{r1, r2} {
if d, _ := h.eng.Run(context.Background(), id); d.State != store.RunSucceeded {
t.Errorf("run %d: %s (%s)", id, d.State, d.HaltReason)
}
}
if lim.overlap || lim.ran != 2 {
t.Errorf("overlap=%v ran=%d; want two sequential attempts", lim.overlap, lim.ran)
}
}
7 changes: 7 additions & 0 deletions internal/stage/stage.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,3 +79,10 @@ type Executor interface {
type Cleaner interface {
Cleanup(ctx context.Context, resource string) (done bool, err error)
}

// Limiter is implemented by executors that can run only a limited number of
// attempts at once (a sandboxd instance has one host port). The engine keeps
// further ready stages pending until a slot frees. Zero means no limit.
type Limiter interface {
MaxConcurrent() int
}
Loading