From 578bf0cc8f8dcbec0361e9aa0918e90cd530e5ed Mon Sep 17 00:00:00 2001 From: jerryfane Date: Fri, 6 Nov 2026 03:09:22 +0100 Subject: [PATCH 01/11] =?UTF-8?q?feat:=20start=20pipelines=20from=20extern?= =?UTF-8?q?al=20events=20=E2=80=94=20signed=20HTTP=20trigger,=20CLI=20trig?= =?UTF-8?q?ger,=20idempotency,=20bounded=20queue,=20payload=20retention=20?= =?UTF-8?q?(#18)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/cli/cli.go | 29 ++- internal/cli/events.go | 216 +++++++++++++++++++++ internal/config/config.go | 29 ++- internal/engine/advance.go | 23 ++- internal/engine/engine.go | 23 ++- internal/engine/engine_test.go | 4 +- internal/engine/events.go | 322 +++++++++++++++++++++++++++++++ internal/engine/events_test.go | 281 +++++++++++++++++++++++++++ internal/engine/status.go | 56 +++++- internal/spec/spec.go | 16 ++ internal/store/events.go | 291 ++++++++++++++++++++++++++++ internal/store/store.go | 39 +++- internal/trigger/trigger.go | 176 +++++++++++++++++ internal/trigger/trigger_test.go | 140 ++++++++++++++ 14 files changed, 1622 insertions(+), 23 deletions(-) create mode 100644 internal/cli/events.go create mode 100644 internal/engine/events.go create mode 100644 internal/engine/events_test.go create mode 100644 internal/store/events.go create mode 100644 internal/trigger/trigger.go create mode 100644 internal/trigger/trigger_test.go diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 88b2a5f..76552e1 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -50,9 +50,17 @@ Runs: resume [--from STAGE] [--wait] re-run a parked run from a stage cancel stop a running run +Events: + trigger [--event-id ID] [--payload-file F] [--content-type T] [--wait] [--json] + queue an event run (payload from the file, else stdin) + events [--pipeline N] [--state S] [--json] accepted events, newest first + drop-event discard a queued event and its payload + purge delete a finished event run's files and payload + Service: - serve [--status-addr 127.0.0.1:PORT] [--poll 5s] run schedules and stages until stopped - tick one pass: fire due schedules, run them to the end + serve [--status-addr ADDR] [--trigger-addr ADDR] [--poll 5s] + run schedules, events and stages until stopped + tick one pass: fire due schedules and queued events, run them to the end gateway-proxy [--listen ADDR] [--upstream URL] [--token-file FILE] model credential proxy for sandboxed agent stages tunnel-clear-stale --ssh-host HOST --remote-listen ADDR:PORT @@ -126,6 +134,7 @@ func Main(args []string, stdout, stderr io.Writer) int { "add": a.add, "list": a.list, "show": a.show, "enable": a.enable, "disable": a.disable, "remove": a.remove, "run": a.run, "runs": a.runs, "resume": a.resume, "cancel": a.cancel, "serve": a.serve, "tick": a.tick, "status": a.status, + "trigger": a.trigger, "events": a.events, "drop-event": a.dropEvent, "purge": a.purge, } fn, ok := commands[cmd] if !ok { @@ -152,6 +161,7 @@ type exitCode int func (e exitCode) Error() string { return fmt.Sprintf("exit %d", int(e)) } type app struct { + cfg config.Config home string store *store.Store engine *engine.Engine @@ -232,7 +242,7 @@ func open(ctx context.Context, home, configPath string, stderr io.Writer) (*app, }} } eng := engine.New(st, home, executors, baseEnv(), log) - return &app{home: home, store: st, engine: eng, stderr: stderr, log: log}, nil + return &app{cfg: cfg, home: home, store: st, engine: eng, stderr: stderr, log: log}, nil } func (a *app) close() { _ = a.store.Close() } @@ -674,8 +684,10 @@ func (a *app) execute(ctx context.Context, id int64, wait bool) (bool, error) { func (a *app) serve(ctx context.Context, args []string) error { fs := flag.NewFlagSet("serve", flag.ContinueOnError) statusAddr := fs.String("status-addr", "", "loopback address for GET /status (off when empty)") + triggerAddr := fs.String("trigger-addr", "", "address for POST /v1/pipelines/{name}/events (off when empty)") + allowRemote := fs.Bool("trigger-allow-remote", false, "allow a non-loopback --trigger-addr") poll := fs.Duration("poll", engine.DefaultPoll, "") - if _, err := parse(fs, args, 0, "serve [--status-addr 127.0.0.1:PORT] [--poll 5s]"); err != nil { + if _, err := parse(fs, args, 0, "serve [--status-addr 127.0.0.1:PORT] [--trigger-addr 127.0.0.1:PORT [--trigger-allow-remote]] [--poll 5s]"); err != nil { return err } lock, err := homelock.TryAcquire(filepath.Join(a.home, "stagehand.lock"), homelock.ModeServe) @@ -693,6 +705,13 @@ func (a *app) serve(ctx context.Context, args []string) error { } defer srv.Close() } + if *triggerAddr != "" { + srv, err := a.triggerServer(*triggerAddr, *allowRemote) + if err != nil { + return err + } + defer srv.Close() + } a.engine.Poll = *poll a.log.Info("serving", "home", a.home, "version", buildinfo.String()) err = a.engine.Loop(ctx, engine.LoopOptions{Schedule: true}) @@ -719,7 +738,7 @@ func (a *app) tick(ctx context.Context, args []string) error { if err := a.engine.Recover(ctx); err != nil { return err } - return a.engine.Loop(ctx, engine.LoopOptions{Schedule: true, UntilIdle: true}) + return a.engine.Loop(ctx, engine.LoopOptions{Schedule: true, Events: true, UntilIdle: true}) } // statusServer serves GET /status on a loopback address only. diff --git a/internal/cli/events.go b/internal/cli/events.go new file mode 100644 index 0000000..15890a9 --- /dev/null +++ b/internal/cli/events.go @@ -0,0 +1,216 @@ +package cli + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "text/tabwriter" + "time" + + "github.com/gitmoot/stagehand/internal/engine" + "github.com/gitmoot/stagehand/internal/homelock" + "github.com/gitmoot/stagehand/internal/store" + "github.com/gitmoot/stagehand/internal/trigger" +) + +// trigger queues an event from the local command line. It is trusted (the +// caller can already read the home), so it needs no signature. +func (a *app) trigger(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("trigger", flag.ContinueOnError) + eventID := fs.String("event-id", "", "idempotency key; the same key yields one run") + payloadFile := fs.String("payload-file", "", "payload file (default: stdin)") + contentType := fs.String("content-type", "", "payload media type") + wait := fs.Bool("wait", false, "wait for the event's run to finish") + asJSON := fs.Bool("json", false, "") + pos, err := parse(fs, args, 1, "trigger [--event-id ID] [--payload-file F] [--content-type T] [--wait] [--json]") + if err != nil { + return err + } + var payload io.Reader = os.Stdin + if *payloadFile != "" { + f, err := os.Open(*payloadFile) + if err != nil { + return err + } + defer f.Close() + payload = f + } + res, err := a.engine.Trigger(ctx, engine.EventRequest{ + Pipeline: pos[0], Key: *eventID, ContentType: *contentType, Source: engine.SourceCLI, + Payload: payload, MaxBytes: a.cfg.Events.PayloadLimit(), + }) + if err != nil { + return err + } + ev := res.Event + if res.Duplicate { + fmt.Fprintf(a.stderr, "event %q already received (event %d, %s); no new run\n", ev.Key, ev.ID, ev.State) + } else { + fmt.Fprintf(a.stderr, "event %d queued (%d bytes)\n", ev.ID, ev.PayloadBytes) + } + if !*wait { + if *asJSON { + return a.printJSON(map[string]any{"duplicate": res.Duplicate, "event": engine.EventRefOf(ev), "state": ev.State, "run_id": ev.RunID}) + } + fmt.Fprintf(a.stdout, "event %d: %s\n", ev.ID, ev.State) + return nil + } + runID, err := a.drainUntilStarted(ctx, ev.ID) + if err != nil { + return err + } + return a.finish(ctx, runID, true, *asJSON) +} + +// drainUntilStarted waits for event id to get a run, executing queued events +// here when no other process executes stages. +func (a *app) drainUntilStarted(ctx context.Context, id int64) (int64, error) { + for { + ev, err := a.store.GetEvent(ctx, id) + if err != nil { + return 0, err + } + switch ev.State { + case store.EventStarted: + return ev.RunID, nil + case store.EventDropped: + return 0, fmt.Errorf("event %d was dropped", id) + } + lock, err := homelock.TryAcquire(filepath.Join(a.home, "stagehand.lock"), homelock.ModeCLI) + if err == nil { + rerr := a.engine.Recover(ctx) + if rerr == nil { + // Start (and run) queued events until this one has a run. + rerr = a.engine.Loop(ctx, engine.LoopOptions{Events: true, UntilIdle: true}) + } + lock.Release() + if rerr != nil { + return 0, rerr + } + continue + } + if !errors.Is(err, homelock.ErrHeld) { + return 0, err + } + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-time.After(lockPoll): + } + } +} + +func (a *app) events(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("events", flag.ContinueOnError) + pipeline := fs.String("pipeline", "", "only this pipeline") + state := fs.String("state", "", "queued, started or dropped") + limit := fs.Int("limit", 50, "") + asJSON := fs.Bool("json", false, "") + if _, err := parse(fs, args, 0, "events [--pipeline NAME] [--state S] [--limit N] [--json]"); err != nil { + return err + } + evs, err := a.store.ListEvents(ctx, *pipeline, *state, *limit) + if err != nil { + return err + } + if *asJSON { + type row struct { + *engine.EventRef + Pipeline string `json:"pipeline"` + State string `json:"state"` + RunID int64 `json:"run_id,omitempty"` + } + rows := []row{} + for _, e := range evs { + rows = append(rows, row{EventRef: engine.EventRefOf(e), Pipeline: e.Pipeline, State: e.State, RunID: e.RunID}) + } + return a.printJSON(rows) + } + tw := tabwriter.NewWriter(a.stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, "EVENT\tPIPELINE\tKEY\tSTATE\tRUN\tRECEIVED\tBYTES\tSOURCE") + for _, e := range evs { + run := "-" + if e.RunID != 0 { + run = "#" + strconv.FormatInt(e.RunID, 10) + } + fmt.Fprintf(tw, "%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", e.ID, e.Pipeline, e.Key, e.State, run, timeText(&e.ReceivedAt), e.PayloadBytes, e.Source) + } + return tw.Flush() +} + +func (a *app) dropEvent(ctx context.Context, args []string) error { + pos, err := parse(flag.NewFlagSet("drop-event", flag.ContinueOnError), args, 1, "drop-event ") + if err != nil { + return err + } + id, err := strconv.ParseInt(pos[0], 10, 64) + if err != nil { + return fmt.Errorf("event id %q is not a number", pos[0]) + } + ev, err := a.engine.DropEvent(ctx, id) + if err != nil { + return err + } + fmt.Fprintf(a.stdout, "event %d (%s) dropped; its payload is deleted\n", ev.ID, ev.Key) + return nil +} + +func (a *app) purge(ctx context.Context, args []string) error { + pos, err := parse(flag.NewFlagSet("purge", flag.ContinueOnError), args, 1, "purge ") + if err != nil { + return err + } + id, err := strconv.ParseInt(pos[0], 10, 64) + if err != nil { + return fmt.Errorf("run id %q is not a number", pos[0]) + } + if err := a.engine.Purge(ctx, id); err != nil { + return err + } + fmt.Fprintf(a.stdout, "run #%d: files and payload deleted; it can no longer be resumed\n", id) + return nil +} + +// triggerServer serves the signed event endpoint. +func (a *app) triggerServer(addr string, allowRemote bool) (*http.Server, error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("trigger-addr: %v", err) + } + if ip := net.ParseIP(host); !allowRemote && host != "localhost" && (ip == nil || !ip.IsLoopback()) { + return nil, fmt.Errorf("trigger-addr %s is not loopback; pass --trigger-allow-remote to expose it", addr) + } + secretFile := a.cfg.Events.SecretFile + if secretFile == "" { + return nil, errors.New("trigger-addr needs events.secret_file in the runner config (a 0600 file with the HMAC key)") + } + if _, err := readSecretFile(secretFile); err != nil { + return nil, fmt.Errorf("events.secret_file: %w", err) + } + ln, err := net.Listen("tcp", addr) + if err != nil { + return nil, err + } + mux := http.NewServeMux() + trigger.Handler{ + Engine: a.engine, + Secret: func() ([]byte, error) { s, err := readSecretFile(secretFile); return []byte(s), err }, + MaxBytes: a.cfg.Events.PayloadLimit(), + Log: a.log, + }.Routes(mux) + srv := &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 10 * time.Minute} + go func() { + if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { + a.log.Error("trigger server", "err", err) + } + }() + a.log.Info("trigger endpoint", "url", "http://"+ln.Addr().String()+"/v1/pipelines/{name}/events") + return srv, nil +} diff --git a/internal/config/config.go b/internal/config/config.go index 4cac2da..c3c8025 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -26,7 +26,28 @@ const ( // Config is the settings file. type Config struct { - Agent Agent `yaml:"agent"` + Agent Agent `yaml:"agent"` + Events Events `yaml:"events"` +} + +// DefaultMaxPayloadBytes is the default event payload limit (owner, 6 Nov). +const DefaultMaxPayloadBytes = 64 << 20 + +// Events configures event triggers on this host. +type Events struct { + // SecretFile holds the HMAC key for the HTTP trigger (0600). Without it + // the HTTP trigger refuses to start; the local CLI trigger needs none. + SecretFile string `yaml:"secret_file"` + // MaxPayloadBytes bounds one event's payload (default 64 MiB). + MaxPayloadBytes int64 `yaml:"max_payload_bytes"` +} + +// PayloadLimit is the configured limit or the default. +func (e Events) PayloadLimit() int64 { + if e.MaxPayloadBytes > 0 { + return e.MaxPayloadBytes + } + return DefaultMaxPayloadBytes } // Agent configures agent stages. @@ -91,6 +112,12 @@ func Load(path string, required bool) (Config, error) { // Validate checks the settings. func (c Config) Validate() error { + if c.Events.SecretFile != "" && !filepath.IsAbs(c.Events.SecretFile) { + return errors.New("config: events.secret_file must be an absolute path") + } + if c.Events.MaxPayloadBytes < 0 { + return errors.New("config: events.max_payload_bytes must not be negative") + } a := c.Agent switch a.Executor { case ExecutorNone: diff --git a/internal/engine/advance.go b/internal/engine/advance.go index 4b70021..998122c 100644 --- a/internal/engine/advance.go +++ b/internal/engine/advance.go @@ -259,7 +259,18 @@ func (e *Engine) start(ctx context.Context, run store.Run, sp spec.Spec, stg spe var env []string if stg.Kind() == spec.KindShell { var err error - if env, err = e.stageEnv(run, sp, stg, st.Attempts, workDir, resultPath); err != nil { + var ev *store.Event + if run.EventID != 0 { + got, err := e.Store.GetEvent(ctx, run.EventID) + if err != nil { + return fail("load event: " + err.Error()) + } + if got.PayloadDeletedAt != nil { + return fail("the event's payload was deleted; this run cannot continue") + } + ev = &got + } + if env, err = e.stageEnv(run, ev, sp, stg, st.Attempts, workDir, resultPath); err != nil { return fail(err.Error()) } } @@ -355,7 +366,7 @@ func firstLine(s string) string { // stageEnv builds a shell stage's complete environment: BaseEnv, the // STAGEHAND_* variables, then each env_keys entry from env_file (preferred) or // the spec's env defaults. A listed key found in neither is an error. -func (e *Engine) stageEnv(run store.Run, sp spec.Spec, stg spec.Stage, attempt int, workDir, resultPath string) ([]string, error) { +func (e *Engine) stageEnv(run store.Run, ev *store.Event, sp spec.Spec, stg spec.Stage, attempt int, workDir, resultPath string) ([]string, error) { vars := map[string]string{} var order []string set := func(k, v string) { @@ -375,6 +386,14 @@ func (e *Engine) stageEnv(run store.Run, sp spec.Spec, stg spec.Stage, attempt i set("STAGEHAND_ATTEMPT", fmt.Sprint(attempt)) set("STAGEHAND_WORK", workDir) set("STAGEHAND_RESULT", resultPath) + set("STAGEHAND_TRIGGER", run.Trigger) + if ev != nil { + // The payload is passed only as a path: stagehand never puts event + // content into a command line or environment value. + set("STAGEHAND_EVENT_ID", ev.Key) + set("STAGEHAND_EVENT_FILE", ev.PayloadPath) + set("STAGEHAND_EVENT_CONTENT_TYPE", ev.ContentType) + } if len(stg.EnvKeys) > 0 { var secrets map[string]string if sp.EnvFile != "" { diff --git a/internal/engine/engine.go b/internal/engine/engine.go index a953204..51d081b 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -146,7 +146,9 @@ func (e *Engine) AddPipeline(ctx context.Context, raw []byte, sourceDir string, return p, e.Store.UpdatePipeline(ctx, p) } -// SetEnabled turns a pipeline's schedule on or off. Enabling a schedule +// SetEnabled turns a pipeline's schedule and event intake on or off; a +// disabled pipeline refuses new events and leaves queued ones waiting until +// it is enabled again. Enabling a schedule // without `at` makes it due at once; with `at` it is due at the next slot. func (e *Engine) SetEnabled(ctx context.Context, name string, enabled bool) (store.Pipeline, error) { p, err := e.Store.GetPipeline(ctx, name) @@ -349,7 +351,10 @@ type LoopOptions struct { Schedule bool // Only, when non-nil, limits execution to these runs. Only map[int64]bool - // UntilIdle returns once no selected run is running. + // Events starts runs for queued events (implied by Schedule). + Events bool + // UntilIdle returns once no selected run is running (and, with Events, + // no enabled pipeline has a queued event left). UntilIdle bool } @@ -369,14 +374,28 @@ func (e *Engine) Loop(ctx context.Context, opts LoopOptions) error { e.Log.Error("schedule", "err", err) } } + if opts.Schedule || opts.Events { + if err := e.startEvents(ctx); err != nil && ctx.Err() == nil { + e.Log.Error("start events", "err", err) + } + } busy, err := e.advanceAll(ctx, opts.Only) if err != nil && ctx.Err() == nil { e.Log.Error("advance", "err", err) } + if !busy && (opts.Schedule || opts.Events) && opts.UntilIdle { + // A run that just settled may leave events waiting behind it. + if waiting, qerr := e.eventsWaiting(ctx); qerr == nil && waiting { + busy = true + } + } if time.Since(lastSweep) >= CleanupInterval || (opts.UntilIdle && !busy) { if serr := e.SweepCleanups(ctx); serr != nil && ctx.Err() == nil { e.Log.Error("cleanup sweep", "err", serr) } + if rerr := e.applyRetention(ctx); rerr != nil && ctx.Err() == nil { + e.Log.Error("retention", "err", rerr) + } lastSweep = time.Now() } if opts.UntilIdle && !busy && err == nil { diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index e548ba5..45329e4 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -480,7 +480,7 @@ func TestShellStageEnvIsExact(t *testing.T) { t.Fatal(err) } e := &Engine{BaseEnv: []string{"PATH=/bin"}} - env, err := e.stageEnv(store.Run{ID: 7, Pipeline: "p"}, sp, sp.Stages[0], 1, "/w", "/r.json") + env, err := e.stageEnv(store.Run{ID: 7, Pipeline: "p"}, nil, sp, sp.Stages[0], 1, "/w", "/r.json") if err != nil { t.Fatal(err) } @@ -499,7 +499,7 @@ func TestShellStageEnvIsExact(t *testing.T) { } sp.Stages[0].EnvKeys = []string{"MISSING"} - if _, err := e.stageEnv(store.Run{}, sp, sp.Stages[0], 1, "/w", "/r"); err == nil { + if _, err := e.stageEnv(store.Run{}, nil, sp, sp.Stages[0], 1, "/w", "/r"); err == nil { t.Error("missing env key did not fail") } } diff --git a/internal/engine/events.go b/internal/engine/events.go new file mode 100644 index 0000000..22244fe --- /dev/null +++ b/internal/engine/events.go @@ -0,0 +1,322 @@ +package engine + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "mime" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/gitmoot/stagehand/internal/spec" + "github.com/gitmoot/stagehand/internal/store" +) + +// Event sources. +const ( + SourceHTTP = "http" + SourceCLI = "cli" +) + +var ( + // ErrEventsDisabled: the pipeline has no `events:` block. + ErrEventsDisabled = errors.New("pipeline does not accept events (add `events: {max_queued: N}` to its spec)") + // ErrPipelineDisabled: the pipeline is disabled. + ErrPipelineDisabled = errors.New("pipeline is disabled") + // ErrPayloadTooLarge: the payload exceeds the host limit. + ErrPayloadTooLarge = errors.New("payload too large") + // ErrBadEventKey: the caller's event id has an invalid form. + ErrBadEventKey = errors.New("event id must be 1-128 characters from A-Z a-z 0-9 . _ : -") +) + +var eventKeyRE = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +// ValidEventKey reports whether key may be used as an event id. +func ValidEventKey(key string) bool { return eventKeyRE.MatchString(key) } + +// EventRequest is one trigger. +type EventRequest struct { + Pipeline string + // Key is the caller's idempotency key; empty means "always new". + Key string + ContentType string + Source string + Payload io.Reader + // MaxBytes bounds the payload. + MaxBytes int64 +} + +// EventResult reports what Trigger did. +type EventResult struct { + Event store.Event + Duplicate bool +} + +func (e *Engine) payloadDir() string { return filepath.Join(e.Home, "events") } + +// Trigger accepts an event: it writes the payload to a private file, then +// records the event, unless one with the same key exists (Duplicate) or the +// pipeline's queue is full (store.ErrQueueFull). Nothing is kept on refusal. +func (e *Engine) Trigger(ctx context.Context, req EventRequest) (EventResult, error) { + p, err := e.Store.GetPipeline(ctx, req.Pipeline) + if err != nil { + return EventResult{}, err + } + sp, err := spec.Load(p.SpecYAML) + if err != nil { + return EventResult{}, fmt.Errorf("stored spec for %q: %w", req.Pipeline, err) + } + if sp.Events == nil { + return EventResult{}, ErrEventsDisabled + } + if !p.Enabled { + return EventResult{}, fmt.Errorf("%q: %w", req.Pipeline, ErrPipelineDisabled) + } + key := req.Key + if key == "" { + key = "auto-" + randomHex(12) + } else if !ValidEventKey(key) { + return EventResult{}, ErrBadEventKey + } + // A duplicate is answered before reading the body, so a retried webhook + // costs nothing. + if existing, err := e.Store.FindEvent(ctx, req.Pipeline, key); err == nil { + return EventResult{Event: existing, Duplicate: true}, nil + } else if !errors.Is(err, store.ErrNotFound) { + return EventResult{}, err + } + + if err := os.MkdirAll(e.payloadDir(), 0o700); err != nil { + return EventResult{}, err + } + tmp, err := os.CreateTemp(e.payloadDir(), ".incoming-*") + if err != nil { + return EventResult{}, err + } + defer os.Remove(tmp.Name()) // no-op once renamed + h := sha256.New() + limit := req.MaxBytes + if limit <= 0 { + limit = 64 << 20 + } + n, err := io.Copy(io.MultiWriter(tmp, h), io.LimitReader(req.Payload, limit+1)) + if cerr := tmp.Close(); err == nil { + err = cerr + } + if err != nil { + return EventResult{}, fmt.Errorf("read payload: %w", err) + } + if n > limit { + return EventResult{}, fmt.Errorf("%w: more than %d bytes", ErrPayloadTooLarge, limit) + } + + final := filepath.Join(e.payloadDir(), randomHex(16)+".payload") + ev := store.Event{ + Pipeline: req.Pipeline, Key: key, Source: req.Source, ReceivedAt: e.now(), + ContentType: sanitizeContentType(req.ContentType), PayloadPath: final, + PayloadBytes: n, PayloadSHA256: hex.EncodeToString(h.Sum(nil)), + } + stored, dup, err := e.Store.InsertEvent(ctx, ev, sp.Events.MaxQueued, func() error { + return os.Rename(tmp.Name(), final) + }) + if err != nil { + os.Remove(final) + return EventResult{}, err + } + if dup { + // Lost a race with an identical event: keep the first one. + return EventResult{Event: stored, Duplicate: true}, nil + } + e.Log.Info("event accepted", "pipeline", req.Pipeline, "event", stored.ID, "key", key, "bytes", n, "source", req.Source) + e.signal() + return EventResult{Event: stored}, nil +} + +// startEvents starts the oldest queued event of every enabled pipeline that +// has no running run. +func (e *Engine) startEvents(ctx context.Context) error { + pipelines, err := e.Store.ListPipelines(ctx) + if err != nil { + return err + } + var errs []error + for _, p := range pipelines { + if !p.Enabled { + continue + } + ev, err := e.Store.OldestQueuedEvent(ctx, p.Name) + if err != nil || ev == nil { + if err != nil { + errs = append(errs, err) + } + continue + } + if active, err := e.Store.HasActiveRun(ctx, p.Name); err != nil || active { + continue + } + sp, err := spec.Load(p.SpecYAML) + if err != nil { + continue + } + ids := make([]string, len(sp.Stages)) + for i, st := range sp.Stages { + ids[i] = st.ID + } + id, err := e.Store.CreateEventRun(ctx, store.Run{ + Pipeline: p.Name, Trigger: "event", SpecYAML: p.SpecYAML, SpecHash: p.SpecHash, + SourceDir: p.SourceDir, CreatedAt: e.now(), + }, ids, ev.ID) + switch { + case errors.Is(err, store.ErrActiveRun), errors.Is(err, store.ErrNotFound): + continue // lost a race; the next pass retries + case err != nil: + errs = append(errs, err) + continue + } + e.Log.Info("run created", "pipeline", p.Name, "run", id, "trigger", "event", "event", ev.ID, "key", ev.Key) + } + return errors.Join(errs...) +} + +// applyRetention deletes the run directory and payload of every succeeded +// event run (owner decision, 6 Nov: confidential payloads do not outlive a +// successful run), and sweeps payload files no event refers to. +func (e *Engine) applyRetention(ctx context.Context) error { + runs, err := e.Store.RetentionDue(ctx) + if err != nil { + return err + } + var errs []error + for _, r := range runs { + if err := e.deleteRunFiles(ctx, r); err != nil { + errs = append(errs, err) + } + } + if err := e.sweepOrphanPayloads(ctx); err != nil { + errs = append(errs, err) + } + return errors.Join(errs...) +} + +func (e *Engine) deleteRunFiles(ctx context.Context, r store.Run) error { + if err := os.RemoveAll(e.RunDir(r.ID)); err != nil { + return fmt.Errorf("run %d: delete files: %w", r.ID, err) + } + if r.EventID != 0 { + ev, err := e.Store.GetEvent(ctx, r.EventID) + if err != nil { + return err + } + if err := os.Remove(ev.PayloadPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("run %d: delete payload: %w", r.ID, err) + } + } + if err := e.Store.MarkFilesDeleted(ctx, r.ID, r.EventID, e.now()); err != nil { + return err + } + e.Log.Info("deleted run files and payload", "run", r.ID, "event", r.EventID) + return nil +} + +// sweepOrphanPayloads removes payload files that no live event references: +// leftovers of an intake interrupted between write and commit. +func (e *Engine) sweepOrphanPayloads(ctx context.Context) error { + entries, err := os.ReadDir(e.payloadDir()) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + live, err := e.Store.LivePayloadPaths(ctx) + if err != nil { + return err + } + for _, ent := range entries { + path := filepath.Join(e.payloadDir(), ent.Name()) + if strings.HasPrefix(ent.Name(), ".incoming-") { + // An intake in progress in another process owns it; only + // remove ones old enough to be abandoned. + if info, err := ent.Info(); err == nil && e.now().Sub(info.ModTime()) < orphanAge { + continue + } + } else if live[path] { + continue + } + if err := os.Remove(path); err == nil { + e.Log.Warn("removed orphaned payload", "file", path) + } + } + return nil +} + +// orphanAge is how old an unfinished intake file must be before the sweep +// removes it. +const orphanAge = 15 * time.Minute + +// Purge deletes the files and payload of a finished event run that will not +// be resumed. It refuses a running run. +func (e *Engine) Purge(ctx context.Context, runID int64) error { + r, err := e.Store.GetRun(ctx, runID) + if err != nil { + return err + } + if r.State == store.RunRunning { + return fmt.Errorf("run %d is running; cancel it first", runID) + } + if r.FilesDeletedAt != nil { + return nil + } + return e.deleteRunFiles(ctx, r) +} + +// DropEvent discards a queued event and its payload. +func (e *Engine) DropEvent(ctx context.Context, id int64) (store.Event, error) { + return e.Store.DropEvent(ctx, id, e.now()) +} + +// sanitizeContentType keeps a well-formed media type (without parameters) +// and drops anything else. +func sanitizeContentType(ct string) string { + mt, _, err := mime.ParseMediaType(ct) + if err != nil || len(mt) > 100 { + return "" + } + return mt +} + +func randomHex(n int) string { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + panic(err) + } + return hex.EncodeToString(b) +} + +// eventsWaiting reports whether any enabled pipeline has a queued event. +func (e *Engine) eventsWaiting(ctx context.Context) (bool, error) { + pipelines, err := e.Store.ListPipelines(ctx) + if err != nil { + return false, err + } + for _, p := range pipelines { + if !p.Enabled { + continue + } + n, err := e.Store.QueuedEventCount(ctx, p.Name) + if err != nil { + return false, err + } + if n > 0 { + return true, nil + } + } + return false, nil +} diff --git a/internal/engine/events_test.go b/internal/engine/events_test.go new file mode 100644 index 0000000..9724119 --- /dev/null +++ b/internal/engine/events_test.go @@ -0,0 +1,281 @@ +package engine + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/gitmoot/stagehand/internal/shellexec" + "github.com/gitmoot/stagehand/internal/spec" + "github.com/gitmoot/stagehand/internal/stage" + "github.com/gitmoot/stagehand/internal/store" +) + +const eventPipeline = "name: ev\nevents: {max_queued: 3}\nstages:\n - {id: a, cmd: x}\n" + +func (h *harness) trigger(key, body string) (EventResult, error) { + h.t.Helper() + return h.eng.Trigger(context.Background(), EventRequest{Pipeline: "ev", Key: key, Source: SourceCLI, Payload: strings.NewReader(body), MaxBytes: 1 << 20}) +} + +func (h *harness) drain() { + h.t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := h.eng.Loop(ctx, LoopOptions{Events: true, UntilIdle: true}); err != nil { + h.t.Fatal(err) + } +} + +func TestSameEventIDYieldsOneRun(t *testing.T) { + h := newHarness(t, nil) + h.add(eventPipeline, true) + first, err := h.trigger("deck-42", "one") + if err != nil || first.Duplicate { + t.Fatalf("first: %+v %v", first, err) + } + second, err := h.trigger("deck-42", "retried body") + if err != nil || !second.Duplicate || second.Event.ID != first.Event.ID { + t.Fatalf("retry: %+v %v; want the first event back as a duplicate", second, err) + } + h.drain() + // A retry after the run finished is still a duplicate. + if third, err := h.trigger("deck-42", "late"); err != nil || !third.Duplicate { + t.Fatalf("late retry: %+v %v", third, err) + } + runs, _ := h.eng.Runs(context.Background(), "ev", 10) + if len(runs) != 1 || h.ex.callCount("a") != 1 { + t.Fatalf("%d runs, %d executions; want exactly one", len(runs), h.ex.callCount("a")) + } +} + +// A burst: events beyond max_queued are refused and nothing is stored for +// them; every accepted event runs exactly once, one run at a time. +func TestBurstIsBoundedAndEveryAcceptedEventRunsOnce(t *testing.T) { + h := newHarness(t, nil) + h.add(eventPipeline, true) + var accepted, refused int + for i := range 10 { + _, err := h.trigger("", strings.Repeat("x", i)) + switch { + case err == nil: + accepted++ + case errors.Is(err, store.ErrQueueFull): + refused++ + default: + t.Fatal(err) + } + } + if accepted != 3 || refused != 7 { + t.Fatalf("accepted %d, refused %d; want 3 and 7", accepted, refused) + } + files, _ := os.ReadDir(h.eng.payloadDir()) + if len(files) != 3 { + t.Errorf("%d payload files; refused events must leave nothing", len(files)) + } + h.drain() + runs, _ := h.eng.Runs(context.Background(), "ev", 10) + if len(runs) != 3 || h.ex.callCount("a") != 3 { + t.Fatalf("%d runs, %d executions; want 3", len(runs), h.ex.callCount("a")) + } + for _, r := range runs { + if r.State != store.RunSucceeded || r.Trigger != "event" || r.Event == nil { + t.Errorf("run %+v", r) + } + } + // Queue has room again. + if _, err := h.trigger("", "after"); err != nil { + t.Errorf("after draining: %v", err) + } +} + +func TestEventRunsNeverOverlapWithinAPipeline(t *testing.T) { + h := newHarness(t, nil) + lim := &limitedExec{} + h.eng.Executors[spec.KindShell] = &overlapExec{inner: lim} + h.add(eventPipeline, true) + for range 3 { + if _, err := h.trigger("", "x"); err != nil { + t.Fatal(err) + } + } + h.drain() + if lim.overlap || lim.ran != 3 { + t.Fatalf("overlap=%v ran=%d", lim.overlap, lim.ran) + } +} + +// overlapExec turns limitedExec's overlap check on for shell stages without +// its concurrency limit, to prove the one-run-per-pipeline rule alone keeps +// event runs sequential. +type overlapExec struct{ inner *limitedExec } + +func (o *overlapExec) Execute(ctx context.Context, job stage.Job) stage.Result { + return o.inner.Execute(ctx, job) +} + +// The payload reaches a real shell stage only as a file: shell +// metacharacters in it are delivered byte for byte and run nothing. +func TestPayloadWithShellMetacharactersIsDataNotCode(t *testing.T) { + h := newHarness(t, nil) + h.eng.Executors[spec.KindShell] = shellexec.Executor{} + h.eng.BaseEnv = []string{"PATH=/usr/bin:/bin"} + canary := filepath.Join(t.TempDir(), "pwned") + payload := "Subject: $(touch " + canary + ") `touch " + canary + "` ; touch " + canary + " | && \n\x00'\"\\" + h.add("name: ev\nevents: {max_queued: 3}\nstages:\n - {id: copy, cmd: 'cp \"$STAGEHAND_EVENT_FILE\" \"$STAGEHAND_WORK/got\"; echo \"$STAGEHAND_TRIGGER $STAGEHAND_EVENT_ID\" > \"$STAGEHAND_WORK/meta\"'}\n - {id: after, needs: [copy], cmd: 'false'}\n", true) + if _, err := h.trigger("mail-1", payload); err != nil { + t.Fatal(err) + } + h.drain() + if _, err := os.Stat(canary); err == nil { + t.Fatal("payload content was executed") + } + runs, _ := h.eng.Runs(context.Background(), "ev", 1) + got, err := os.ReadFile(filepath.Join(h.eng.RunDir(runs[0].ID), "work", "got")) + if err != nil || !bytes.Equal(got, []byte(payload)) { + t.Fatalf("payload not delivered byte-exact: %q, %v", got, err) + } + meta, _ := os.ReadFile(filepath.Join(h.eng.RunDir(runs[0].ID), "work", "meta")) + if strings.TrimSpace(string(meta)) != "event mail-1" { + t.Errorf("meta %q", meta) + } +} + +func TestRetentionDeletesOnSuccessAndKeepsParkedRuns(t *testing.T) { + h := newHarness(t, map[string][]string{"a": {"ok", "fail", "ok"}}) + h.add(eventPipeline, true) + ok, _ := h.trigger("ok", "secret deck") + bad, _ := h.trigger("bad", "another secret") + h.drain() + ctx := context.Background() + + okEv, _ := h.eng.Store.GetEvent(ctx, ok.Event.ID) + okRun, _ := h.eng.Store.GetRun(ctx, okEv.RunID) + if okRun.State != store.RunSucceeded || okRun.FilesDeletedAt == nil || okEv.PayloadDeletedAt == nil { + t.Fatalf("succeeded run: state %s, files deleted %v, payload deleted %v", okRun.State, okRun.FilesDeletedAt, okEv.PayloadDeletedAt) + } + if _, err := os.Stat(okEv.PayloadPath); !os.IsNotExist(err) { + t.Error("payload of a succeeded run still on disk") + } + if _, err := os.Stat(h.eng.RunDir(okRun.ID)); !os.IsNotExist(err) { + t.Error("run directory of a succeeded run still on disk") + } + + badEv, _ := h.eng.Store.GetEvent(ctx, bad.Event.ID) + badRun, _ := h.eng.Store.GetRun(ctx, badEv.RunID) + if badRun.State != store.RunFailed || badRun.FilesDeletedAt != nil { + t.Fatalf("failed run: %s, files deleted %v", badRun.State, badRun.FilesDeletedAt) + } + if _, err := os.Stat(badEv.PayloadPath); err != nil { + t.Fatal("payload of a parked run was deleted; it must survive for resume") + } + // Resume works and, once it succeeds, the payload goes. + if err := h.eng.Resume(ctx, badRun.ID, ""); err != nil { + t.Fatal(err) + } + h.runToEnd(badRun.ID) + if err := h.eng.applyRetention(ctx); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(badEv.PayloadPath); !os.IsNotExist(err) { + t.Error("payload kept after the resumed run succeeded") + } +} + +func TestPurgedRunCannotResume(t *testing.T) { + h := newHarness(t, map[string][]string{"a": {"fail"}}) + h.add(eventPipeline, true) + res, _ := h.trigger("p", "secret") + h.drain() + ctx := context.Background() + ev, _ := h.eng.Store.GetEvent(ctx, res.Event.ID) + if err := h.eng.Purge(ctx, ev.RunID); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(ev.PayloadPath); !os.IsNotExist(err) { + t.Error("purge left the payload") + } + if err := h.eng.Resume(ctx, ev.RunID, ""); err == nil { + t.Error("resumed a run whose files were deleted") + } +} + +func TestEventsRefusedWhenNotOptedInOrDisabledOrTooLarge(t *testing.T) { + h := newHarness(t, nil) + h.add("name: ev\nstages:\n - {id: a, cmd: x}\n", true) + if _, err := h.trigger("", "x"); !errors.Is(err, ErrEventsDisabled) { + t.Errorf("no events block: %v", err) + } + h.add(eventPipeline, false) // replacing keeps the pipeline enabled + h.eng.SetEnabled(context.Background(), "ev", false) + if _, err := h.trigger("", "x"); !errors.Is(err, ErrPipelineDisabled) { + t.Errorf("disabled pipeline: %v", err) + } + h.eng.SetEnabled(context.Background(), "ev", true) + _, err := h.eng.Trigger(context.Background(), EventRequest{Pipeline: "ev", Payload: strings.NewReader("12345"), MaxBytes: 4}) + if !errors.Is(err, ErrPayloadTooLarge) { + t.Errorf("oversized: %v", err) + } + if _, err := h.trigger("bad key with spaces", "x"); !errors.Is(err, ErrBadEventKey) { + t.Errorf("bad key: %v", err) + } + if files, _ := os.ReadDir(h.eng.payloadDir()); len(files) != 0 { + t.Errorf("refused events left %d files", len(files)) + } +} + +// Events and a schedule on one pipeline share the one-run-at-a-time rule. +func TestScheduleAndEventsCoexist(t *testing.T) { + h := newHarness(t, nil) + h.add("name: ev\nschedule: {interval: 1h}\nevents: {max_queued: 3}\nstages:\n - {id: a, cmd: x}\n", true) + if _, err := h.trigger("e1", "x"); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := h.eng.Loop(ctx, LoopOptions{Schedule: true, UntilIdle: true}); err != nil { + t.Fatal(err) + } + runs, _ := h.eng.Runs(context.Background(), "ev", 10) + triggers := map[string]int{} + for _, r := range runs { + triggers[r.Trigger]++ + } + if triggers["schedule"] != 1 || triggers["event"] != 1 { + t.Fatalf("runs by trigger %v; want one scheduled and one event run", triggers) + } +} + +func TestConcurrentDuplicateTriggersStoreOneEvent(t *testing.T) { + h := newHarness(t, nil) + h.add(eventPipeline, true) + var wg sync.WaitGroup + errs := make(chan error, 8) + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := h.trigger("same", "x"); err != nil { + errs <- err + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } + evs, _ := h.eng.Store.ListEvents(context.Background(), "ev", "", 50) + if len(evs) != 1 { + t.Fatalf("%d events stored for one key", len(evs)) + } + if files, _ := os.ReadDir(h.eng.payloadDir()); len(files) != 1 { + t.Errorf("%d payload files for one event", len(files)) + } +} diff --git a/internal/engine/status.go b/internal/engine/status.go index b3b38bd..1d7d42e 100644 --- a/internal/engine/status.go +++ b/internal/engine/status.go @@ -25,9 +25,12 @@ type PipelineStatus struct { Schedule *spec.Schedule `json:"schedule,omitempty"` NextDueAt *time.Time `json:"next_due_at"` ActiveRunID *int64 `json:"active_run_id"` - LastRun *RunSummary `json:"last_run"` - SpecHash string `json:"spec_hash"` - Error string `json:"error,omitempty"` + // Events is set when the pipeline accepts events. + Events *spec.Events `json:"events,omitempty"` + QueuedEvents int `json:"queued_events"` + LastRun *RunSummary `json:"last_run"` + SpecHash string `json:"spec_hash"` + Error string `json:"error,omitempty"` } // RunSummary is a run without its stage detail. @@ -43,6 +46,34 @@ type RunSummary struct { Cost float64 `json:"cost"` InputTokens int64 `json:"input_tokens"` OutputTokens int64 `json:"output_tokens"` + // Event is set for event-triggered runs. + Event *EventRef `json:"event,omitempty"` + // FilesDeletedAt: the run's files and payload were deleted (retention). + FilesDeletedAt *time.Time `json:"files_deleted_at,omitempty"` +} + +// EventRef identifies the event behind a run. The payload itself is never +// included, only a reference to it. +type EventRef struct { + ID int64 `json:"id"` + Key string `json:"key"` + Source string `json:"source"` + ReceivedAt time.Time `json:"received_at"` + ContentType string `json:"content_type,omitempty"` + PayloadBytes int64 `json:"payload_bytes"` + PayloadSHA256 string `json:"payload_sha256"` + PayloadPath string `json:"payload_path,omitempty"` + DeletedAt *time.Time `json:"payload_deleted_at,omitempty"` +} + +// EventRefOf describes an event without its payload. +func EventRefOf(ev store.Event) *EventRef { + ref := &EventRef{ID: ev.ID, Key: ev.Key, Source: ev.Source, ReceivedAt: ev.ReceivedAt, ContentType: ev.ContentType, + PayloadBytes: ev.PayloadBytes, PayloadSHA256: ev.PayloadSHA256, DeletedAt: ev.PayloadDeletedAt} + if ev.PayloadDeletedAt == nil { + ref.PayloadPath = ev.PayloadPath + } + return ref } // RunDetail is a run with its stages and attempts. @@ -93,7 +124,7 @@ func (e *Engine) Status(ctx context.Context, version string) (Status, error) { if sp, err := spec.Load(p.SpecYAML); err != nil { ps.Error = err.Error() } else { - ps.Description, ps.Schedule = sp.Description, sp.Schedule + ps.Description, ps.Schedule, ps.Events = sp.Description, sp.Schedule, sp.Events } last, err := e.Store.LastRun(ctx, p.Name) if err != nil { @@ -106,6 +137,9 @@ func (e *Engine) Status(ctx context.Context, version string) (Status, error) { } ps.LastRun = &sum } + if ps.QueuedEvents, err = e.Store.QueuedEventCount(ctx, p.Name); err != nil { + return Status{}, err + } active, err := e.Store.ActiveRunID(ctx, p.Name) if err != nil { return Status{}, err @@ -140,11 +174,19 @@ func (e *Engine) summarize(ctx context.Context, r store.Run) (RunSummary, error) if err != nil { return RunSummary{}, err } - return RunSummary{ + sum := RunSummary{ ID: r.ID, Pipeline: r.Pipeline, Trigger: r.Trigger, State: r.State, CreatedAt: r.CreatedAt, FinishedAt: r.FinishedAt, HaltStage: r.HaltStage, HaltReason: r.HaltReason, - Cost: cost, InputTokens: in, OutputTokens: out, - }, nil + Cost: cost, InputTokens: in, OutputTokens: out, FilesDeletedAt: r.FilesDeletedAt, + } + if r.EventID != 0 { + ev, err := e.Store.GetEvent(ctx, r.EventID) + if err != nil { + return RunSummary{}, err + } + sum.Event = EventRefOf(ev) + } + return sum, nil } // Run returns one run with its stages and attempts. diff --git a/internal/spec/spec.go b/internal/spec/spec.go index 5812fe4..aa2d035 100644 --- a/internal/spec/spec.go +++ b/internal/spec/spec.go @@ -66,6 +66,7 @@ type Spec struct { Name string `yaml:"name"` Description string `yaml:"description,omitempty"` Schedule *Schedule `yaml:"schedule,omitempty"` + Events *Events `yaml:"events,omitempty"` EnvFile string `yaml:"env_file,omitempty"` Env map[string]string `yaml:"env,omitempty"` Stages []Stage `yaml:"stages"` @@ -81,6 +82,18 @@ type Schedule struct { At string `yaml:"at,omitempty" json:"at,omitempty"` } +// Events opts a pipeline in to being started by external events (HTTP +// trigger or `stagehand trigger`). Each accepted event queues one run; runs +// start one at a time in arrival order. +type Events struct { + // MaxQueued bounds how many events may wait. Beyond it a trigger is + // refused (nothing stored), so a burst cannot pile up unbounded work. + MaxQueued int `yaml:"max_queued" json:"max_queued"` +} + +// MaxEventQueue is the largest allowed events.max_queued. +const MaxEventQueue = 10000 + // Stage is one node of the DAG. Exactly one of Cmd (shell) or Agent is set. type Stage struct { ID string `yaml:"id"` @@ -182,6 +195,9 @@ func (s Spec) Validate() error { add("schedule: %v", err) } } + if s.Events != nil && (s.Events.MaxQueued < 1 || s.Events.MaxQueued > MaxEventQueue) { + add("events.max_queued %d must be between 1 and %d", s.Events.MaxQueued, MaxEventQueue) + } if len(s.Stages) == 0 { add("stages: at least one stage is required") } diff --git a/internal/store/events.go b/internal/store/events.go new file mode 100644 index 0000000..ea13a70 --- /dev/null +++ b/internal/store/events.go @@ -0,0 +1,291 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "strings" + "time" +) + +// Event states. +const ( + // EventQueued is waiting for its pipeline to be free. + EventQueued = "queued" + // EventStarted has a run (see RunID). + EventStarted = "started" + // EventDropped was removed by an operator before it ran. + EventDropped = "dropped" +) + +var ( + // ErrQueueFull is returned when a pipeline already has its maximum of + // queued events; nothing was stored. + ErrQueueFull = errors.New("event queue is full") +) + +// Event is one accepted trigger of a pipeline. +type Event struct { + ID int64 + Pipeline string + // Key is the caller's idempotency key (or a generated one). One key per + // pipeline yields one event. + Key string + State string + Source string + ReceivedAt time.Time + ContentType string + PayloadPath string + PayloadBytes int64 + PayloadSHA256 string + PayloadDeletedAt *time.Time + RunID int64 +} + +const eventCols = `id, pipeline, event_key, state, source, received_at, content_type, payload_path, payload_bytes, payload_sha256, payload_deleted_at, COALESCE(run_id, 0)` + +func scanEvent(row interface{ Scan(...any) error }) (Event, error) { + var e Event + var received string + var deleted sql.NullString + if err := row.Scan(&e.ID, &e.Pipeline, &e.Key, &e.State, &e.Source, &received, &e.ContentType, + &e.PayloadPath, &e.PayloadBytes, &e.PayloadSHA256, &deleted, &e.RunID); err != nil { + return Event{}, err + } + var err error + if e.ReceivedAt, err = parseTime(received); err != nil { + return Event{}, err + } + if e.PayloadDeletedAt, err = parseTimePtr(deleted); err != nil { + return Event{}, err + } + return e, nil +} + +// InsertEvent stores an event unless one with the same pipeline and key +// exists (duplicate=true, the existing event is returned) or the pipeline +// already has maxQueued queued events (ErrQueueFull). place is called inside +// the transaction, after the checks and before commit, to move the payload +// file into place; if it fails nothing is stored. +func (s *Store) InsertEvent(ctx context.Context, e Event, maxQueued int, place func() error) (Event, bool, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return Event{}, false, err + } + defer tx.Rollback() + existing, err := scanEvent(tx.QueryRowContext(ctx, `SELECT `+eventCols+` FROM events WHERE pipeline=? AND event_key=?`, e.Pipeline, e.Key)) + switch { + case err == nil: + return existing, true, nil + case !errors.Is(err, sql.ErrNoRows): + return Event{}, false, err + } + var queued int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM events WHERE pipeline=? AND state=?`, e.Pipeline, EventQueued).Scan(&queued); err != nil { + return Event{}, false, err + } + if queued >= maxQueued { + return Event{}, false, fmt.Errorf("pipeline %q has %d queued events: %w", e.Pipeline, queued, ErrQueueFull) + } + res, err := tx.ExecContext(ctx, `INSERT INTO events (pipeline, event_key, state, source, received_at, content_type, payload_path, payload_bytes, payload_sha256) + VALUES (?,?,?,?,?,?,?,?,?)`, e.Pipeline, e.Key, EventQueued, e.Source, fmtTime(e.ReceivedAt), e.ContentType, e.PayloadPath, e.PayloadBytes, e.PayloadSHA256) + if err != nil { + return Event{}, false, err + } + if e.ID, err = res.LastInsertId(); err != nil { + return Event{}, false, err + } + if place != nil { + if err := place(); err != nil { + return Event{}, false, fmt.Errorf("store payload: %w", err) + } + } + e.State = EventQueued + return e, false, tx.Commit() +} + +// GetEvent returns one event. +func (s *Store) GetEvent(ctx context.Context, id int64) (Event, error) { + e, err := scanEvent(s.db.QueryRowContext(ctx, `SELECT `+eventCols+` FROM events WHERE id=?`, id)) + if errors.Is(err, sql.ErrNoRows) { + return Event{}, fmt.Errorf("event %d %w", id, ErrNotFound) + } + return e, err +} + +// FindEvent returns a pipeline's event by key. +func (s *Store) FindEvent(ctx context.Context, pipeline, key string) (Event, error) { + e, err := scanEvent(s.db.QueryRowContext(ctx, `SELECT `+eventCols+` FROM events WHERE pipeline=? AND event_key=?`, pipeline, key)) + if errors.Is(err, sql.ErrNoRows) { + return Event{}, fmt.Errorf("event %q of %q %w", key, pipeline, ErrNotFound) + } + return e, err +} + +// ListEvents returns events, newest first, optionally for one pipeline +// and/or one state. +func (s *Store) ListEvents(ctx context.Context, pipeline, state string, limit int) ([]Event, error) { + if limit <= 0 { + limit = 50 + } + q := `SELECT ` + eventCols + ` FROM events WHERE 1=1` + var args []any + if pipeline != "" { + q += ` AND pipeline=?` + args = append(args, pipeline) + } + if state != "" { + q += ` AND state=?` + args = append(args, state) + } + q += ` ORDER BY id DESC LIMIT ?` + args = append(args, limit) + rows, err := s.db.QueryContext(ctx, q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Event + for rows.Next() { + e, err := scanEvent(rows) + if err != nil { + return nil, err + } + out = append(out, e) + } + return out, rows.Err() +} + +// QueuedEventCount is how many events of a pipeline wait. +func (s *Store) QueuedEventCount(ctx context.Context, pipeline string) (int, error) { + var n int + err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM events WHERE pipeline=? AND state=?`, pipeline, EventQueued).Scan(&n) + return n, err +} + +// OldestQueuedEvent is the next event a pipeline should run, if any. +func (s *Store) OldestQueuedEvent(ctx context.Context, pipeline string) (*Event, error) { + e, err := scanEvent(s.db.QueryRowContext(ctx, `SELECT `+eventCols+` FROM events WHERE pipeline=? AND state=? ORDER BY id LIMIT 1`, pipeline, EventQueued)) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &e, nil +} + +// CreateEventRun creates a run for a queued event and marks the event +// started, atomically. It fails with ErrActiveRun if the pipeline has a +// running run, and with ErrNotFound if the event is no longer queued. +func (s *Store) CreateEventRun(ctx context.Context, r Run, stageIDs []string, eventID int64) (int64, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer tx.Rollback() + res, err := tx.ExecContext(ctx, `INSERT INTO runs (pipeline, trigger, spec_yaml, spec_hash, source_dir, state, created_at, event_id) VALUES (?,?,?,?,?,?,?,?)`, + r.Pipeline, r.Trigger, r.SpecYAML, r.SpecHash, r.SourceDir, RunRunning, fmtTime(r.CreatedAt), eventID) + if err != nil { + if strings.Contains(err.Error(), "UNIQUE constraint failed") { + return 0, fmt.Errorf("pipeline %q: %w", r.Pipeline, ErrActiveRun) + } + return 0, err + } + id, err := res.LastInsertId() + if err != nil { + return 0, err + } + for i, sid := range stageIDs { + if _, err := tx.ExecContext(ctx, `INSERT INTO run_stages (run_id, stage_id, position, state) VALUES (?,?,?,?)`, id, sid, i, StagePending); err != nil { + return 0, err + } + } + upd, err := tx.ExecContext(ctx, `UPDATE events SET state=?, run_id=? WHERE id=? AND state=?`, EventStarted, id, eventID, EventQueued) + if err := expectOne(upd, err, fmt.Sprintf("queued event %d", eventID)); err != nil { + return 0, err + } + return id, tx.Commit() +} + +// DropEvent removes a queued event's payload and marks it dropped. +func (s *Store) DropEvent(ctx context.Context, id int64, at time.Time) (Event, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return Event{}, err + } + defer tx.Rollback() + e, err := scanEvent(tx.QueryRowContext(ctx, `SELECT `+eventCols+` FROM events WHERE id=?`, id)) + if errors.Is(err, sql.ErrNoRows) { + return Event{}, fmt.Errorf("event %d %w", id, ErrNotFound) + } + if err != nil { + return Event{}, err + } + if e.State != EventQueued { + return Event{}, fmt.Errorf("event %d is %s; only a queued event can be dropped (use `purge ` for a started one)", id, e.State) + } + if err := removePayload(e.PayloadPath); err != nil { + return Event{}, err + } + if _, err := tx.ExecContext(ctx, `UPDATE events SET state=?, payload_deleted_at=? WHERE id=?`, EventDropped, fmtTime(at), id); err != nil { + return Event{}, err + } + e.State = EventDropped + return e, tx.Commit() +} + +// RetentionDue lists succeeded event runs whose files are not deleted yet. +func (s *Store) RetentionDue(ctx context.Context) ([]Run, error) { + return s.queryRuns(ctx, `WHERE event_id IS NOT NULL AND state=? AND files_deleted_at IS NULL ORDER BY id`, RunSucceeded) +} + +// MarkFilesDeleted records that a run's directory and its event's payload +// are gone. +func (s *Store) MarkFilesDeleted(ctx context.Context, runID, eventID int64, at time.Time) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `UPDATE runs SET files_deleted_at=? WHERE id=?`, fmtTime(at), runID); err != nil { + return err + } + if eventID != 0 { + if _, err := tx.ExecContext(ctx, `UPDATE events SET payload_deleted_at=COALESCE(payload_deleted_at, ?) WHERE id=?`, fmtTime(at), eventID); err != nil { + return err + } + } + return tx.Commit() +} + +// LivePayloadPaths returns every payload path not yet deleted, for the +// orphan sweep. +func (s *Store) LivePayloadPaths(ctx context.Context) (map[string]bool, error) { + rows, err := s.db.QueryContext(ctx, `SELECT payload_path FROM events WHERE payload_deleted_at IS NULL AND payload_path != ''`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]bool{} + for rows.Next() { + var p string + if err := rows.Scan(&p); err != nil { + return nil, err + } + out[p] = true + } + return out, rows.Err() +} + +func removePayload(path string) error { + if path == "" { + return nil + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} diff --git a/internal/store/store.go b/internal/store/store.go index 9ae2a36..4aa71f4 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -155,6 +155,25 @@ var migrations = []string{ tries INTEGER NOT NULL DEFAULT 0, last_error TEXT NOT NULL DEFAULT '' );`, + // 3: event triggers (#18). + `CREATE TABLE events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + pipeline TEXT NOT NULL, + event_key TEXT NOT NULL, + state TEXT NOT NULL, + source TEXT NOT NULL, + received_at TEXT NOT NULL, + content_type TEXT NOT NULL DEFAULT '', + payload_path TEXT NOT NULL DEFAULT '', + payload_bytes INTEGER NOT NULL DEFAULT 0, + payload_sha256 TEXT NOT NULL DEFAULT '', + payload_deleted_at TEXT, + run_id INTEGER, + UNIQUE (pipeline, event_key) + ); + CREATE INDEX events_queue ON events(pipeline, state, id); + ALTER TABLE runs ADD COLUMN event_id INTEGER; + ALTER TABLE runs ADD COLUMN files_deleted_at TEXT;`, } func (s *Store) migrate(ctx context.Context) error { @@ -314,6 +333,11 @@ type Run struct { CancelRequested bool CreatedAt time.Time FinishedAt *time.Time + // EventID is the events row that started this run (0 if none). + EventID int64 + // FilesDeletedAt is when the run's directory and payload were deleted + // (retention); such a run can no longer be resumed. + FilesDeletedAt *time.Time } // RunStage is the state of one stage within a run. @@ -353,14 +377,14 @@ type Attempt struct { FinishedAt *time.Time } -const runCols = `id, pipeline, trigger, spec_yaml, spec_hash, source_dir, state, halt_stage, halt_reason, cancel_requested, created_at, finished_at` +const runCols = `id, pipeline, trigger, spec_yaml, spec_hash, source_dir, state, halt_stage, halt_reason, cancel_requested, created_at, finished_at, COALESCE(event_id, 0), files_deleted_at` func scanRun(row interface{ Scan(...any) error }) (Run, error) { var r Run var created string - var finished sql.NullString + var finished, deleted sql.NullString if err := row.Scan(&r.ID, &r.Pipeline, &r.Trigger, &r.SpecYAML, &r.SpecHash, &r.SourceDir, &r.State, - &r.HaltStage, &r.HaltReason, &r.CancelRequested, &created, &finished); err != nil { + &r.HaltStage, &r.HaltReason, &r.CancelRequested, &created, &finished, &r.EventID, &deleted); err != nil { return Run{}, err } var err error @@ -370,6 +394,9 @@ func scanRun(row interface{ Scan(...any) error }) (Run, error) { if r.FinishedAt, err = parseTimePtr(finished); err != nil { return Run{}, err } + if r.FilesDeletedAt, err = parseTimePtr(deleted); err != nil { + return Run{}, err + } return r, nil } @@ -490,7 +517,8 @@ func (s *Store) ResumeRun(ctx context.Context, id int64, reset []string) error { } defer tx.Rollback() var pipeline, state string - if err := tx.QueryRowContext(ctx, `SELECT pipeline, state FROM runs WHERE id=?`, id).Scan(&pipeline, &state); err != nil { + var deleted sql.NullString + if err := tx.QueryRowContext(ctx, `SELECT pipeline, state, files_deleted_at FROM runs WHERE id=?`, id).Scan(&pipeline, &state, &deleted); err != nil { if errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("run %d %w", id, ErrNotFound) } @@ -501,6 +529,9 @@ func (s *Store) ResumeRun(ctx context.Context, id int64, reset []string) error { default: return fmt.Errorf("run %d is %s; only a blocked, failed or cancelled run can be resumed", id, state) } + if deleted.Valid { + return fmt.Errorf("run %d cannot be resumed: its files and payload were deleted at %s", id, deleted.String) + } _, err = tx.ExecContext(ctx, `UPDATE runs SET state=?, halt_stage='', halt_reason='', cancel_requested=0, finished_at=NULL WHERE id=?`, RunRunning, id) if err != nil { if strings.Contains(err.Error(), "UNIQUE constraint failed") { diff --git a/internal/trigger/trigger.go b/internal/trigger/trigger.go new file mode 100644 index 0000000..47cfed6 --- /dev/null +++ b/internal/trigger/trigger.go @@ -0,0 +1,176 @@ +// Package trigger is the HTTP front door for event-triggered runs: +// POST /v1/pipelines/{name}/events, authenticated with HMAC-SHA256. +// +// Signature: hex(HMAC-SHA256(secret, timestamp + "\n" + pipeline + "\n" + +// event_id + "\n" + body)), sent as `X-Stagehand-Signature: sha256=` +// with `X-Stagehand-Timestamp: ` (within MaxSkew of now) and +// an optional `X-Stagehand-Event-Id` for idempotency. +package trigger + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strconv" + "strings" + "time" + + "github.com/gitmoot/stagehand/internal/engine" + "github.com/gitmoot/stagehand/internal/store" +) + +// MaxSkew is how far a request's timestamp may be from now. +const MaxSkew = 5 * time.Minute + +// Headers. +const ( + HeaderTimestamp = "X-Stagehand-Timestamp" + HeaderSignature = "X-Stagehand-Signature" + HeaderEventID = "X-Stagehand-Event-Id" +) + +// Sign returns the signature header value for a request. +func Sign(secret []byte, timestamp int64, pipeline, eventID string, body []byte) string { + mac := hmac.New(sha256.New, secret) + fmt.Fprintf(mac, "%d\n%s\n%s\n", timestamp, pipeline, eventID) + mac.Write(body) + return "sha256=" + hex.EncodeToString(mac.Sum(nil)) +} + +// Accepter takes an authenticated event. +type Accepter interface { + Trigger(ctx context.Context, req engine.EventRequest) (engine.EventResult, error) +} + +// Handler serves the trigger endpoint. +type Handler struct { + Engine Accepter + // Secret returns the current HMAC key (re-read so it can be rotated). + Secret func() ([]byte, error) + MaxBytes int64 + Now func() time.Time + Log *slog.Logger +} + +// Response is the JSON answer. +type Response struct { + Accepted bool `json:"accepted"` + Duplicate bool `json:"duplicate,omitempty"` + Event *engine.EventRef `json:"event,omitempty"` + State string `json:"state,omitempty"` + RunID int64 `json:"run_id,omitempty"` + Error string `json:"error,omitempty"` +} + +func (h Handler) now() time.Time { + if h.Now != nil { + return h.Now() + } + return time.Now() +} + +// Routes registers the endpoint on mux. +func (h Handler) Routes(mux *http.ServeMux) { + mux.HandleFunc("POST /v1/pipelines/{name}/events", h.serve) +} + +func (h Handler) reply(w http.ResponseWriter, status int, r Response) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(r) +} + +func (h Handler) fail(w http.ResponseWriter, r *http.Request, status int, msg string) { + if h.Log != nil { + h.Log.Warn("trigger refused", "status", status, "pipeline", r.PathValue("name"), "remote", r.RemoteAddr, "reason", msg) + } + h.reply(w, status, Response{Error: msg}) +} + +func (h Handler) serve(w http.ResponseWriter, r *http.Request) { + pipeline := r.PathValue("name") + eventID := r.Header.Get(HeaderEventID) + if eventID != "" && !engine.ValidEventKey(eventID) { + h.fail(w, r, http.StatusBadRequest, engine.ErrBadEventKey.Error()) + return + } + ts, err := strconv.ParseInt(r.Header.Get(HeaderTimestamp), 10, 64) + if err != nil { + h.fail(w, r, http.StatusUnauthorized, "missing or malformed "+HeaderTimestamp) + return + } + if d := h.now().Sub(time.Unix(ts, 0)); d > MaxSkew || d < -MaxSkew { + h.fail(w, r, http.StatusUnauthorized, "timestamp outside the allowed window") + return + } + sig := r.Header.Get(HeaderSignature) + if !strings.HasPrefix(sig, "sha256=") { + h.fail(w, r, http.StatusUnauthorized, "missing or malformed "+HeaderSignature) + return + } + // The body must be read to verify it; it is bounded first. + limit := h.MaxBytes + body, err := io.ReadAll(io.LimitReader(r.Body, limit+1)) + if err != nil { + h.fail(w, r, http.StatusBadRequest, "read body: "+err.Error()) + return + } + if int64(len(body)) > limit { + h.fail(w, r, http.StatusRequestEntityTooLarge, fmt.Sprintf("payload larger than %d bytes", limit)) + return + } + secret, err := h.Secret() + if err != nil { + if h.Log != nil { + h.Log.Error("trigger secret unavailable", "err", err) + } + h.reply(w, http.StatusServiceUnavailable, Response{Error: "trigger secret unavailable"}) + return + } + want := Sign(secret, ts, pipeline, eventID, body) + if !hmac.Equal([]byte(sig), []byte(want)) { + h.fail(w, r, http.StatusUnauthorized, "bad signature") + return + } + + res, err := h.Engine.Trigger(r.Context(), engine.EventRequest{ + Pipeline: pipeline, Key: eventID, ContentType: r.Header.Get("Content-Type"), + Source: engine.SourceHTTP, Payload: bytes.NewReader(body), MaxBytes: limit, + }) + switch { + case errors.Is(err, store.ErrNotFound): + h.fail(w, r, http.StatusNotFound, "no such pipeline") + case errors.Is(err, engine.ErrEventsDisabled): + h.fail(w, r, http.StatusNotFound, err.Error()) + case errors.Is(err, engine.ErrPipelineDisabled): + h.fail(w, r, http.StatusConflict, err.Error()) + case errors.Is(err, store.ErrQueueFull): + w.Header().Set("Retry-After", "60") + h.fail(w, r, http.StatusTooManyRequests, err.Error()) + case errors.Is(err, engine.ErrPayloadTooLarge): + h.fail(w, r, http.StatusRequestEntityTooLarge, err.Error()) + case errors.Is(err, engine.ErrBadEventKey): + h.fail(w, r, http.StatusBadRequest, err.Error()) + case err != nil: + if h.Log != nil { + h.Log.Error("trigger", "pipeline", pipeline, "err", err) + } + h.reply(w, http.StatusInternalServerError, Response{Error: "internal error"}) + default: + resp := Response{Accepted: true, Duplicate: res.Duplicate, Event: engine.EventRefOf(res.Event), State: res.Event.State, RunID: res.Event.RunID} + resp.Event.PayloadPath = "" // never reveal host paths to a caller + status := http.StatusAccepted + if res.Duplicate { + status = http.StatusOK + } + h.reply(w, status, resp) + } +} diff --git a/internal/trigger/trigger_test.go b/internal/trigger/trigger_test.go new file mode 100644 index 0000000..cf5cf25 --- /dev/null +++ b/internal/trigger/trigger_test.go @@ -0,0 +1,140 @@ +package trigger + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/gitmoot/stagehand/internal/engine" + "github.com/gitmoot/stagehand/internal/spec" + "github.com/gitmoot/stagehand/internal/stage" + "github.com/gitmoot/stagehand/internal/store" +) + +var secret = []byte("test-secret-0123456789abcdef") + +func newServer(t *testing.T) (*httptest.Server, *engine.Engine) { + t.Helper() + home := t.TempDir() + st, err := store.Open(context.Background(), filepath.Join(home, "stagehand.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + eng := engine.New(st, home, map[spec.Kind]stage.Executor{}, nil, nil) + if _, err := eng.AddPipeline(context.Background(), []byte("name: ev\nevents: {max_queued: 2}\nstages:\n - {id: a, cmd: x}\n"), "/", false, true); err != nil { + t.Fatal(err) + } + mux := http.NewServeMux() + Handler{Engine: eng, Secret: func() ([]byte, error) { return secret, nil }, MaxBytes: 1024}.Routes(mux) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv, eng +} + +type call struct { + pipeline, eventID string + body []byte + ts int64 + sig string // "" = sign correctly + noSig, noTS bool +} + +func do(t *testing.T, srv *httptest.Server, c call) (int, Response) { + t.Helper() + if c.pipeline == "" { + c.pipeline = "ev" + } + if c.ts == 0 { + c.ts = time.Now().Unix() + } + req, _ := http.NewRequest("POST", srv.URL+"/v1/pipelines/"+c.pipeline+"/events", bytes.NewReader(c.body)) + if !c.noTS { + req.Header.Set(HeaderTimestamp, strconv.FormatInt(c.ts, 10)) + } + if c.eventID != "" { + req.Header.Set(HeaderEventID, c.eventID) + } + if !c.noSig { + sig := c.sig + if sig == "" { + sig = Sign(secret, c.ts, c.pipeline, c.eventID, c.body) + } + req.Header.Set(HeaderSignature, sig) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + var r Response + json.Unmarshal(raw, &r) + return resp.StatusCode, r +} + +func TestUnauthenticatedCallsAreRefusedAndStoreNothing(t *testing.T) { + srv, eng := newServer(t) + body := []byte(`{"from":"cmo@example.com"}`) + cases := map[string]call{ + "no signature": {body: body, noSig: true}, + "no timestamp": {body: body, noTS: true}, + "wrong secret": {body: body, sig: Sign([]byte("other"), time.Now().Unix(), "ev", "", body)}, + "signed other body": {body: body, sig: Sign(secret, time.Now().Unix(), "ev", "", []byte("x"))}, + "signed other pipe": {body: body, sig: Sign(secret, time.Now().Unix(), "other", "", body)}, + "stale timestamp": {body: body, ts: time.Now().Add(-10 * time.Minute).Unix()}, + "future timestamp": {body: body, ts: time.Now().Add(10 * time.Minute).Unix()}, + "signed other eventid": {body: body, eventID: "a", sig: Sign(secret, time.Now().Unix(), "ev", "b", body)}, + } + for name, c := range cases { + if code, _ := do(t, srv, c); code != http.StatusUnauthorized { + t.Errorf("%s: status %d, want 401", name, code) + } + } + if evs, _ := eng.Store.ListEvents(context.Background(), "", "", 50); len(evs) != 0 { + t.Fatalf("%d events stored from unauthenticated calls", len(evs)) + } +} + +func TestSignedCallQueuesAndDuplicateIsAnsweredOnce(t *testing.T) { + srv, eng := newServer(t) + code, r := do(t, srv, call{eventID: "msg-1", body: []byte("deck")}) + if code != http.StatusAccepted || !r.Accepted || r.Duplicate || r.Event.PayloadPath != "" { + t.Fatalf("first: %d %+v", code, r) + } + code, r2 := do(t, srv, call{eventID: "msg-1", body: []byte("deck")}) + if code != http.StatusOK || !r2.Duplicate || r2.Event.ID != r.Event.ID { + t.Fatalf("retry: %d %+v", code, r2) + } + if evs, _ := eng.Store.ListEvents(context.Background(), "ev", "", 50); len(evs) != 1 { + t.Fatalf("%d events", len(evs)) + } +} + +func TestQueueFullTooLargeAndUnknown(t *testing.T) { + srv, _ := newServer(t) + for i := range 2 { + if code, _ := do(t, srv, call{body: []byte{byte(i)}}); code != http.StatusAccepted { + t.Fatalf("event %d: %d", i, code) + } + } + if code, _ := do(t, srv, call{body: []byte("3")}); code != http.StatusTooManyRequests { + t.Errorf("queue full: %d, want 429", code) + } + if code, _ := do(t, srv, call{body: bytes.Repeat([]byte("x"), 2000)}); code != http.StatusRequestEntityTooLarge { + t.Errorf("oversized: %d, want 413", code) + } + if code, _ := do(t, srv, call{pipeline: "nope", body: []byte("x")}); code != http.StatusNotFound { + t.Errorf("unknown pipeline: %d, want 404", code) + } + if code, _ := do(t, srv, call{eventID: "has spaces", body: []byte("x")}); code != http.StatusBadRequest { + t.Errorf("bad event id: %d, want 400", code) + } +} From 392017ff0079ee34b365f7d93c5825b44bc8afe8 Mon Sep 17 00:00:00 2001 From: jerryfane Date: Fri, 6 Nov 2026 03:12:18 +0100 Subject: [PATCH 02/11] test(store): insert-level event idempotency (#18) --- internal/store/store_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 3eb9857..ed82c6b 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "testing" + "time" ) // A home created by an earlier binary must gain every later table when a @@ -38,3 +39,29 @@ func TestOpenUpgradesAnExistingVersion1Home(t *testing.T) { t.Fatalf("pending cleanups on an upgraded home: %v", err) } } + +// Two intakes of the same event can both pass the engine's early duplicate +// check (slow request bodies overlap). The insert itself must then answer +// "duplicate" for the second, without placing its payload. +func TestInsertEventIsIdempotentInsideTheTransaction(t *testing.T) { + s, err := Open(context.Background(), filepath.Join(t.TempDir(), "stagehand.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + ev := Event{Pipeline: "p", Key: "k", Source: "http", ReceivedAt: time.Now(), PayloadPath: "/a"} + placed := 0 + place := func() error { placed++; return nil } + first, dup, err := s.InsertEvent(context.Background(), ev, 5, place) + if err != nil || dup { + t.Fatalf("first: dup=%v err=%v", dup, err) + } + ev.PayloadPath = "/b" + second, dup, err := s.InsertEvent(context.Background(), ev, 5, place) + if err != nil || !dup || second.ID != first.ID || second.PayloadPath != "/a" { + t.Fatalf("second: %+v dup=%v err=%v; want the first event as a duplicate", second, dup, err) + } + if placed != 1 { + t.Errorf("payload placed %d times", placed) + } +} From 362aa25e4c2f44e502a0ea25b9c5dec77295d058 Mon Sep 17 00:00:00 2001 From: jerryfane Date: Fri, 6 Nov 2026 03:13:41 +0100 Subject: [PATCH 03/11] docs: event triggers (#18) --- README.md | 131 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 16c80d5..7121cab 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ stages: | `schedule.interval` | Go duration, at least `1m`. Without a schedule, the pipeline runs only on demand | | `schedule.jitter` | Random extra delay before each due time, from 0 up to (not including) `jitter`. Must be smaller than `interval` | | `schedule.at` | RFC 3339 time. Due times become `at + k*interval` | +| `events.max_queued` | Opt in to event triggers; at most this many events wait (see [Starting pipelines from events](#starting-pipelines-from-events)) | | `env_file` | Absolute path to a `KEY=VALUE` file of secrets. It must be private (mode `0600`) | | `env` | Non-secret default values | | `stages` | The stages | @@ -190,6 +191,7 @@ Unit files and an example config are in [`deploy/`](deploy/). - `STAGEHAND_PIPELINE`, `STAGEHAND_RUN_ID`, `STAGEHAND_STAGE`, `STAGEHAND_ATTEMPT`; - `STAGEHAND_WORK`, the run's shared work directory, where stages pass files to each other; - `STAGEHAND_RESULT`; + - `STAGEHAND_TRIGGER`, and for event runs `STAGEHAND_EVENT_ID`, `STAGEHAND_EVENT_FILE`, `STAGEHAND_EVENT_CONTENT_TYPE`; - the variables in its `env_keys`, and nothing else. - **A failed or blocked stage parks the run.** Stages that depend on it are skipped. Stages that don't depend on it still finish. The run ends as `failed` or `blocked`. @@ -214,6 +216,126 @@ Unit files and an example config are in [`deploy/`](deploy/). - **Stopping:** on `SIGTERM`, stagehand stops the running stages and starts nothing new. A run whose next stage was ready stays `running`, and the next runner continues it. +## Starting pipelines from events + +A pipeline can also be started by something that happens outside stagehand, for example a mail +poller seeing a new message or a webhook receiver getting a call. Each accepted event queues one +run, and its payload becomes that run's input. Receivers live outside stagehand; they just call the +trigger. + +```yaml +name: deck-review +events: + max_queued: 20 # opt in; at most 20 events wait, more are refused +schedule: {interval: 24h} # optional: schedule and events can share a pipeline +stages: + - id: intake + cmd: cp "$STAGEHAND_EVENT_FILE" "$STAGEHAND_WORK/mail.eml" + - id: score + needs: [intake] + agent: omp + inputs: [mail.eml] + # … +``` + +**How events run** +- Runs start in arrival order. A pipeline still has at most one running run at a time, and agent + stages still share the sandbox one at a time. +- When `max_queued` events are already waiting, a new trigger is refused and nothing is stored, so + the caller knows to retry. Accepted events are never lost or run twice. +- A disabled pipeline refuses events; its queued events wait until it is enabled again. + +**What a stage sees.** The payload is passed **only as a file**. stagehand never puts event content +into a command line or an environment value. Shell stages get: + +| Variable | Value | +|---|---| +| `STAGEHAND_TRIGGER` | `event`, `schedule` or `manual` | +| `STAGEHAND_EVENT_ID` | the event id (the caller's, or a generated one) | +| `STAGEHAND_EVENT_FILE` | path to the payload, exactly the bytes received | +| `STAGEHAND_EVENT_CONTENT_TYPE` | the sender's media type, if it was well formed | + +Agent stages see the payload only if a shell stage copies it into `$STAGEHAND_WORK` and lists it in +`inputs`. + +**Idempotency.** Send an event id. The same id for the same pipeline always gives one event and one +run, so a webhook that retries cannot start duplicates. Without an id, every trigger is a new +event. Ids are 1–128 characters from `A-Z a-z 0-9 . _ : -`. + +### HTTP trigger + +`stagehand serve --trigger-addr 127.0.0.1:8798` serves `POST /v1/pipelines/{name}/events`. + +- **Address:** loopback only, unless `--trigger-allow-remote` is also given. Put a TLS proxy in + front of it before exposing it. +- **Body:** the raw payload. +- **Authentication:** every request must be signed with HMAC-SHA256, using the key in the host + config's `events.secret_file` (a 0600 file): + +``` +X-Stagehand-Timestamp: +X-Stagehand-Event-Id: +X-Stagehand-Signature: sha256= +``` + +```sh +KEY=$(cat /etc/stagehand/trigger.key); TS=$(date +%s); ID=msg-123 +SIG=$( { printf '%s\n%s\n%s\n' "$TS" deck-review "$ID"; cat mail.eml; } | openssl dgst -sha256 -hmac "$KEY" -r | cut -d' ' -f1) +curl -sS -X POST http://127.0.0.1:8798/v1/pipelines/deck-review/events \ + -H "X-Stagehand-Timestamp: $TS" -H "X-Stagehand-Event-Id: $ID" -H "X-Stagehand-Signature: sha256=$SIG" \ + -H 'Content-Type: message/rfc822' --data-binary @mail.eml +``` + +| Status | Meaning | +|---|---| +| 202 | Accepted and queued (JSON gives the event id, size and SHA-256) | +| 200 | Duplicate: this event id was already received; the original event and its run are returned | +| 401 | Missing, stale or wrong signature | +| 404 | No such pipeline, or it has no `events:` block | +| 409 | Pipeline disabled | +| 413 | Payload larger than `events.max_payload_bytes` (default 64 MiB) | +| 429 | Queue full (`Retry-After: 60`); nothing was stored | + +### CLI trigger + +```sh +stagehand trigger deck-review --event-id msg-123 --payload-file mail.eml [--content-type message/rfc822] [--wait] [--json] +``` + +- Reads the payload from stdin if `--payload-file` is not given. +- Needs no signature: whoever can run it can already read the home. +- `--wait` waits for the event's run and exits non-zero unless it succeeded. If no `serve` is + running, it runs the queued events itself. + +Related commands: + +| Command | What it does | +|---|---| +| `events [--pipeline N] [--state queued]` | Lists accepted events | +| `drop-event ` | Discards a queued event and its payload | +| `status --json` | Shows `queued_events` for each pipeline | +| `show ` | Shows the run's event: id, source, size, SHA-256; never the payload | + +### Payload retention + +Payloads can be confidential (for example, mail with attachments). + +- **What is kept:** an event's payload, and the files of its run (work, logs, results), while the + event is queued and while its run is running or parked, so a parked run can be resumed. +- **When it is deleted:** as soon as the run **succeeds**. The run's database record stays: states, + exit codes, cost and summaries. +- **Purging a parked run:** `stagehand purge ` deletes a parked event run's files and payload + once it will not be resumed; after that it cannot be resumed. + +### Host settings + +```yaml +# runner config +events: + secret_file: /etc/stagehand/trigger.key # required for --trigger-addr; 0600 + max_payload_bytes: 67108864 # default 64 MiB +``` + ## Commands ``` @@ -228,9 +350,13 @@ run [--wait] [--json] start a run runs [--limit N] [--json] newest runs of a pipeline resume [--from STAGE] [--wait] re-run a parked run from a stage cancel stop a running run -serve [--status-addr 127.0.0.1:PORT] run schedules and stages until stopped -tick one pass: fire due schedules, run them to the end +serve [--status-addr ADDR] [--trigger-addr ADDR] run schedules, events and stages until stopped +tick one pass: fire due schedules and queued events, run them to the end status [--json] last and next run per pipeline +trigger [--event-id ID] [--payload-file F] [--wait] queue an event run +events [--pipeline N] [--state S] [--json] accepted events +drop-event discard a queued event and its payload +purge delete a finished event run's files and payload version ``` @@ -262,6 +388,7 @@ version "schedule": {"interval": "24h", "jitter": "10m"}, "next_due_at": "2026-11-04T03:04:12Z", "active_run_id": null, + "queued_events": 0, "last_run": { "id": 12, "trigger": "schedule", "state": "succeeded", "created_at": "2026-11-03T03:02:00Z", "finished_at": "2026-11-03T03:09:41Z", From 7922b5a89049ef33a09b4ec674889315294af2a6 Mon Sep 17 00:00:00 2001 From: jerryfane Date: Fri, 6 Nov 2026 03:16:30 +0100 Subject: [PATCH 04/11] fix(engine): delete a succeeded event run's payload at once, not at the next sweep (#18) --- internal/engine/advance.go | 13 ++++++++++- internal/engine/events_test.go | 40 +++++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/internal/engine/advance.go b/internal/engine/advance.go index 998122c..6ba81b4 100644 --- a/internal/engine/advance.go +++ b/internal/engine/advance.go @@ -176,7 +176,18 @@ func (e *Engine) advance(ctx context.Context, run store.Run) (bool, error) { } if allOK { e.Log.Info("run succeeded", "run", run.ID) - return false, e.Store.FinishRun(ctx, run.ID, store.RunSucceeded, "", "", e.now()) + if err := e.Store.FinishRun(ctx, run.ID, store.RunSucceeded, "", "", e.now()); err != nil { + return false, err + } + if run.EventID != 0 { + // Retention (owner, 6 Nov): a succeeded event run's payload and + // files go at once. A failure here is retried by the sweep. + run.State = store.RunSucceeded + if err := e.deleteRunFiles(ctx, run); err != nil { + e.Log.Error("retention; the sweep will retry", "run", run.ID, "err", err) + } + } + return false, nil } final := store.RunBlocked var haltStage, haltReason string diff --git a/internal/engine/events_test.go b/internal/engine/events_test.go index 9724119..94795d4 100644 --- a/internal/engine/events_test.go +++ b/internal/engine/events_test.go @@ -180,9 +180,7 @@ func TestRetentionDeletesOnSuccessAndKeepsParkedRuns(t *testing.T) { t.Fatal(err) } h.runToEnd(badRun.ID) - if err := h.eng.applyRetention(ctx); err != nil { - t.Fatal(err) - } + // Deleted at once when the run succeeds, not only by the periodic sweep. if _, err := os.Stat(badEv.PayloadPath); !os.IsNotExist(err) { t.Error("payload kept after the resumed run succeeded") } @@ -279,3 +277,39 @@ func TestConcurrentDuplicateTriggersStoreOneEvent(t *testing.T) { t.Errorf("%d payload files for one event", len(files)) } } + +// The payload goes the moment its run succeeds, not at the next periodic +// sweep (which runs only once a minute under serve). +func TestPayloadDeletedAtTheMomentTheRunSucceeds(t *testing.T) { + h := newHarness(t, nil) + h.add(eventPipeline, true) + res, err := h.trigger("now", "confidential") + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err := h.eng.startEvents(ctx); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(10 * time.Second) + for { + busy, err := h.eng.advanceAll(ctx, nil) + if err != nil { + t.Fatal(err) + } + if !busy { + break + } + if time.Now().After(deadline) { + t.Fatal("run did not settle") + } + time.Sleep(10 * time.Millisecond) + } + ev, _ := h.eng.Store.GetEvent(ctx, res.Event.ID) + if r, _ := h.eng.Store.GetRun(ctx, ev.RunID); r.State != store.RunSucceeded { + t.Fatalf("run %s", r.State) + } + if _, err := os.Stat(ev.PayloadPath); !os.IsNotExist(err) { + t.Fatal("payload still on disk right after the run succeeded") + } +} From 3be4eb7d2f64483ac91673cd1025a1502a2ec38e Mon Sep 17 00:00:00 2001 From: jerryfane Date: Fri, 6 Nov 2026 03:38:18 +0100 Subject: [PATCH 05/11] fix(events): claim deletion before unlinking, drop before unlink, never sweep a fresh payload (#19 review) --- internal/engine/advance.go | 3 +- internal/engine/events.go | 68 +++++++++++++----- internal/engine/events_test.go | 125 +++++++++++++++++++++++++++++++++ internal/store/events.go | 63 +++++++++++++---- 4 files changed, 224 insertions(+), 35 deletions(-) diff --git a/internal/engine/advance.go b/internal/engine/advance.go index 6ba81b4..5ec6562 100644 --- a/internal/engine/advance.go +++ b/internal/engine/advance.go @@ -182,8 +182,7 @@ func (e *Engine) advance(ctx context.Context, run store.Run) (bool, error) { if run.EventID != 0 { // Retention (owner, 6 Nov): a succeeded event run's payload and // files go at once. A failure here is retried by the sweep. - run.State = store.RunSucceeded - if err := e.deleteRunFiles(ctx, run); err != nil { + if err := e.deleteRunFiles(ctx, run.ID); err != nil { e.Log.Error("retention; the sweep will retry", "run", run.ID, "err", err) } } diff --git a/internal/engine/events.go b/internal/engine/events.go index 22244fe..eae7e9a 100644 --- a/internal/engine/events.go +++ b/internal/engine/events.go @@ -12,7 +12,6 @@ import ( "os" "path/filepath" "regexp" - "strings" "time" "github.com/gitmoot/stagehand/internal/spec" @@ -189,13 +188,23 @@ func (e *Engine) startEvents(ctx context.Context) error { // event run (owner decision, 6 Nov: confidential payloads do not outlive a // successful run), and sweeps payload files no event refers to. func (e *Engine) applyRetention(ctx context.Context) error { + var errs []error runs, err := e.Store.RetentionDue(ctx) if err != nil { return err } - var errs []error for _, r := range runs { - if err := e.deleteRunFiles(ctx, r); err != nil { + if err := e.deleteRunFiles(ctx, r.ID); err != nil && !errors.Is(err, store.ErrNotDeletable) { + errs = append(errs, err) + } + } + // Finish deletions interrupted between the claim and the unlinks. + claimed, err := e.Store.DeletedRuns(ctx) + if err != nil { + return err + } + for _, r := range claimed { + if err := e.removeRunFiles(ctx, r); err != nil { errs = append(errs, err) } } @@ -205,7 +214,23 @@ func (e *Engine) applyRetention(ctx context.Context) error { return errors.Join(errs...) } -func (e *Engine) deleteRunFiles(ctx context.Context, r store.Run) error { +// deleteRunFiles claims a finished run's files for deletion (atomically +// against resume), then removes them. +func (e *Engine) deleteRunFiles(ctx context.Context, runID int64) error { + r, err := e.Store.ClaimFilesDeletion(ctx, runID, e.now()) + if err != nil { + return err + } + if err := e.removeRunFiles(ctx, r); err != nil { + return err + } + e.Log.Info("deleted run files and payload", "run", r.ID, "event", r.EventID) + return nil +} + +// removeRunFiles removes a claimed run's directory and payload. It is +// idempotent, so an interrupted deletion can simply be repeated. +func (e *Engine) removeRunFiles(ctx context.Context, r store.Run) error { if err := os.RemoveAll(e.RunDir(r.ID)); err != nil { return fmt.Errorf("run %d: delete files: %w", r.ID, err) } @@ -218,15 +243,14 @@ func (e *Engine) deleteRunFiles(ctx context.Context, r store.Run) error { return fmt.Errorf("run %d: delete payload: %w", r.ID, err) } } - if err := e.Store.MarkFilesDeleted(ctx, r.ID, r.EventID, e.now()); err != nil { - return err - } - e.Log.Info("deleted run files and payload", "run", r.ID, "event", r.EventID) return nil } // sweepOrphanPayloads removes payload files that no live event references: -// leftovers of an intake interrupted between write and commit. +// leftovers of an intake interrupted before its commit, or of a drop or +// deletion interrupted after its commit. Files modified within orphanAge are +// never touched: an intake in another process may have placed its payload +// and not committed yet. func (e *Engine) sweepOrphanPayloads(ctx context.Context) error { entries, err := os.ReadDir(e.payloadDir()) if errors.Is(err, os.ErrNotExist) { @@ -235,19 +259,25 @@ func (e *Engine) sweepOrphanPayloads(ctx context.Context) error { if err != nil { return err } + var old []string + for _, ent := range entries { + info, err := ent.Info() + if err != nil || e.now().Sub(info.ModTime()) < orphanAge { + continue + } + old = append(old, filepath.Join(e.payloadDir(), ent.Name())) + } + if len(old) == 0 { + return nil + } + // Read the live set AFTER choosing candidates: any intake that placed + // one of them committed long ago (or never will). live, err := e.Store.LivePayloadPaths(ctx) if err != nil { return err } - for _, ent := range entries { - path := filepath.Join(e.payloadDir(), ent.Name()) - if strings.HasPrefix(ent.Name(), ".incoming-") { - // An intake in progress in another process owns it; only - // remove ones old enough to be abandoned. - if info, err := ent.Info(); err == nil && e.now().Sub(info.ModTime()) < orphanAge { - continue - } - } else if live[path] { + for _, path := range old { + if live[path] { continue } if err := os.Remove(path); err == nil { @@ -274,7 +304,7 @@ func (e *Engine) Purge(ctx context.Context, runID int64) error { if r.FilesDeletedAt != nil { return nil } - return e.deleteRunFiles(ctx, r) + return e.deleteRunFiles(ctx, runID) } // DropEvent discards a queued event and its payload. diff --git a/internal/engine/events_test.go b/internal/engine/events_test.go index 94795d4..6bba3f0 100644 --- a/internal/engine/events_test.go +++ b/internal/engine/events_test.go @@ -313,3 +313,128 @@ func TestPayloadDeletedAtTheMomentTheRunSucceeds(t *testing.T) { t.Fatal("payload still on disk right after the run succeeded") } } + +// An intake in another process may have placed its payload file and not yet +// committed the event. A sweep at that moment must leave the file alone. +func TestSweepSparesAPayloadWhoseIntakeHasNotCommitted(t *testing.T) { + h := newHarness(t, nil) + h.add(eventPipeline, true) + h.now = time.Now() + ctx := context.Background() + // The sweeping process (e.g. serve) has the store open already. + other, err := store.Open(ctx, filepath.Join(h.eng.Home, "stagehand.db")) + if err != nil { + t.Fatal(err) + } + defer other.Close() + peer := New(other, h.eng.Home, nil, nil, nil) + peer.Now = func() time.Time { return h.now } + var swept bool + _, _, err = h.eng.Store.InsertEvent(ctx, store.Event{Pipeline: "ev", Key: "k", Source: SourceHTTP, ReceivedAt: h.now, + PayloadPath: filepath.Join(h.eng.payloadDir(), "placed.payload")}, 3, func() error { + os.MkdirAll(h.eng.payloadDir(), 0o700) + if err := os.WriteFile(filepath.Join(h.eng.payloadDir(), "placed.payload"), []byte("deck"), 0o600); err != nil { + return err + } + // The sweep runs in the other process while this intake's + // transaction is still open. + swept = true + return peer.sweepOrphanPayloads(ctx) + }) + if err != nil { + t.Fatal(err) + } + if !swept { + t.Fatal("sweep did not run") + } + if _, err := os.Stat(filepath.Join(h.eng.payloadDir(), "placed.payload")); err != nil { + t.Fatal("sweep deleted the payload of an intake that was about to commit") + } +} + +// Purge claims the deletion in the database before touching files, so a +// concurrent resume is refused instead of resuming a run whose files vanish. +func TestPurgeAndResumeAreMutuallyExclusive(t *testing.T) { + h := newHarness(t, map[string][]string{"a": {"fail"}}) + h.add(eventPipeline, true) + res, _ := h.trigger("p", "secret") + h.drain() + ctx := context.Background() + ev, _ := h.eng.Store.GetEvent(ctx, res.Event.ID) + // A claim without the file removal, as if purge stopped right after it. + if _, err := h.eng.Store.ClaimFilesDeletion(ctx, ev.RunID, h.now); err != nil { + t.Fatal(err) + } + if err := h.eng.Resume(ctx, ev.RunID, ""); err == nil { + t.Fatal("resume accepted a run already claimed for deletion") + } + // And the other order: once resumed (running), it cannot be claimed. + h2 := newHarness(t, map[string][]string{"a": {"fail", "hang"}}) + h2.add(eventPipeline, true) + res2, _ := h2.trigger("q", "secret") + h2.drain() + ev2, _ := h2.eng.Store.GetEvent(ctx, res2.Event.ID) + if err := h2.eng.Resume(ctx, ev2.RunID, ""); err != nil { + t.Fatal(err) + } + if _, err := h2.eng.Store.ClaimFilesDeletion(ctx, ev2.RunID, h2.now); !errors.Is(err, store.ErrNotDeletable) { + t.Fatalf("claimed a running run: %v", err) + } + if err := h2.eng.Purge(ctx, ev2.RunID); err == nil { + t.Fatal("purged a running run") + } + if _, err := os.Stat(ev2.PayloadPath); err != nil { + t.Fatal("payload of the resumed run is gone") + } +} + +// A deletion interrupted after its claim is finished by the next sweep. +func TestInterruptedDeletionIsFinished(t *testing.T) { + h := newHarness(t, map[string][]string{"a": {"fail"}}) + h.add(eventPipeline, true) + res, _ := h.trigger("p", "secret") + h.drain() + ctx := context.Background() + ev, _ := h.eng.Store.GetEvent(ctx, res.Event.ID) + if _, err := h.eng.Store.ClaimFilesDeletion(ctx, ev.RunID, h.now); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(ev.PayloadPath); err != nil { + t.Fatal("setup: payload should still be on disk") + } + if err := h.eng.applyRetention(ctx); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(ev.PayloadPath); !os.IsNotExist(err) { + t.Error("claimed payload not removed by the sweep") + } + if _, err := os.Stat(h.eng.RunDir(ev.RunID)); !os.IsNotExist(err) { + t.Error("claimed run directory not removed by the sweep") + } +} + +// Drop commits "dropped" before unlinking: if the unlink never happens the +// event is still not runnable, and the sweep removes the file later. +func TestDroppedEventNeverRunsEvenIfItsFileSurvives(t *testing.T) { + h := newHarness(t, nil) + h.add(eventPipeline, true) + h.eng.SetEnabled(context.Background(), "ev", false) + h.eng.SetEnabled(context.Background(), "ev", true) + res, _ := h.trigger("d", "secret") + ctx := context.Background() + // Make the unlink fail: replace the payload with a non-empty directory. + os.Remove(res.Event.PayloadPath) + os.MkdirAll(filepath.Join(res.Event.PayloadPath, "x"), 0o700) + _, err := h.eng.DropEvent(ctx, res.Event.ID) + if err == nil { + t.Fatal("setup: unlink should have failed") + } + ev, _ := h.eng.Store.GetEvent(ctx, res.Event.ID) + if ev.State != store.EventDropped || ev.PayloadDeletedAt == nil { + t.Fatalf("event %s after a failed unlink; it must already be dropped", ev.State) + } + h.drain() + if h.ex.callCount("a") != 0 { + t.Fatal("a dropped event ran") + } +} diff --git a/internal/store/events.go b/internal/store/events.go index ea13a70..f6fa991 100644 --- a/internal/store/events.go +++ b/internal/store/events.go @@ -227,38 +227,73 @@ func (s *Store) DropEvent(ctx context.Context, id int64, at time.Time) (Event, e if e.State != EventQueued { return Event{}, fmt.Errorf("event %d is %s; only a queued event can be dropped (use `purge ` for a started one)", id, e.State) } - if err := removePayload(e.PayloadPath); err != nil { + // Record the drop first: once committed the event can never start, so a + // crash before the unlink below leaves only an orphan file, which the + // sweep removes. Unlinking first could leave a runnable event with no + // input. + now := fmtTime(at) + if _, err := tx.ExecContext(ctx, `UPDATE events SET state=?, payload_deleted_at=? WHERE id=?`, EventDropped, now, id); err != nil { return Event{}, err } - if _, err := tx.ExecContext(ctx, `UPDATE events SET state=?, payload_deleted_at=? WHERE id=?`, EventDropped, fmtTime(at), id); err != nil { + if err := tx.Commit(); err != nil { return Event{}, err } e.State = EventDropped - return e, tx.Commit() + e.PayloadDeletedAt = &at + if err := removePayload(e.PayloadPath); err != nil { + return e, fmt.Errorf("event %d dropped, but its payload file is still there (the sweep will remove it): %w", id, err) + } + return e, nil } -// RetentionDue lists succeeded event runs whose files are not deleted yet. +// RetentionDue lists succeeded event runs whose files are not claimed for +// deletion yet. func (s *Store) RetentionDue(ctx context.Context) ([]Run, error) { return s.queryRuns(ctx, `WHERE event_id IS NOT NULL AND state=? AND files_deleted_at IS NULL ORDER BY id`, RunSucceeded) } -// MarkFilesDeleted records that a run's directory and its event's payload -// are gone. -func (s *Store) MarkFilesDeleted(ctx context.Context, runID, eventID int64, at time.Time) error { +// DeletedRuns lists runs whose files were claimed for deletion, so an +// interrupted deletion can be finished. +func (s *Store) DeletedRuns(ctx context.Context) ([]Run, error) { + return s.queryRuns(ctx, `WHERE files_deleted_at IS NOT NULL ORDER BY id`) +} + +// ErrNotDeletable is returned when a run cannot have its files deleted now +// (it is running, or it is not a finished event run). +var ErrNotDeletable = errors.New("run files cannot be deleted now") + +// ClaimFilesDeletion irreversibly marks a finished run's files and its +// event's payload as deleted, BEFORE anything is removed from disk. It only +// succeeds on a run that is not running and not already claimed, in one +// transaction, so it is serialized against ResumeRun (which refuses a +// claimed run). The caller then deletes the files; an interrupted deletion +// is finished from DeletedRuns. +func (s *Store) ClaimFilesDeletion(ctx context.Context, runID int64, at time.Time) (Run, error) { tx, err := s.db.BeginTx(ctx, nil) if err != nil { - return err + return Run{}, err } defer tx.Rollback() - if _, err := tx.ExecContext(ctx, `UPDATE runs SET files_deleted_at=? WHERE id=?`, fmtTime(at), runID); err != nil { - return err + res, err := tx.ExecContext(ctx, `UPDATE runs SET files_deleted_at=? WHERE id=? AND state!=? AND files_deleted_at IS NULL`, + fmtTime(at), runID, RunRunning) + if err != nil { + return Run{}, err + } + if n, err := res.RowsAffected(); err != nil { + return Run{}, err + } else if n == 0 { + return Run{}, fmt.Errorf("run %d: %w (running, already deleted, or missing)", runID, ErrNotDeletable) + } + r, err := scanRun(tx.QueryRowContext(ctx, `SELECT `+runCols+` FROM runs WHERE id=?`, runID)) + if err != nil { + return Run{}, err } - if eventID != 0 { - if _, err := tx.ExecContext(ctx, `UPDATE events SET payload_deleted_at=COALESCE(payload_deleted_at, ?) WHERE id=?`, fmtTime(at), eventID); err != nil { - return err + if r.EventID != 0 { + if _, err := tx.ExecContext(ctx, `UPDATE events SET payload_deleted_at=COALESCE(payload_deleted_at, ?) WHERE id=?`, fmtTime(at), r.EventID); err != nil { + return Run{}, err } } - return tx.Commit() + return r, tx.Commit() } // LivePayloadPaths returns every payload path not yet deleted, for the From a06ce234e53c65f9da03f83f29566df1b966e137 Mon Sep 17 00:00:00 2001 From: jerryfane Date: Fri, 6 Nov 2026 03:39:46 +0100 Subject: [PATCH 06/11] docs: retention crash safety (#19 review) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 7121cab..8458797 100644 --- a/README.md +++ b/README.md @@ -326,6 +326,10 @@ Payloads can be confidential (for example, mail with attachments). exit codes, cost and summaries. - **Purging a parked run:** `stagehand purge ` deletes a parked event run's files and payload once it will not be resumed; after that it cannot be resumed. +- **Crash safety:** a deletion is recorded in the database before any file is removed, so it can + never race a resume. A deletion interrupted by a crash is finished by the next clean-up sweep, + at start-up and every minute. Payload files no event refers to are removed by the same sweep, + but only once they are 15 minutes old, so an intake still committing is never touched. ### Host settings From 2fdb7de5afeebed203724c88d1d9fd2d192d8376 Mon Sep 17 00:00:00 2001 From: jerryfane Date: Fri, 6 Nov 2026 04:02:09 +0100 Subject: [PATCH 07/11] fix(events): sweep only files it can lock; intakes hold their payload lock until commit (#19 review round 2) --- internal/engine/engine.go | 7 +++- internal/engine/events.go | 73 ++++++++++++++++++++++++---------- internal/engine/events_test.go | 52 ++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 22 deletions(-) diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 51d081b..b1ff5e8 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -58,8 +58,11 @@ type Engine struct { // stopping is set once Loop is shutting down; advance then only folds // results and never starts a stage. stopping atomic.Bool - wake chan struct{} - wg sync.WaitGroup + // beforeEventCommit, if set (tests only), runs after an intake has + // placed its payload file and before its event commits. + beforeEventCommit func(payloadPath string) + wake chan struct{} + wg sync.WaitGroup } type flightKey struct { diff --git a/internal/engine/events.go b/internal/engine/events.go index eae7e9a..2d7fb1e 100644 --- a/internal/engine/events.go +++ b/internal/engine/events.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "regexp" + "syscall" "time" "github.com/gitmoot/stagehand/internal/spec" @@ -99,6 +100,17 @@ func (e *Engine) Trigger(ctx context.Context, req EventRequest) (EventResult, er if err != nil { return EventResult{}, err } + // The intake holds an exclusive lock on its payload file from creation + // until its event is committed (or abandoned). The lock follows the + // inode through the rename, and the orphan sweep skips any file it + // cannot lock, so a file that belongs to an intake in progress is never + // swept, however old its contents. + if err := syscall.Flock(int(tmp.Fd()), syscall.LOCK_EX); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return EventResult{}, fmt.Errorf("lock payload file: %w", err) + } + defer tmp.Close() // releases the lock, after the commit below defer os.Remove(tmp.Name()) // no-op once renamed h := sha256.New() limit := req.MaxBytes @@ -106,8 +118,8 @@ func (e *Engine) Trigger(ctx context.Context, req EventRequest) (EventResult, er limit = 64 << 20 } n, err := io.Copy(io.MultiWriter(tmp, h), io.LimitReader(req.Payload, limit+1)) - if cerr := tmp.Close(); err == nil { - err = cerr + if err == nil { + err = tmp.Sync() } if err != nil { return EventResult{}, fmt.Errorf("read payload: %w", err) @@ -123,7 +135,13 @@ func (e *Engine) Trigger(ctx context.Context, req EventRequest) (EventResult, er PayloadBytes: n, PayloadSHA256: hex.EncodeToString(h.Sum(nil)), } stored, dup, err := e.Store.InsertEvent(ctx, ev, sp.Events.MaxQueued, func() error { - return os.Rename(tmp.Name(), final) + if err := os.Rename(tmp.Name(), final); err != nil { + return err + } + if e.beforeEventCommit != nil { + e.beforeEventCommit(final) + } + return nil }) if err != nil { os.Remove(final) @@ -248,9 +266,14 @@ func (e *Engine) removeRunFiles(ctx context.Context, r store.Run) error { // sweepOrphanPayloads removes payload files that no live event references: // leftovers of an intake interrupted before its commit, or of a drop or -// deletion interrupted after its commit. Files modified within orphanAge are -// never touched: an intake in another process may have placed its payload -// and not committed yet. +// deletion interrupted after its commit. +// +// It never races an intake: each intake holds an exclusive lock on its file +// until its event is committed, and the sweep only removes a file whose lock +// it holds itself, checking the committed live set after taking the lock. A +// file whose lock it cannot take belongs to an intake in progress. Files +// modified within orphanAge are skipped as well, which covers the instant +// between an intake creating its file and locking it. func (e *Engine) sweepOrphanPayloads(ctx context.Context) error { entries, err := os.ReadDir(e.payloadDir()) if errors.Is(err, os.ErrNotExist) { @@ -259,30 +282,40 @@ func (e *Engine) sweepOrphanPayloads(ctx context.Context) error { if err != nil { return err } - var old []string for _, ent := range entries { info, err := ent.Info() - if err != nil || e.now().Sub(info.ModTime()) < orphanAge { + if err != nil || !info.Mode().IsRegular() || e.now().Sub(info.ModTime()) < orphanAge { continue } - old = append(old, filepath.Join(e.payloadDir(), ent.Name())) + path := filepath.Join(e.payloadDir(), ent.Name()) + if err := e.sweepOne(ctx, path); err != nil { + return err + } } - if len(old) == 0 { - return nil + return nil +} + +// sweepOne removes path if it is unlocked and no live event references it. +func (e *Engine) sweepOne(ctx context.Context, path string) error { + f, err := os.Open(path) + if err != nil { + return nil // gone already } - // Read the live set AFTER choosing candidates: any intake that placed - // one of them committed long ago (or never will). + defer f.Close() + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + return nil // an intake holds it + } + // With the lock held, no intake can still be committing this file: + // whatever the live set says now is final for it. live, err := e.Store.LivePayloadPaths(ctx) if err != nil { return err } - for _, path := range old { - if live[path] { - continue - } - if err := os.Remove(path); err == nil { - e.Log.Warn("removed orphaned payload", "file", path) - } + if live[path] { + return nil + } + if err := os.Remove(path); err == nil { + e.Log.Warn("removed orphaned payload", "file", path) } return nil } diff --git a/internal/engine/events_test.go b/internal/engine/events_test.go index 6bba3f0..ebef8b3 100644 --- a/internal/engine/events_test.go +++ b/internal/engine/events_test.go @@ -438,3 +438,55 @@ func TestDroppedEventNeverRunsEvenIfItsFileSurvives(t *testing.T) { t.Fatal("a dropped event ran") } } + +// The reviewer's scenario: a payload whose bytes were written long ago (a +// slow producer that waited before closing) is placed just before a sweep in +// another process, while its event has not committed. Its age must not make +// it an orphan. +func TestSweepSparesAnAgedPayloadWhoseIntakeHasNotCommitted(t *testing.T) { + h := newHarness(t, nil) + h.add(eventPipeline, true) + h.now = time.Now() + ctx := context.Background() + other, err := store.Open(ctx, filepath.Join(h.eng.Home, "stagehand.db")) + if err != nil { + t.Fatal(err) + } + defer other.Close() + peer := New(other, h.eng.Home, nil, nil, nil) + peer.Now = func() time.Time { return h.now } + var placed string + h.eng.beforeEventCommit = func(path string) { + placed = path + old := h.now.Add(-2 * time.Hour) + os.Chtimes(path, old, old) + if err := peer.sweepOrphanPayloads(ctx); err != nil { + t.Error(err) + } + } + res, err := h.trigger("aged", "deck bytes") + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(placed); err != nil { + t.Fatalf("an aged payload of an uncommitted intake was swept: %v", err) + } + ev, _ := h.eng.Store.GetEvent(ctx, res.Event.ID) + if ev.State != store.EventQueued || ev.PayloadDeletedAt != nil { + t.Fatalf("event %+v", ev) + } + // Once committed and unlocked, a genuinely unreferenced old file goes. + stray := filepath.Join(h.eng.payloadDir(), "stray.payload") + os.WriteFile(stray, []byte("x"), 0o600) + old := h.now.Add(-2 * time.Hour) + os.Chtimes(stray, old, old) + if err := peer.sweepOrphanPayloads(ctx); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stray); !os.IsNotExist(err) { + t.Error("an unreferenced, unlocked old file was not swept") + } + if _, err := os.Stat(placed); err != nil { + t.Error("the committed event's payload was swept") + } +} From 30f8fa98644e2784ec7ab9362f8de175b8c069f2 Mon Sep 17 00:00:00 2001 From: jerryfane Date: Fri, 6 Nov 2026 04:02:42 +0100 Subject: [PATCH 08/11] docs: sweep/intake lock (#19) --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8458797..8d32684 100644 --- a/README.md +++ b/README.md @@ -328,8 +328,9 @@ Payloads can be confidential (for example, mail with attachments). once it will not be resumed; after that it cannot be resumed. - **Crash safety:** a deletion is recorded in the database before any file is removed, so it can never race a resume. A deletion interrupted by a crash is finished by the next clean-up sweep, - at start-up and every minute. Payload files no event refers to are removed by the same sweep, - but only once they are 15 minutes old, so an intake still committing is never touched. + at start-up and every minute. Payload files no event refers to are removed by the same sweep. + An intake holds a lock on its file until its event is committed, and the sweep never removes a + file it cannot lock, so an intake in progress is never touched, however long its sender took. ### Host settings From e1a30c8fcd06002b488806a49e675a468f185b48 Mon Sep 17 00:00:00 2001 From: jerryfane Date: Fri, 6 Nov 2026 04:17:58 +0100 Subject: [PATCH 09/11] fix(events): purge retry finishes deletion, fair schedule/event arbitration, scoped trigger --wait, NFS-safe sweep lock (#19 review round 3) --- internal/cli/cli_test.go | 34 ++++++++++++ internal/cli/events.go | 33 ++++++++---- internal/engine/advance.go | 10 ++++ internal/engine/engine.go | 25 ++++++--- internal/engine/events.go | 59 ++++++++++++++++++--- internal/engine/events_test.go | 95 +++++++++++++++++++++++++++++++++- 6 files changed, 229 insertions(+), 27 deletions(-) diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index d877b8e..cdbe6eb 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -70,3 +70,37 @@ func TestConcurrentStandaloneRunsBothExecute(t *testing.T) { t.Fatalf("b runs %s", out) } } + +// trigger --wait returns once its own event's run is done, even while other +// pipelines keep running and newer events keep the queue non-empty. +func TestTriggerWaitReturnsWhenItsOwnRunIsDone(t *testing.T) { + home := t.TempDir() + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "fast.yaml"), []byte("name: fast\nevents: {max_queued: 5}\nstages:\n - {id: a, cmd: 'true'}\n"), 0o600) + os.WriteFile(filepath.Join(dir, "slow.yaml"), []byte("name: slow\nevents: {max_queued: 5}\nstages:\n - {id: a, cmd: 'sleep 30'}\n"), 0o600) + for _, f := range []string{"fast.yaml", "slow.yaml"} { + if code, _, errs := runCLI(home, "add", filepath.Join(dir, f), "--enable"); code != 0 { + t.Fatal(errs) + } + } + empty := filepath.Join(dir, "empty") + os.WriteFile(empty, nil, 0o600) + // A long event in another pipeline is queued first. + if code, _, errs := runCLI(home, "trigger", "slow", "--payload-file", empty); code != 0 { + t.Fatal(errs) + } + done := make(chan int, 1) + start := time.Now() + go func() { code, _, _ := runCLI(home, "trigger", "fast", "--payload-file", empty, "--wait"); done <- code }() + select { + case code := <-done: + if code != 0 { + t.Fatalf("exit %d", code) + } + if time.Since(start) > 15*time.Second { + t.Errorf("took %v; it waited for the other pipeline", time.Since(start)) + } + case <-time.After(25 * time.Second): + t.Fatal("trigger --wait did not return after its own run finished") + } +} diff --git a/internal/cli/events.go b/internal/cli/events.go index 15890a9..c18dc73 100644 --- a/internal/cli/events.go +++ b/internal/cli/events.go @@ -62,33 +62,43 @@ func (a *app) trigger(ctx context.Context, args []string) error { fmt.Fprintf(a.stdout, "event %d: %s\n", ev.ID, ev.State) return nil } - runID, err := a.drainUntilStarted(ctx, ev.ID) + runID, err := a.drainEvent(ctx, ev.Pipeline, ev.ID) if err != nil { return err } return a.finish(ctx, runID, true, *asJSON) } -// drainUntilStarted waits for event id to get a run, executing queued events -// here when no other process executes stages. -func (a *app) drainUntilStarted(ctx context.Context, id int64) (int64, error) { +// drainEvent waits until event id has a run that has settled, and returns +// the run. When no other process executes stages it does the work here, +// limited to the event's own pipeline (whose events run one at a time, in +// order), and stops as soon as that run settles: newer arrivals or other +// pipelines never keep it waiting. +func (a *app) drainEvent(ctx context.Context, pipeline string, id int64) (int64, error) { + settled := func(ctx context.Context) bool { + ev, err := a.store.GetEvent(ctx, id) + if err != nil || ev.State != store.EventStarted { + return ev.State == store.EventDropped + } + r, err := a.store.GetRun(ctx, ev.RunID) + return err == nil && r.State != store.RunRunning + } for { ev, err := a.store.GetEvent(ctx, id) if err != nil { return 0, err } - switch ev.State { - case store.EventStarted: - return ev.RunID, nil - case store.EventDropped: + if ev.State == store.EventDropped { return 0, fmt.Errorf("event %d was dropped", id) } + if ev.State == store.EventStarted && settled(ctx) { + return ev.RunID, nil + } lock, err := homelock.TryAcquire(filepath.Join(a.home, "stagehand.lock"), homelock.ModeCLI) if err == nil { rerr := a.engine.Recover(ctx) if rerr == nil { - // Start (and run) queued events until this one has a run. - rerr = a.engine.Loop(ctx, engine.LoopOptions{Events: true, UntilIdle: true}) + rerr = a.engine.Loop(ctx, engine.LoopOptions{Events: true, Pipeline: pipeline, Done: settled, UntilIdle: true}) } lock.Release() if rerr != nil { @@ -96,7 +106,8 @@ func (a *app) drainUntilStarted(ctx context.Context, id int64) (int64, error) { } continue } - if !errors.Is(err, homelock.ErrHeld) { + var held *homelock.HeldError + if !errors.As(err, &held) { return 0, err } select { diff --git a/internal/engine/advance.go b/internal/engine/advance.go index 5ec6562..ffbb293 100644 --- a/internal/engine/advance.go +++ b/internal/engine/advance.go @@ -454,6 +454,16 @@ func (e *Engine) schedule(ctx context.Context) error { if active { continue // fire as soon as the active run settles } + if sp.Events != nil { + ev, err := e.Store.OldestQueuedEvent(ctx, p.Name) + if err != nil { + errs = append(errs, err) + continue + } + if eventBeforeSchedule(ev, p) { + continue // an older event goes first; the schedule stays due + } + } if _, err := e.createRun(ctx, p, sp, "schedule"); err != nil && !errors.Is(err, store.ErrActiveRun) { errs = append(errs, err) continue diff --git a/internal/engine/engine.go b/internal/engine/engine.go index b1ff5e8..4985feb 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -356,6 +356,12 @@ type LoopOptions struct { Only map[int64]bool // Events starts runs for queued events (implied by Schedule). Events bool + // Pipeline, when set, limits event starts and advancing to this one + // pipeline. + Pipeline string + // Done, when set, ends the loop as soon as it reports true after an + // advance pass with nothing of the selected work in flight. + Done func(context.Context) bool // UntilIdle returns once no selected run is running (and, with Events, // no enabled pipeline has a queued event left). UntilIdle bool @@ -378,17 +384,21 @@ func (e *Engine) Loop(ctx context.Context, opts LoopOptions) error { } } if opts.Schedule || opts.Events { - if err := e.startEvents(ctx); err != nil && ctx.Err() == nil { + if err := e.startEvents(ctx, opts.Pipeline); err != nil && ctx.Err() == nil { e.Log.Error("start events", "err", err) } } - busy, err := e.advanceAll(ctx, opts.Only) + busy, err := e.advanceAll(ctx, opts.Only, opts.Pipeline) if err != nil && ctx.Err() == nil { e.Log.Error("advance", "err", err) } if !busy && (opts.Schedule || opts.Events) && opts.UntilIdle { // A run that just settled may leave events waiting behind it. - if waiting, qerr := e.eventsWaiting(ctx); qerr == nil && waiting { + if waiting, qerr := e.eventsWaiting(ctx, opts.Pipeline); qerr == nil && waiting { + busy = true + } + // A schedule that yielded to an older event is still due. + if due, serr := e.scheduleDue(ctx); opts.Schedule && serr == nil && due { busy = true } } @@ -401,6 +411,9 @@ func (e *Engine) Loop(ctx context.Context, opts LoopOptions) error { } lastSweep = time.Now() } + if opts.Done != nil && err == nil && opts.Done(ctx) && e.inflightCount() == 0 { + return nil + } if opts.UntilIdle && !busy && err == nil { return nil } @@ -427,7 +440,7 @@ func (e *Engine) stopAll(only map[int64]bool) { e.wg.Wait() ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - if _, err := e.advanceAll(ctx, only); err != nil { + if _, err := e.advanceAll(ctx, only, ""); err != nil { e.Log.Error("final advance", "err", err) } } @@ -469,14 +482,14 @@ func (e *Engine) inflightCount() int { // advanceAll advances every selected running run. busy reports whether any // selected run is still running afterwards. -func (e *Engine) advanceAll(ctx context.Context, only map[int64]bool) (busy bool, err error) { +func (e *Engine) advanceAll(ctx context.Context, only map[int64]bool, pipeline string) (busy bool, err error) { runs, err := e.Store.ActiveRuns(ctx) if err != nil { return e.inflightCount() > 0, err } var errs []error for _, r := range runs { - if only != nil && !only[r.ID] { + if (only != nil && !only[r.ID]) || (pipeline != "" && r.Pipeline != pipeline) { continue } running, err := e.advance(ctx, r) diff --git a/internal/engine/events.go b/internal/engine/events.go index 2d7fb1e..464ec8d 100644 --- a/internal/engine/events.go +++ b/internal/engine/events.go @@ -158,14 +158,14 @@ func (e *Engine) Trigger(ctx context.Context, req EventRequest) (EventResult, er // startEvents starts the oldest queued event of every enabled pipeline that // has no running run. -func (e *Engine) startEvents(ctx context.Context) error { +func (e *Engine) startEvents(ctx context.Context, only string) error { pipelines, err := e.Store.ListPipelines(ctx) if err != nil { return err } var errs []error for _, p := range pipelines { - if !p.Enabled { + if !p.Enabled || (only != "" && p.Name != only) { continue } ev, err := e.Store.OldestQueuedEvent(ctx, p.Name) @@ -182,6 +182,9 @@ func (e *Engine) startEvents(ctx context.Context) error { if err != nil { continue } + if sp.Schedule != nil && p.NextDueAt != nil && !e.now().Before(*p.NextDueAt) && !eventBeforeSchedule(ev, p) { + continue // the due schedule is older; it goes first + } ids := make([]string, len(sp.Stages)) for i, st := range sp.Stages { ids[i] = st.ID @@ -297,13 +300,21 @@ func (e *Engine) sweepOrphanPayloads(ctx context.Context) error { // sweepOne removes path if it is unlocked and no live event references it. func (e *Engine) sweepOne(ctx context.Context, path string) error { - f, err := os.Open(path) - if err != nil { + // Read-write: on NFS, Linux emulates flock with fcntl locks, and an + // exclusive one needs a writable descriptor. + f, err := os.OpenFile(path, os.O_RDWR, 0) + if errors.Is(err, os.ErrNotExist) { return nil // gone already } + if err != nil { + return fmt.Errorf("sweep %s: %w", path, err) + } defer f.Close() if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - return nil // an intake holds it + if errors.Is(err, syscall.EWOULDBLOCK) { + return nil // an intake holds it + } + return fmt.Errorf("sweep %s: lock: %w", path, err) } // With the lock held, no intake can still be committing this file: // whatever the live set says now is final for it. @@ -335,7 +346,8 @@ func (e *Engine) Purge(ctx context.Context, runID int64) error { return fmt.Errorf("run %d is running; cancel it first", runID) } if r.FilesDeletedAt != nil { - return nil + // Claimed earlier; removal may not have completed. It is idempotent. + return e.removeRunFiles(ctx, r) } return e.deleteRunFiles(ctx, runID) } @@ -364,13 +376,13 @@ func randomHex(n int) string { } // eventsWaiting reports whether any enabled pipeline has a queued event. -func (e *Engine) eventsWaiting(ctx context.Context) (bool, error) { +func (e *Engine) eventsWaiting(ctx context.Context, only string) (bool, error) { pipelines, err := e.Store.ListPipelines(ctx) if err != nil { return false, err } for _, p := range pipelines { - if !p.Enabled { + if !p.Enabled || (only != "" && p.Name != only) { continue } n, err := e.Store.QueuedEventCount(ctx, p.Name) @@ -383,3 +395,34 @@ func (e *Engine) eventsWaiting(ctx context.Context) (bool, error) { } return false, nil } + +// eventBeforeSchedule reports whether a pipeline's oldest queued event +// arrived before its schedule came due. Whichever is older takes the +// pipeline's single run slot, so neither source can starve the other. +func eventBeforeSchedule(ev *store.Event, p store.Pipeline) bool { + if ev == nil { + return false + } + if p.NextDueAt == nil { + return true + } + return !ev.ReceivedAt.After(*p.NextDueAt) +} + +// scheduleDue reports whether an enabled, scheduled pipeline is due now. +func (e *Engine) scheduleDue(ctx context.Context) (bool, error) { + pipelines, err := e.Store.ListPipelines(ctx) + if err != nil { + return false, err + } + now := e.now() + for _, p := range pipelines { + if p.Enabled && p.NextDueAt != nil && !now.Before(*p.NextDueAt) { + sp, err := spec.Load(p.SpecYAML) + if err == nil && sp.Schedule != nil { + return true, nil + } + } + } + return false, nil +} diff --git a/internal/engine/events_test.go b/internal/engine/events_test.go index ebef8b3..0b0cc97 100644 --- a/internal/engine/events_test.go +++ b/internal/engine/events_test.go @@ -288,12 +288,12 @@ func TestPayloadDeletedAtTheMomentTheRunSucceeds(t *testing.T) { t.Fatal(err) } ctx := context.Background() - if err := h.eng.startEvents(ctx); err != nil { + if err := h.eng.startEvents(ctx, ""); err != nil { t.Fatal(err) } deadline := time.Now().Add(10 * time.Second) for { - busy, err := h.eng.advanceAll(ctx, nil) + busy, err := h.eng.advanceAll(ctx, nil, "") if err != nil { t.Fatal(err) } @@ -490,3 +490,94 @@ func TestSweepSparesAnAgedPayloadWhoseIntakeHasNotCommitted(t *testing.T) { t.Error("the committed event's payload was swept") } } + +// Retrying purge after an interrupted deletion finishes it, instead of +// reporting success while the confidential files stay. +func TestPurgeRetryFinishesAnInterruptedDeletion(t *testing.T) { + h := newHarness(t, map[string][]string{"a": {"fail"}}) + h.add(eventPipeline, true) + res, _ := h.trigger("p", "secret") + h.drain() + ctx := context.Background() + ev, _ := h.eng.Store.GetEvent(ctx, res.Event.ID) + if _, err := h.eng.Store.ClaimFilesDeletion(ctx, ev.RunID, h.now); err != nil { + t.Fatal(err) + } + if err := h.eng.Purge(ctx, ev.RunID); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(ev.PayloadPath); !os.IsNotExist(err) { + t.Error("purge retry left the payload") + } + if _, err := os.Stat(h.eng.RunDir(ev.RunID)); !os.IsNotExist(err) { + t.Error("purge retry left the run directory") + } +} + +// A schedule that is due again every time its run ends must not starve +// queued events: whichever is older gets the slot. +func TestDueScheduleDoesNotStarveQueuedEvents(t *testing.T) { + h := newHarness(t, nil) + // Frozen clock: the 1-minute schedule is due on every pass. + h.add("name: ev\nschedule: {interval: 1m}\nevents: {max_queued: 3}\nstages:\n - {id: a, cmd: x}\n", true) + ctx := context.Background() + // The schedule has been due since before the event arrived; it goes + // first, then the event, even though the schedule is due again. + h.now = h.now.Add(30 * time.Second) + if _, err := h.trigger("e1", "x"); err != nil { + t.Fatal(err) + } + for range 4 { + if err := h.eng.schedule(ctx); err != nil { + t.Fatal(err) + } + if err := h.eng.startEvents(ctx, ""); err != nil { + t.Fatal(err) + } + runs, _ := h.eng.Runs(ctx, "ev", 10) + h.eng.Store.FinishRun(ctx, runs[0].ID, store.RunSucceeded, "", "", h.now) + h.now = h.now.Add(2 * time.Minute) // longer than the interval + } + evs, _ := h.eng.Store.ListEvents(ctx, "ev", store.EventQueued, 10) + if len(evs) != 0 { + t.Fatalf("event starved: %d still queued after 4 slots", len(evs)) + } + runs, _ := h.eng.Runs(ctx, "ev", 10) + if runs[len(runs)-1].Trigger != "schedule" || runs[len(runs)-2].Trigger != "event" { + t.Errorf("order %v; want the older schedule first, then the event", triggers(runs)) + } +} + +func triggers(rs []RunSummary) []string { + var out []string + for i := len(rs) - 1; i >= 0; i-- { + out = append(out, rs[i].Trigger) + } + return out +} + +// The sweep must take its exclusive lock on a writable descriptor (NFS +// emulates flock with fcntl locks, which need one) and must not mistake a +// permanent locking error for an intake in progress. +func TestSweepOpensCandidatesWritable(t *testing.T) { + h := newHarness(t, nil) + os.MkdirAll(h.eng.payloadDir(), 0o700) + ro := filepath.Join(h.eng.payloadDir(), "readonly.payload") + os.WriteFile(ro, []byte("x"), 0o400) + old := h.now.Add(-time.Hour) + os.Chtimes(ro, old, old) + err := h.eng.sweepOrphanPayloads(context.Background()) + if os.Geteuid() == 0 { + // root can open a 0400 file read-write; the file is swept. + if err != nil { + t.Fatal(err) + } + if _, serr := os.Stat(ro); !os.IsNotExist(serr) { + t.Error("unreferenced old file not swept") + } + return + } + if err == nil { + t.Fatal("a file the sweep cannot open writable was silently treated as in use") + } +} From 8bc20da92bb890228d72e0b9852acf98a840437b Mon Sep 17 00:00:00 2001 From: jerryfane Date: Fri, 6 Nov 2026 04:19:33 +0100 Subject: [PATCH 10/11] docs: schedule/event fairness, trigger --wait scope (#19) --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d32684..178f131 100644 --- a/README.md +++ b/README.md @@ -244,6 +244,9 @@ stages: - When `max_queued` events are already waiting, a new trigger is refused and nothing is stored, so the caller knows to retry. Accepted events are never lost or run twice. - A disabled pipeline refuses events; its queued events wait until it is enabled again. +- **Schedule and events on one pipeline:** whichever has waited longer gets the next run, the due + schedule or the oldest queued event. A schedule that is due again every time a run ends cannot + starve events, and a stream of events cannot starve the schedule. **What a stage sees.** The payload is passed **only as a file**. stagehand never puts event content into a command line or an environment value. Shell stages get: @@ -305,7 +308,8 @@ stagehand trigger deck-review --event-id msg-123 --payload-file mail.eml [--cont - Reads the payload from stdin if `--payload-file` is not given. - Needs no signature: whoever can run it can already read the home. - `--wait` waits for the event's run and exits non-zero unless it succeeded. If no `serve` is - running, it runs the queued events itself. + running, it runs its own pipeline's queued events itself (in order, up to and including this + one) and returns as soon as this event's run ends. Related commands: From eb8b0986d4d674af7cc15eb2419164c1c9519daa Mon Sep 17 00:00:00 2001 From: jerryfane Date: Fri, 6 Nov 2026 04:33:49 +0100 Subject: [PATCH 11/11] fix(events): events-only loops never yield to schedules; honor Done before dispatching (#19 review round 4) --- internal/cli/events.go | 3 +++ internal/engine/engine.go | 7 +++++- internal/engine/events.go | 6 +++-- internal/engine/events_test.go | 42 ++++++++++++++++++++++++++++++++-- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/internal/cli/events.go b/internal/cli/events.go index c18dc73..64ceab3 100644 --- a/internal/cli/events.go +++ b/internal/cli/events.go @@ -96,6 +96,9 @@ func (a *app) drainEvent(ctx context.Context, pipeline string, id int64) (int64, } lock, err := homelock.TryAcquire(filepath.Join(a.home, "stagehand.lock"), homelock.ModeCLI) if err == nil { + // Loop checks Done before dispatching anything, so if the run + // settled while another process held the lock, nothing newer is + // started here. rerr := a.engine.Recover(ctx) if rerr == nil { rerr = a.engine.Loop(ctx, engine.LoopOptions{Events: true, Pipeline: pipeline, Done: settled, UntilIdle: true}) diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 4985feb..2ec581c 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -378,13 +378,18 @@ func (e *Engine) Loop(ctx context.Context, opts LoopOptions) error { } lastSweep := time.Now() for { + // A caller's goal already met (e.g. its run settled while another + // process held the lock) ends the loop before it dispatches work. + if opts.Done != nil && opts.Done(ctx) && e.inflightCount() == 0 { + return nil + } if opts.Schedule { if err := e.schedule(ctx); err != nil && ctx.Err() == nil { e.Log.Error("schedule", "err", err) } } if opts.Schedule || opts.Events { - if err := e.startEvents(ctx, opts.Pipeline); err != nil && ctx.Err() == nil { + if err := e.startEvents(ctx, opts.Pipeline, opts.Schedule); err != nil && ctx.Err() == nil { e.Log.Error("start events", "err", err) } } diff --git a/internal/engine/events.go b/internal/engine/events.go index 464ec8d..41a1511 100644 --- a/internal/engine/events.go +++ b/internal/engine/events.go @@ -158,7 +158,7 @@ func (e *Engine) Trigger(ctx context.Context, req EventRequest) (EventResult, er // startEvents starts the oldest queued event of every enabled pipeline that // has no running run. -func (e *Engine) startEvents(ctx context.Context, only string) error { +func (e *Engine) startEvents(ctx context.Context, only string, yieldToSchedule bool) error { pipelines, err := e.Store.ListPipelines(ctx) if err != nil { return err @@ -182,7 +182,9 @@ func (e *Engine) startEvents(ctx context.Context, only string) error { if err != nil { continue } - if sp.Schedule != nil && p.NextDueAt != nil && !e.now().Before(*p.NextDueAt) && !eventBeforeSchedule(ev, p) { + // Yield to an older due schedule only when this loop also runs + // schedules; otherwise nothing would ever dispatch it. + if yieldToSchedule && sp.Schedule != nil && p.NextDueAt != nil && !e.now().Before(*p.NextDueAt) && !eventBeforeSchedule(ev, p) { continue // the due schedule is older; it goes first } ids := make([]string, len(sp.Stages)) diff --git a/internal/engine/events_test.go b/internal/engine/events_test.go index 0b0cc97..4cf99b9 100644 --- a/internal/engine/events_test.go +++ b/internal/engine/events_test.go @@ -288,7 +288,7 @@ func TestPayloadDeletedAtTheMomentTheRunSucceeds(t *testing.T) { t.Fatal(err) } ctx := context.Background() - if err := h.eng.startEvents(ctx, ""); err != nil { + if err := h.eng.startEvents(ctx, "", true); err != nil { t.Fatal(err) } deadline := time.Now().Add(10 * time.Second) @@ -531,7 +531,7 @@ func TestDueScheduleDoesNotStarveQueuedEvents(t *testing.T) { if err := h.eng.schedule(ctx); err != nil { t.Fatal(err) } - if err := h.eng.startEvents(ctx, ""); err != nil { + if err := h.eng.startEvents(ctx, "", true); err != nil { t.Fatal(err) } runs, _ := h.eng.Runs(ctx, "ev", 10) @@ -581,3 +581,41 @@ func TestSweepOpensCandidatesWritable(t *testing.T) { t.Fatal("a file the sweep cannot open writable was silently treated as in use") } } + +// An events-only loop (trigger --wait without serve) cannot dispatch +// schedules, so it must not yield an event to a due schedule. +func TestEventsOnlyLoopDoesNotWaitForASchedule(t *testing.T) { + h := newHarness(t, nil) + // Enabled with a schedule: due at once (first_due = now). + h.add("name: ev\nschedule: {interval: 1h}\nevents: {max_queued: 3}\nstages:\n - {id: a, cmd: x}\n", true) + h.now = h.now.Add(time.Minute) // the event is newer than the due time + res, err := h.trigger("e", "x") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := h.eng.Loop(ctx, LoopOptions{Events: true, UntilIdle: true}); err != nil { + t.Fatal(err) + } + ev, _ := h.eng.Store.GetEvent(context.Background(), res.Event.ID) + if ev.State != store.EventStarted { + t.Fatalf("event %s; an events-only loop stalled behind a schedule it cannot run", ev.State) + } +} + +// A loop whose goal is already met dispatches nothing. +func TestLoopWithDoneAlreadyTrueStartsNothing(t *testing.T) { + h := newHarness(t, nil) + h.add(eventPipeline, true) + if _, err := h.trigger("newer", "x"); err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err := h.eng.Loop(ctx, LoopOptions{Events: true, UntilIdle: true, Done: func(context.Context) bool { return true }}); err != nil { + t.Fatal(err) + } + if n := h.ex.callCount("a"); n != 0 { + t.Fatalf("started %d attempts although the caller's goal was already met", n) + } +}