diff --git a/README.md b/README.md index d9acf4d..e0f1bcf 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,8 @@ Nothing is installed system-wide. Existing Pitot services and data directories a The default topology is 8 workers per process at 2,500 items/s per worker with 1,000-item OTLP batches. A mixed step launches the same topology independently for traces, logs, and metrics. Without read load, the sustained proof reruns the mixed topology with the highest **accepted** throughput. With `--query-rate`, it chooses the highest-throughput topology that also completed at least 95% of the requested reads with no errors or scheduler shedding. If no sweep point qualifies, it repeats the best read-completion point and fails the sustained-read quality gate rather than publishing a write-only peak as mixed capacity. Both reports record the selected sustained process count per signal. The wider publish sweep adds independent exporters to get past telemetrygen's per-process limits; see [the source analysis](docs/telemetrygen.md). +Read cost grows with retained data, so reads measured against a nearly empty server are optimistic. `--prefill 10m` runs an ingest-only mixed phase after warmup at `--prefill-processes` per signal (default 16, using the preset's workers and rate). The runner then waits up to 10 minutes for the exported ingest queue and rollup backlog gauges to read zero in two consecutive scrapes. Prefill rows are excluded from totals, peaks and topology selection. The `prefill-settled` gate fails when the backlog does not drain or the gauges are missing. + For publication runs, dedicate the default ccx33 to Pitot and use the separate ccx33 for generators, with three repetitions. Run ingest-only and read-under-write as two groups sharing the same fresh pair of VMs; see [the publication commands](docs/publishing.md#publication-runs). For example: ```sh diff --git a/cmd/pitot-bench/main.go b/cmd/pitot-bench/main.go index 9c33d10..108df14 100644 --- a/cmd/pitot-bench/main.go +++ b/cmd/pitot-bench/main.go @@ -104,6 +104,8 @@ func commonFlags(name string, args []string) (bench.Config, error) { fs.IntVar(&cfg.WorkerOverride, "workers", 0, "override workers per telemetrygen process") fs.Float64Var(&cfg.RateOverride, "rate", 0, "override items/s per telemetrygen worker") fs.DurationVar(&cfg.DurationOverride, "duration", 0, "override every measurement duration (use for diagnostics)") + fs.DurationVar(&cfg.Prefill, "prefill", 0, "ingest-only mixed load run after warmup so measured phases start on a populated server") + fs.IntVar(&cfg.PrefillProcesses, "prefill-processes", 16, "process count per signal for the prefill phase") if err := fs.Parse(args); err != nil { return cfg, err } @@ -189,6 +191,8 @@ func managed(ctx context.Context, args []string) error { fs.IntVar(&cfg.WorkerOverride, "workers", 0, "override workers per process") fs.Float64Var(&cfg.RateOverride, "rate", 0, "override items per worker per second") fs.DurationVar(&cfg.DurationOverride, "duration", 0, "override measurement duration") + fs.DurationVar(&cfg.Prefill, "prefill", 0, "ingest-only prefill duration") + fs.IntVar(&cfg.PrefillProcesses, "prefill-processes", 16, "prefill process count per signal") if err := fs.Parse(args); err != nil { return err } diff --git a/internal/bench/managed.go b/internal/bench/managed.go index 9cbe440..e65db02 100644 --- a/internal/bench/managed.go +++ b/internal/bench/managed.go @@ -53,6 +53,8 @@ type ManagedConfig struct { WorkerOverride int RateOverride float64 DurationOverride time.Duration + Prefill time.Duration + PrefillProcesses int } var setupTokenPattern = regexp.MustCompile(`setup_token=([^\s]+)`) @@ -255,6 +257,7 @@ func managedRunnerConfig(managed ManagedConfig, httpAddr, dataDir string) Config QueryRate: managed.QueryRate, SameHost: managed.GeneratorSSH == "", TelemetrygenVersion: managed.TelemetrygenVersion, ProcessOverride: managed.ProcessOverride, WorkerOverride: managed.WorkerOverride, RateOverride: managed.RateOverride, DurationOverride: managed.DurationOverride, + Prefill: managed.Prefill, PrefillProcesses: managed.PrefillProcesses, } cfg.GeneratorSSH = managed.GeneratorSSH cfg.GeneratorSSHPort = managed.GeneratorSSHPort diff --git a/internal/bench/preset.go b/internal/bench/preset.go index 8fc8a77..4b77775 100644 --- a/internal/bench/preset.go +++ b/internal/bench/preset.go @@ -67,6 +67,18 @@ func Plan(cfg Config) ([]PhaseSpec, error) { } } phases := []PhaseSpec{makePhase("warmup", "warmup", Mixed, s.warmup, 1, 1)} + if cfg.Prefill < 0 { + return nil, fmt.Errorf("prefill must not be negative") + } + if cfg.Prefill > 0 { + if cfg.PrefillProcesses <= 0 { + return nil, fmt.Errorf("prefill processes must be positive") + } + // Read cost scales with retained data, so a read gate measured on a + // nearly empty server is optimistic. Prefill populates the store + // without read load before any measured phase. + phases = append(phases, makePhase("prefill", "prefill", Mixed, cfg.Prefill, cfg.PrefillProcesses, 1)) + } for repetition := 1; repetition <= cfg.Repeat; repetition++ { for _, signal := range Signals { if cfg.SignalSelection == "mixed" { diff --git a/internal/bench/preset_test.go b/internal/bench/preset_test.go index 9b63fe5..6096618 100644 --- a/internal/bench/preset_test.go +++ b/internal/bench/preset_test.go @@ -2,6 +2,7 @@ package bench import ( "slices" + "strings" "testing" "time" ) @@ -181,3 +182,59 @@ func TestValidationUsesPublishShape(t *testing.T) { t.Fatal("validate early stop disabled") } } + +func TestPlanPrefillRunsAfterWarmupWithoutReads(t *testing.T) { + plan, err := Plan(Config{Preset: "publish", Repeat: 1, BatchSize: 1000, QueryRate: 20, SignalSelection: "mixed", Prefill: 10 * time.Minute, PrefillProcesses: 16}) + if err != nil { + t.Fatal(err) + } + if p := plan[1]; p.Kind != "prefill" || p.Signal != Mixed || p.Duration != 10*time.Minute || p.ProcessesPerSignal != 16 || p.QueryRate != 0 { + t.Fatalf("prefill phase: %+v", p) + } + if _, err := Plan(Config{Preset: "quick", Repeat: 1, BatchSize: 1000, Prefill: time.Minute}); err == nil { + t.Fatal("prefill without processes accepted") + } + if _, err := Plan(Config{Preset: "quick", Repeat: 1, BatchSize: 1000, Prefill: -time.Minute, PrefillProcesses: 1}); err == nil { + t.Fatal("negative prefill accepted") + } + off := settingsFrom(Config{Preset: "publish", PrefillProcesses: 16}) + on := settingsFrom(Config{Preset: "publish", Prefill: time.Minute, PrefillProcesses: 16}) + if off.PrefillProcesses != 0 || workloadHash(off, nil) == workloadHash(on, nil) { + t.Fatalf("prefill missing from workload identity: %+v %+v", off, on) + } + cfg := Config{Host: "root@h", Preset: "publish", Prefill: time.Minute, PrefillProcesses: 16} + if cmd := managedRemoteCommand("/s", cfg, "v", "a"); !strings.Contains(cmd, "'--prefill' '1m0s' '--prefill-processes' '16'") { + t.Fatalf("remote command: %s", cmd) + } + if cmd := reproductionRunCommand(cfg); !strings.Contains(cmd, "--prefill 1m0s --prefill-processes 16") { + t.Fatalf("reproduction: %s", cmd) + } + if got := managedRunnerConfig(ManagedConfig{Prefill: time.Minute, PrefillProcesses: 16}, "h:1", "d"); got.Prefill != time.Minute || got.PrefillProcesses != 16 { + t.Fatalf("managed config: %+v", got) + } +} + +func TestSummarizeGatesPrefillSettling(t *testing.T) { + trial := func(kind string, unsettled bool) Trial { + return Trial{Spec: PhaseSpec{Name: kind, Kind: kind, Signal: Mixed}, Accepted: Counts{Spans: 100}, Unsettled: unsettled} + } + for _, unsettled := range []bool{false, true} { + report := Report{Trials: []Trial{trial("prefill", unsettled), trial("sustained", false)}} + s := summarize(report, 0, 0) + if s.Accepted.Spans != 100 { + t.Fatalf("prefill rows counted in totals: %v", s.Accepted.Spans) + } + var found bool + for _, g := range s.Gates { + if g.Name == "prefill-settled" { + found = true + if g.Passed == unsettled { + t.Fatalf("gate %+v for unsettled=%v", g, unsettled) + } + } + } + if !found || s.Passed == unsettled { + t.Fatalf("unsettled=%v passed=%v found=%v", unsettled, s.Passed, found) + } + } +} diff --git a/internal/bench/remote.go b/internal/bench/remote.go index e7ec0bb..05f420f 100644 --- a/internal/bench/remote.go +++ b/internal/bench/remote.go @@ -531,7 +531,7 @@ func managedRemoteCommand(stage string, cfg Config, version, artifact string) st args = append(args, "--telemetrygen-version", cfg.TelemetrygenVersion) args = append(args, "--provisioned-host", redactedHost(cfg.Host), "--host-description", cfg.HostDescription) args = append(args, "--provisioned-ssh-port", fmt.Sprint(cfg.SSHPort), "--provisioned-ssh-identity", cfg.SSHIdentity) - args = append(args, "--processes", fmt.Sprint(cfg.ProcessOverride), "--workers", fmt.Sprint(cfg.WorkerOverride), "--rate", fmt.Sprint(cfg.RateOverride), "--duration", cfg.DurationOverride.String()) + args = append(args, "--processes", fmt.Sprint(cfg.ProcessOverride), "--workers", fmt.Sprint(cfg.WorkerOverride), "--rate", fmt.Sprint(cfg.RateOverride), "--duration", cfg.DurationOverride.String(), "--prefill", cfg.Prefill.String(), "--prefill-processes", fmt.Sprint(cfg.PrefillProcesses)) binaries := []string{stage + "/pitot-bench", stage + "/pitot"} prefix := "" if cfg.GeneratorHost == "" { diff --git a/internal/bench/report.go b/internal/bench/report.go index 3cb041b..67e9212 100644 --- a/internal/bench/report.go +++ b/internal/bench/report.go @@ -251,6 +251,7 @@ func reproductionRunCommand(c Config) string { if c.DurationOverride > 0 { parts = append(parts, "--duration "+c.DurationOverride.String()) } + parts = append(parts, prefillArgs(c)...) return strings.Join(parts, " ") } env := []string{"PITOT_BENCH_INGEST_TOKEN=…"} @@ -271,9 +272,17 @@ func reproductionRunCommand(c Config) string { env = append(env, "PITOT_BENCH_QUERY_COOKIE=…") parts = append(parts, "--query-url "+shellQuote(c.QueryURL), "--query-rate "+fmt.Sprint(c.QueryRate)) } + parts = append(parts, prefillArgs(c)...) return strings.Join(append(env, parts...), " ") } +func prefillArgs(c Config) []string { + if c.Prefill <= 0 { + return nil + } + return []string{"--prefill " + c.Prefill.String(), "--prefill-processes " + fmt.Sprint(c.PrefillProcesses)} +} + func shellQuote(v string) string { return "'" + strings.ReplaceAll(v, "'", "'\\''") + "'" } func phaseIcon(kind string) string { if kind == "warmup" { @@ -282,6 +291,9 @@ func phaseIcon(kind string) string { if kind == "sustained" { return "◆" } + if kind == "prefill" { + return "▪" + } return "◇" } func resultIcon(class string) string { diff --git a/internal/bench/runner.go b/internal/bench/runner.go index a4932b2..67b8400 100644 --- a/internal/bench/runner.go +++ b/internal/bench/runner.go @@ -101,6 +101,13 @@ func (r *Runner) Run(ctx context.Context) (*Report, error) { if err != nil { return r.abandon(report, fmt.Errorf("phase %s: %w", spec.Name, err)) } + if spec.Kind == "prefill" { + trial.SettleSeconds, trial.Unsettled = r.settle(ctx) + if trial.Unsettled { + trial.ClassificationNotes = append(trial.ClassificationNotes, fmt.Sprintf("server backlog did not drain within %s after prefill", settleLimit)) + } + fmt.Fprintf(r.Out, " settled in %.0fs (unsettled=%v)\n", trial.SettleSeconds, trial.Unsettled) + } report.Trials = append(report.Trials, trial) report.Settings.Methodology += r.readSummary printTrialLine(r.Out, trial) @@ -285,6 +292,9 @@ func (r *Runner) runPhase(ctx context.Context, spec PhaseSpec, previous []Trial) trial.Read = queries.readEvidence() } trial.Classification, trial.ClassificationNotes = classify(trial, previous) + if spec.Kind == "prefill" { + trial.Classification = "prefill" + } r.readSummary = queries.operationSummary(spec.Name) if trial.Spec.QueryRate > 0 && !queryTargetMet(trial) { trial.ClassificationNotes = append(trial.ClassificationNotes, "read qualification goals were not met: "+strings.Join(readGateFailures(trial), "; ")) @@ -585,7 +595,7 @@ func missingQueryKinds(t Trial) []string { func summarize(report Report, cooldownRSS, cooldownHeap float64) Summary { s := Summary{CooldownRSS: cooldownRSS, CooldownHeap: cooldownHeap, Passed: true} for _, trial := range report.Trials { - if trial.Spec.Kind != "warmup" { + if trial.Spec.Kind != "warmup" && trial.Spec.Kind != "prefill" { s.Accepted.Spans += trial.Accepted.Spans s.Accepted.LogRecords += trial.Accepted.LogRecords s.Accepted.MetricPoints += trial.Accepted.MetricPoints @@ -656,6 +666,16 @@ func summarize(report Report, cooldownRSS, cooldownHeap float64) Summary { } s.Gates = append(s.Gates, Gate{"sustained-query-load", queryOK, detail}) } + for _, t := range report.Trials { + if t.Spec.Kind != "prefill" { + continue + } + detail := fmt.Sprintf("backlog drained %.0fs after prefill", t.SettleSeconds) + if t.Unsettled { + detail = fmt.Sprintf("backlog still present %.0fs after prefill", t.SettleSeconds) + } + s.Gates = append(s.Gates, Gate{"prefill-settled", !t.Unsettled, detail}) + } for _, gate := range s.Gates { s.Passed = s.Passed && gate.Passed } @@ -685,6 +705,7 @@ func settingsFrom(c Config) Settings { SampleInterval: c.SampleInterval, SampleSeconds: c.SampleInterval.Seconds(), GeneratorTimeout: c.GeneratorTimeout, TimeoutSeconds: c.GeneratorTimeout.Seconds(), Cooldown: c.Cooldown, CooldownSeconds: c.Cooldown.Seconds(), DataDir: c.DataDir, QueryURL: c.QueryURL, QueryRate: c.QueryRate, BatchSize: c.BatchSize, TraceChildSpans: 1, + PrefillSeconds: c.Prefill.Seconds(), PrefillProcesses: prefillProcesses(c), Methodology: "Pitot server-side accepted-row counter deltas divided by measured wall time; telemetrygen emits one parent and one child span per trace tick. " + "Split runs bracket each phase with generator-host /proc/stat reads over the generator SSH setup, including startup and export draining. " + "Generator CPU cores = logical CPUs × (total tick delta − idle and iowait tick delta) / total tick delta; guest ticks are not counted twice. " + @@ -857,3 +878,52 @@ func generatorProcessDetail(ok bool, trials []Trial) string { } return detail } + +func prefillProcesses(c Config) int { + if c.Prefill <= 0 { + return 0 + } + return c.PrefillProcesses +} + +// settleLimit bounds the post-prefill wait. A server that cannot drain its +// backlog in this time would carry prefill work into every measured phase. +var settleLimit, settlePoll = 10 * time.Minute, 5 * time.Second + +// settle waits until the ingest queue and rollup catch-up backlog are exported +// and empty in two consecutive scrapes, so measured phases do not inherit +// prefill work. An absent gauge never counts as drained. Compaction backlog is +// not a condition: it includes underfilled groups and need not reach zero. +func (r *Runner) settle(ctx context.Context) (float64, bool) { + started := time.Now() + clean := 0 + for time.Since(started) < settleLimit { + if snap, err := r.scrape(ctx); err == nil && drained(snap, "pitot_ingest_queue_depth") && drained(snap, "pitot_rollup_backlog_chunks") { + clean++ + if clean == 2 { + return round(time.Since(started).Seconds(), 1), false + } + } else { + clean = 0 + } + select { + case <-ctx.Done(): + return round(time.Since(started).Seconds(), 1), true + case <-time.After(settlePoll): + } + } + return round(time.Since(started).Seconds(), 1), true +} + +func drained(s *snapshot, name string) bool { + present := false + for _, v := range s.samples { + if v.name == name { + present = true + if v.value != 0 { + return false + } + } + } + return present +} diff --git a/internal/bench/settle_test.go b/internal/bench/settle_test.go new file mode 100644 index 0000000..38b4f7c --- /dev/null +++ b/internal/bench/settle_test.go @@ -0,0 +1,51 @@ +package bench + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +func TestSettleRequiresTwoCleanScrapesOfExportedGauges(t *testing.T) { + defer func(limit, poll time.Duration) { settleLimit, settlePoll = limit, poll }(settleLimit, settlePoll) + settleLimit, settlePoll = 2*time.Second, time.Millisecond + drainedBody := "pitot_ingest_queue_depth{signal=\"spans\"} 0\npitot_rollup_backlog_chunks{rollup=\"service\"} 0\n" + for _, tc := range []struct { + name string + bodies []string + unsettled bool + scrapes int64 + }{ + {"drains after backlog", []string{"pitot_ingest_queue_depth{signal=\"spans\"} 3\npitot_rollup_backlog_chunks{rollup=\"service\"} 0\n", drainedBody, "pitot_rollup_backlog_chunks{rollup=\"service\"} 1\npitot_ingest_queue_depth{signal=\"spans\"} 0\n", drainedBody, drainedBody}, false, 5}, + {"absent gauge never drains", []string{"pitot_ingest_queue_depth{signal=\"spans\"} 0\n"}, true, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + var n atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + i := int(n.Add(1)) - 1 + if i >= len(tc.bodies) { + i = len(tc.bodies) - 1 + } + _, _ = w.Write([]byte(tc.bodies[i])) + })) + defer srv.Close() + r := &Runner{Config: Config{MetricsURL: srv.URL}, client: srv.Client()} + _, unsettled := r.settle(context.Background()) + if unsettled != tc.unsettled { + t.Fatalf("unsettled=%v", unsettled) + } + if tc.scrapes > 0 && n.Load() != tc.scrapes { + t.Fatalf("scrapes=%d want %d", n.Load(), tc.scrapes) + } + }) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + r := &Runner{Config: Config{MetricsURL: "http://127.0.0.1:1"}, client: http.DefaultClient} + if _, unsettled := r.settle(ctx); !unsettled { + t.Fatal("canceled settle reported settled") + } +} diff --git a/internal/bench/types.go b/internal/bench/types.go index 9dbf60a..e7e5c5b 100644 --- a/internal/bench/types.go +++ b/internal/bench/types.go @@ -56,6 +56,8 @@ type Config struct { WorkerOverride int RateOverride float64 DurationOverride time.Duration + Prefill time.Duration + PrefillProcesses int BatchSize int TelemetrygenVersion string } @@ -126,6 +128,8 @@ type Settings struct { DataDir string `json:"data_dir,omitempty"` QueryURL string `json:"query_url,omitempty"` QueryRate float64 `json:"query_rate,omitempty"` + PrefillSeconds float64 `json:"prefill_seconds,omitempty"` + PrefillProcesses int `json:"prefill_processes,omitempty"` BatchSize int `json:"batch_size"` TraceChildSpans int `json:"trace_child_spans"` Methodology string `json:"methodology"` @@ -200,6 +204,8 @@ type GeneratorCPU struct { } type Trial struct { + SettleSeconds float64 `json:"settle_seconds,omitempty"` + Unsettled bool `json:"unsettled,omitempty"` Read *ReadEvidence `json:"read_evidence,omitempty"` Spec PhaseSpec `json:"settings"` StartedAt time.Time `json:"started_at"`