Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,8 @@ Nothing is installed system-wide. Existing Pitot services and data directories a

The default topology is 8 workers per process at 2,500 items/s per worker with 1,000-item OTLP batches. A mixed step launches the same topology independently for traces, logs, and metrics. Without read load, the sustained proof reruns the mixed topology with the highest **accepted** throughput. With `--query-rate`, it chooses the highest-throughput topology that also completed at least 95% of the requested reads with no errors or scheduler shedding. If no sweep point qualifies, it repeats the best read-completion point and fails the sustained-read quality gate rather than publishing a write-only peak as mixed capacity. Both reports record the selected sustained process count per signal. The wider publish sweep adds independent exporters to get past telemetrygen's per-process limits; see [the source analysis](docs/telemetrygen.md).

Read cost grows with retained data, so reads measured against a nearly empty server are optimistic. `--prefill 10m` runs an ingest-only mixed phase after warmup at `--prefill-processes` per signal (default 16, using the preset's workers and rate). The runner then waits up to 10 minutes for the exported ingest queue and rollup backlog gauges to read zero in two consecutive scrapes. Prefill rows are excluded from totals, peaks and topology selection. The `prefill-settled` gate fails when the backlog does not drain or the gauges are missing.

For publication runs, dedicate the default ccx33 to Pitot and use the separate ccx33 for generators, with three repetitions. Run ingest-only and read-under-write as two groups sharing the same fresh pair of VMs; see [the publication commands](docs/publishing.md#publication-runs). For example:

```sh
Expand Down
4 changes: 4 additions & 0 deletions cmd/pitot-bench/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ func commonFlags(name string, args []string) (bench.Config, error) {
fs.IntVar(&cfg.WorkerOverride, "workers", 0, "override workers per telemetrygen process")
fs.Float64Var(&cfg.RateOverride, "rate", 0, "override items/s per telemetrygen worker")
fs.DurationVar(&cfg.DurationOverride, "duration", 0, "override every measurement duration (use for diagnostics)")
fs.DurationVar(&cfg.Prefill, "prefill", 0, "ingest-only mixed load run after warmup so measured phases start on a populated server")
fs.IntVar(&cfg.PrefillProcesses, "prefill-processes", 16, "process count per signal for the prefill phase")
if err := fs.Parse(args); err != nil {
return cfg, err
}
Expand Down Expand Up @@ -189,6 +191,8 @@ func managed(ctx context.Context, args []string) error {
fs.IntVar(&cfg.WorkerOverride, "workers", 0, "override workers per process")
fs.Float64Var(&cfg.RateOverride, "rate", 0, "override items per worker per second")
fs.DurationVar(&cfg.DurationOverride, "duration", 0, "override measurement duration")
fs.DurationVar(&cfg.Prefill, "prefill", 0, "ingest-only prefill duration")
fs.IntVar(&cfg.PrefillProcesses, "prefill-processes", 16, "prefill process count per signal")
if err := fs.Parse(args); err != nil {
return err
}
Expand Down
3 changes: 3 additions & 0 deletions internal/bench/managed.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ type ManagedConfig struct {
WorkerOverride int
RateOverride float64
DurationOverride time.Duration
Prefill time.Duration
PrefillProcesses int
}

var setupTokenPattern = regexp.MustCompile(`setup_token=([^\s]+)`)
Expand Down Expand Up @@ -255,6 +257,7 @@ func managedRunnerConfig(managed ManagedConfig, httpAddr, dataDir string) Config
QueryRate: managed.QueryRate, SameHost: managed.GeneratorSSH == "", TelemetrygenVersion: managed.TelemetrygenVersion,
ProcessOverride: managed.ProcessOverride, WorkerOverride: managed.WorkerOverride,
RateOverride: managed.RateOverride, DurationOverride: managed.DurationOverride,
Prefill: managed.Prefill, PrefillProcesses: managed.PrefillProcesses,
}
cfg.GeneratorSSH = managed.GeneratorSSH
cfg.GeneratorSSHPort = managed.GeneratorSSHPort
Expand Down
12 changes: 12 additions & 0 deletions internal/bench/preset.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,18 @@ func Plan(cfg Config) ([]PhaseSpec, error) {
}
}
phases := []PhaseSpec{makePhase("warmup", "warmup", Mixed, s.warmup, 1, 1)}
if cfg.Prefill < 0 {
return nil, fmt.Errorf("prefill must not be negative")
}
if cfg.Prefill > 0 {
if cfg.PrefillProcesses <= 0 {
return nil, fmt.Errorf("prefill processes must be positive")
}
// Read cost scales with retained data, so a read gate measured on a
// nearly empty server is optimistic. Prefill populates the store
// without read load before any measured phase.
phases = append(phases, makePhase("prefill", "prefill", Mixed, cfg.Prefill, cfg.PrefillProcesses, 1))
}
for repetition := 1; repetition <= cfg.Repeat; repetition++ {
for _, signal := range Signals {
if cfg.SignalSelection == "mixed" {
Expand Down
57 changes: 57 additions & 0 deletions internal/bench/preset_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package bench

import (
"slices"
"strings"
"testing"
"time"
)
Expand Down Expand Up @@ -181,3 +182,59 @@ func TestValidationUsesPublishShape(t *testing.T) {
t.Fatal("validate early stop disabled")
}
}

func TestPlanPrefillRunsAfterWarmupWithoutReads(t *testing.T) {
plan, err := Plan(Config{Preset: "publish", Repeat: 1, BatchSize: 1000, QueryRate: 20, SignalSelection: "mixed", Prefill: 10 * time.Minute, PrefillProcesses: 16})
if err != nil {
t.Fatal(err)
}
if p := plan[1]; p.Kind != "prefill" || p.Signal != Mixed || p.Duration != 10*time.Minute || p.ProcessesPerSignal != 16 || p.QueryRate != 0 {
t.Fatalf("prefill phase: %+v", p)
}
if _, err := Plan(Config{Preset: "quick", Repeat: 1, BatchSize: 1000, Prefill: time.Minute}); err == nil {
t.Fatal("prefill without processes accepted")
}
if _, err := Plan(Config{Preset: "quick", Repeat: 1, BatchSize: 1000, Prefill: -time.Minute, PrefillProcesses: 1}); err == nil {
t.Fatal("negative prefill accepted")
}
off := settingsFrom(Config{Preset: "publish", PrefillProcesses: 16})
on := settingsFrom(Config{Preset: "publish", Prefill: time.Minute, PrefillProcesses: 16})
if off.PrefillProcesses != 0 || workloadHash(off, nil) == workloadHash(on, nil) {
t.Fatalf("prefill missing from workload identity: %+v %+v", off, on)
}
cfg := Config{Host: "root@h", Preset: "publish", Prefill: time.Minute, PrefillProcesses: 16}
if cmd := managedRemoteCommand("/s", cfg, "v", "a"); !strings.Contains(cmd, "'--prefill' '1m0s' '--prefill-processes' '16'") {
t.Fatalf("remote command: %s", cmd)
}
if cmd := reproductionRunCommand(cfg); !strings.Contains(cmd, "--prefill 1m0s --prefill-processes 16") {
t.Fatalf("reproduction: %s", cmd)
}
if got := managedRunnerConfig(ManagedConfig{Prefill: time.Minute, PrefillProcesses: 16}, "h:1", "d"); got.Prefill != time.Minute || got.PrefillProcesses != 16 {
t.Fatalf("managed config: %+v", got)
}
}

func TestSummarizeGatesPrefillSettling(t *testing.T) {
trial := func(kind string, unsettled bool) Trial {
return Trial{Spec: PhaseSpec{Name: kind, Kind: kind, Signal: Mixed}, Accepted: Counts{Spans: 100}, Unsettled: unsettled}
}
for _, unsettled := range []bool{false, true} {
report := Report{Trials: []Trial{trial("prefill", unsettled), trial("sustained", false)}}
s := summarize(report, 0, 0)
if s.Accepted.Spans != 100 {
t.Fatalf("prefill rows counted in totals: %v", s.Accepted.Spans)
}
var found bool
for _, g := range s.Gates {
if g.Name == "prefill-settled" {
found = true
if g.Passed == unsettled {
t.Fatalf("gate %+v for unsettled=%v", g, unsettled)
}
}
}
if !found || s.Passed == unsettled {
t.Fatalf("unsettled=%v passed=%v found=%v", unsettled, s.Passed, found)
}
}
}
2 changes: 1 addition & 1 deletion internal/bench/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -531,7 +531,7 @@ func managedRemoteCommand(stage string, cfg Config, version, artifact string) st
args = append(args, "--telemetrygen-version", cfg.TelemetrygenVersion)
args = append(args, "--provisioned-host", redactedHost(cfg.Host), "--host-description", cfg.HostDescription)
args = append(args, "--provisioned-ssh-port", fmt.Sprint(cfg.SSHPort), "--provisioned-ssh-identity", cfg.SSHIdentity)
args = append(args, "--processes", fmt.Sprint(cfg.ProcessOverride), "--workers", fmt.Sprint(cfg.WorkerOverride), "--rate", fmt.Sprint(cfg.RateOverride), "--duration", cfg.DurationOverride.String())
args = append(args, "--processes", fmt.Sprint(cfg.ProcessOverride), "--workers", fmt.Sprint(cfg.WorkerOverride), "--rate", fmt.Sprint(cfg.RateOverride), "--duration", cfg.DurationOverride.String(), "--prefill", cfg.Prefill.String(), "--prefill-processes", fmt.Sprint(cfg.PrefillProcesses))
binaries := []string{stage + "/pitot-bench", stage + "/pitot"}
prefix := ""
if cfg.GeneratorHost == "" {
Expand Down
12 changes: 12 additions & 0 deletions internal/bench/report.go
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,7 @@ func reproductionRunCommand(c Config) string {
if c.DurationOverride > 0 {
parts = append(parts, "--duration "+c.DurationOverride.String())
}
parts = append(parts, prefillArgs(c)...)
return strings.Join(parts, " ")
}
env := []string{"PITOT_BENCH_INGEST_TOKEN=…"}
Expand All @@ -271,9 +272,17 @@ func reproductionRunCommand(c Config) string {
env = append(env, "PITOT_BENCH_QUERY_COOKIE=…")
parts = append(parts, "--query-url "+shellQuote(c.QueryURL), "--query-rate "+fmt.Sprint(c.QueryRate))
}
parts = append(parts, prefillArgs(c)...)
return strings.Join(append(env, parts...), " ")
}

func prefillArgs(c Config) []string {
if c.Prefill <= 0 {
return nil
}
return []string{"--prefill " + c.Prefill.String(), "--prefill-processes " + fmt.Sprint(c.PrefillProcesses)}
}

func shellQuote(v string) string { return "'" + strings.ReplaceAll(v, "'", "'\\''") + "'" }
func phaseIcon(kind string) string {
if kind == "warmup" {
Expand All @@ -282,6 +291,9 @@ func phaseIcon(kind string) string {
if kind == "sustained" {
return "◆"
}
if kind == "prefill" {
return "▪"
}
return "◇"
}
func resultIcon(class string) string {
Expand Down
72 changes: 71 additions & 1 deletion internal/bench/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,13 @@ func (r *Runner) Run(ctx context.Context) (*Report, error) {
if err != nil {
return r.abandon(report, fmt.Errorf("phase %s: %w", spec.Name, err))
}
if spec.Kind == "prefill" {
trial.SettleSeconds, trial.Unsettled = r.settle(ctx)
if trial.Unsettled {
trial.ClassificationNotes = append(trial.ClassificationNotes, fmt.Sprintf("server backlog did not drain within %s after prefill", settleLimit))
}
fmt.Fprintf(r.Out, " settled in %.0fs (unsettled=%v)\n", trial.SettleSeconds, trial.Unsettled)
}
report.Trials = append(report.Trials, trial)
report.Settings.Methodology += r.readSummary
printTrialLine(r.Out, trial)
Expand Down Expand Up @@ -285,6 +292,9 @@ func (r *Runner) runPhase(ctx context.Context, spec PhaseSpec, previous []Trial)
trial.Read = queries.readEvidence()
}
trial.Classification, trial.ClassificationNotes = classify(trial, previous)
if spec.Kind == "prefill" {
trial.Classification = "prefill"
}
r.readSummary = queries.operationSummary(spec.Name)
if trial.Spec.QueryRate > 0 && !queryTargetMet(trial) {
trial.ClassificationNotes = append(trial.ClassificationNotes, "read qualification goals were not met: "+strings.Join(readGateFailures(trial), "; "))
Expand Down Expand Up @@ -585,7 +595,7 @@ func missingQueryKinds(t Trial) []string {
func summarize(report Report, cooldownRSS, cooldownHeap float64) Summary {
s := Summary{CooldownRSS: cooldownRSS, CooldownHeap: cooldownHeap, Passed: true}
for _, trial := range report.Trials {
if trial.Spec.Kind != "warmup" {
if trial.Spec.Kind != "warmup" && trial.Spec.Kind != "prefill" {
s.Accepted.Spans += trial.Accepted.Spans
s.Accepted.LogRecords += trial.Accepted.LogRecords
s.Accepted.MetricPoints += trial.Accepted.MetricPoints
Expand Down Expand Up @@ -656,6 +666,16 @@ func summarize(report Report, cooldownRSS, cooldownHeap float64) Summary {
}
s.Gates = append(s.Gates, Gate{"sustained-query-load", queryOK, detail})
}
for _, t := range report.Trials {
if t.Spec.Kind != "prefill" {
continue
}
detail := fmt.Sprintf("backlog drained %.0fs after prefill", t.SettleSeconds)
if t.Unsettled {
detail = fmt.Sprintf("backlog still present %.0fs after prefill", t.SettleSeconds)
}
s.Gates = append(s.Gates, Gate{"prefill-settled", !t.Unsettled, detail})
}
for _, gate := range s.Gates {
s.Passed = s.Passed && gate.Passed
}
Expand Down Expand Up @@ -685,6 +705,7 @@ func settingsFrom(c Config) Settings {
SampleInterval: c.SampleInterval, SampleSeconds: c.SampleInterval.Seconds(), GeneratorTimeout: c.GeneratorTimeout,
TimeoutSeconds: c.GeneratorTimeout.Seconds(), Cooldown: c.Cooldown, CooldownSeconds: c.Cooldown.Seconds(), DataDir: c.DataDir,
QueryURL: c.QueryURL, QueryRate: c.QueryRate, BatchSize: c.BatchSize, TraceChildSpans: 1,
PrefillSeconds: c.Prefill.Seconds(), PrefillProcesses: prefillProcesses(c),
Methodology: "Pitot server-side accepted-row counter deltas divided by measured wall time; telemetrygen emits one parent and one child span per trace tick. " +
"Split runs bracket each phase with generator-host /proc/stat reads over the generator SSH setup, including startup and export draining. " +
"Generator CPU cores = logical CPUs × (total tick delta − idle and iowait tick delta) / total tick delta; guest ticks are not counted twice. " +
Expand Down Expand Up @@ -857,3 +878,52 @@ func generatorProcessDetail(ok bool, trials []Trial) string {
}
return detail
}

func prefillProcesses(c Config) int {
if c.Prefill <= 0 {
return 0
}
return c.PrefillProcesses
}

// settleLimit bounds the post-prefill wait. A server that cannot drain its
// backlog in this time would carry prefill work into every measured phase.
var settleLimit, settlePoll = 10 * time.Minute, 5 * time.Second

// settle waits until the ingest queue and rollup catch-up backlog are exported
// and empty in two consecutive scrapes, so measured phases do not inherit
// prefill work. An absent gauge never counts as drained. Compaction backlog is
// not a condition: it includes underfilled groups and need not reach zero.
func (r *Runner) settle(ctx context.Context) (float64, bool) {
started := time.Now()
clean := 0
for time.Since(started) < settleLimit {
if snap, err := r.scrape(ctx); err == nil && drained(snap, "pitot_ingest_queue_depth") && drained(snap, "pitot_rollup_backlog_chunks") {
clean++
if clean == 2 {
return round(time.Since(started).Seconds(), 1), false
}
} else {
clean = 0
}
select {
case <-ctx.Done():
return round(time.Since(started).Seconds(), 1), true
case <-time.After(settlePoll):
}
}
return round(time.Since(started).Seconds(), 1), true
}

func drained(s *snapshot, name string) bool {
present := false
for _, v := range s.samples {
if v.name == name {
present = true
if v.value != 0 {
return false
}
}
}
return present
}
51 changes: 51 additions & 0 deletions internal/bench/settle_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package bench

import (
"context"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
)

func TestSettleRequiresTwoCleanScrapesOfExportedGauges(t *testing.T) {
defer func(limit, poll time.Duration) { settleLimit, settlePoll = limit, poll }(settleLimit, settlePoll)
settleLimit, settlePoll = 2*time.Second, time.Millisecond
drainedBody := "pitot_ingest_queue_depth{signal=\"spans\"} 0\npitot_rollup_backlog_chunks{rollup=\"service\"} 0\n"
for _, tc := range []struct {
name string
bodies []string
unsettled bool
scrapes int64
}{
{"drains after backlog", []string{"pitot_ingest_queue_depth{signal=\"spans\"} 3\npitot_rollup_backlog_chunks{rollup=\"service\"} 0\n", drainedBody, "pitot_rollup_backlog_chunks{rollup=\"service\"} 1\npitot_ingest_queue_depth{signal=\"spans\"} 0\n", drainedBody, drainedBody}, false, 5},
{"absent gauge never drains", []string{"pitot_ingest_queue_depth{signal=\"spans\"} 0\n"}, true, 0},
} {
t.Run(tc.name, func(t *testing.T) {
var n atomic.Int64
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
i := int(n.Add(1)) - 1
if i >= len(tc.bodies) {
i = len(tc.bodies) - 1
}
_, _ = w.Write([]byte(tc.bodies[i]))
}))
defer srv.Close()
r := &Runner{Config: Config{MetricsURL: srv.URL}, client: srv.Client()}
_, unsettled := r.settle(context.Background())
if unsettled != tc.unsettled {
t.Fatalf("unsettled=%v", unsettled)
}
if tc.scrapes > 0 && n.Load() != tc.scrapes {
t.Fatalf("scrapes=%d want %d", n.Load(), tc.scrapes)
}
})
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
r := &Runner{Config: Config{MetricsURL: "http://127.0.0.1:1"}, client: http.DefaultClient}
if _, unsettled := r.settle(ctx); !unsettled {
t.Fatal("canceled settle reported settled")
}
}
6 changes: 6 additions & 0 deletions internal/bench/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ type Config struct {
WorkerOverride int
RateOverride float64
DurationOverride time.Duration
Prefill time.Duration
PrefillProcesses int
BatchSize int
TelemetrygenVersion string
}
Expand Down Expand Up @@ -126,6 +128,8 @@ type Settings struct {
DataDir string `json:"data_dir,omitempty"`
QueryURL string `json:"query_url,omitempty"`
QueryRate float64 `json:"query_rate,omitempty"`
PrefillSeconds float64 `json:"prefill_seconds,omitempty"`
PrefillProcesses int `json:"prefill_processes,omitempty"`
BatchSize int `json:"batch_size"`
TraceChildSpans int `json:"trace_child_spans"`
Methodology string `json:"methodology"`
Expand Down Expand Up @@ -200,6 +204,8 @@ type GeneratorCPU struct {
}

type Trial struct {
SettleSeconds float64 `json:"settle_seconds,omitempty"`
Unsettled bool `json:"unsettled,omitempty"`
Read *ReadEvidence `json:"read_evidence,omitempty"`
Spec PhaseSpec `json:"settings"`
StartedAt time.Time `json:"started_at"`
Expand Down
Loading