diff --git a/internal/fleetobserve/metrics.go b/internal/fleetobserve/metrics.go index 2376a1f..59c979f 100644 --- a/internal/fleetobserve/metrics.go +++ b/internal/fleetobserve/metrics.go @@ -169,15 +169,14 @@ func RenderPrometheus(snapshot Snapshot, now time.Time, maxStaleness time.Durati labeledGaugeHeader(&output, "gha_fleet_queue_oldest_queued_wait_seconds_by_scale_set", "Longest wait since GitHub queued a still-queued intent, per configured scale set.") for _, scaleSet := range scaleSets { metric(&output, "gha_fleet_queue_oldest_queued_wait_seconds_by_scale_set", map[string]string{"scale_set": scaleSet}, float64(snapshot.Queue.OldestQueuedWaitSecondsByScaleSet[scaleSet])) - - gauge(&output, "gha_fleet_queue_started_wait_samples", "Jobs whose runner started within the completed-wait window.", float64(snapshot.Queue.StartedWaitSamples)) - gauge(&output, "gha_fleet_queue_started_wait_median_seconds", "Median wait, first queued to runner running, over jobs that started in the window.", float64(snapshot.Queue.StartedWaitMedianSeconds)) - gauge(&output, "gha_fleet_queue_started_wait_p90_seconds", "Ninetieth-percentile wait, first queued to runner running, over jobs that started in the window.", float64(snapshot.Queue.StartedWaitP90Seconds)) - gauge(&output, "gha_fleet_queue_started_wait_max_seconds", "Longest wait, first queued to runner running, over jobs that started in the window.", float64(snapshot.Queue.StartedWaitMaxSeconds)) - labeledGaugeHeader(&output, "gha_fleet_queue_started_wait_p90_seconds_by_scale_set", "Ninetieth-percentile completed wait per scale set.") - for _, scaleSet := range scaleSets { - metric(&output, "gha_fleet_queue_started_wait_p90_seconds_by_scale_set", map[string]string{"scale_set": scaleSet}, float64(snapshot.Queue.StartedWaitP90ByScaleSet[scaleSet])) - } + } + gauge(&output, "gha_fleet_queue_started_wait_samples", "Jobs whose runner started within the completed-wait window.", float64(snapshot.Queue.StartedWaitSamples)) + gauge(&output, "gha_fleet_queue_started_wait_median_seconds", "Median wait, first queued to runner running, over jobs that started in the window.", float64(snapshot.Queue.StartedWaitMedianSeconds)) + gauge(&output, "gha_fleet_queue_started_wait_p90_seconds", "Ninetieth-percentile wait, first queued to runner running, over jobs that started in the window.", float64(snapshot.Queue.StartedWaitP90Seconds)) + gauge(&output, "gha_fleet_queue_started_wait_max_seconds", "Longest wait, first queued to runner running, over jobs that started in the window.", float64(snapshot.Queue.StartedWaitMaxSeconds)) + labeledGaugeHeader(&output, "gha_fleet_queue_started_wait_p90_seconds_by_scale_set", "Ninetieth-percentile completed wait per scale set.") + for _, scaleSet := range scaleSets { + metric(&output, "gha_fleet_queue_started_wait_p90_seconds_by_scale_set", map[string]string{"scale_set": scaleSet}, float64(snapshot.Queue.StartedWaitP90ByScaleSet[scaleSet])) } gauge(&output, "gha_fleet_incus_visible_instances", "Instances visible in the restricted Incus project.", float64(snapshot.Incus.VisibleInstances)) gauge(&output, "gha_fleet_incus_visible_maintenance_instances", "Visible exact image builder or smoke instances; observable maintenance capacity, never GitHub job runners.", float64(snapshot.Incus.VisibleMaintenanceInstances)) diff --git a/internal/fleetobserve/metrics_test.go b/internal/fleetobserve/metrics_test.go index f98bc6d..2aaf09b 100644 --- a/internal/fleetobserve/metrics_test.go +++ b/internal/fleetobserve/metrics_test.go @@ -88,6 +88,24 @@ func TestRenderPrometheusIsDeterministicAndBounded(t *testing.T) { } } +func TestRenderPrometheusEmitsEverySeriesOnce(t *testing.T) { + metrics := RenderPrometheus(healthyCollector(t).Collect(t.Context()), observationTime, 45*time.Second) + seen := map[string]bool{} + for line := range strings.Lines(metrics) { + if strings.HasPrefix(line, "#") || strings.TrimSpace(line) == "" { + continue + } + series, _, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok { + t.Fatalf("metric line has no value: %q", line) + } + if seen[series] { + t.Fatalf("Prometheus series emitted more than once: %s", series) + } + seen[series] = true + } +} + func TestRenderPrometheusExposesBoundedDiagnosticConvergence(t *testing.T) { collector := healthyCollector(t) collector.Diagnostics = func(time.Time) (workerdiagnostics.SpoolStats, error) {