diff --git a/config/observability-rules.yaml b/config/observability-rules.yaml index e9a0994..c14de1f 100644 --- a/config/observability-rules.yaml +++ b/config/observability-rules.yaml @@ -2,6 +2,19 @@ schema_version: 2 backend: openobserve organization: default rules: + # Every rule that reads an OTel signal-event stream must aggregate the host + # out of the pipeline's own labels. Those streams carry a `start_time` label + # that changes with each counter run, so an unaggregated expression returns + # one series per run, not one per host: the sum below counted a single run + # instead of the hour, and the alert dispatched once per series. + # + # Measured on 2026-09-02: sixteen failed evaluations in two minutes arrived + # as three series of 4, 4 and 3 -- three identical pages for one incident, + # re-sent every eleven minutes for the hour the window remembers, while + # three failures spread across three runs would have summed to one each and + # never crossed the threshold at all. The same shape sits in + # audit_suppression_burst and kernel_workqueue_hog, where a counter that + # restarts hides half the increase. - id: alert_evaluation_failed severity: page query_language: promql @@ -32,7 +45,7 @@ rules: # one failure would have subtracted a post-restart minimum from a # pre-restart maximum and paged the on-call over nothing. Summing deltas # counts exactly and survives a restart. - expression: sum_over_time(gha_fleet_observability_signal_events{signal_class="alert_evaluation_failed",aggregation_temporality="AGGREGATION_TEMPORALITY_DELTA"}[1h]) + expression: sum by (host_name) (sum_over_time(gha_fleet_observability_signal_events{signal_class="alert_evaluation_failed",aggregation_temporality="AGGREGATION_TEMPORALITY_DELTA"}[1h])) operator: ">" threshold: 2 evaluation_seconds: 60 @@ -59,7 +72,7 @@ rules: # # A window with no events now returns an empty result, which is the honest # answer to "how many events", evaluates without error, and does not fire. - expression: max_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]) + expression: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m])) operator: ">" threshold: 30 # No outer subquery: evaluation and hold are equal, so the renderer emits @@ -412,7 +425,7 @@ rules: query_language: promql stream_name: gha_fleet_host_signal_events # Same `or vector(0)` removal as audit_suppression_burst; see there. - expression: max_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) + expression: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h])) operator: ">" threshold: 10 # No outer subquery: evaluation and hold are equal, so the renderer emits diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index b487478..65cbe44 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -117,8 +117,19 @@ func TestHostSignalSlowBurnsRemainVectorsForOpenObserveSubqueries(t *testing.T) if strings.Contains(alert.QueryCondition.PromQL, "min_over_time((max_over_time(") { t.Fatalf("%s still wraps its windowed delta in a subquery a quiet window cannot satisfy: %s", id, alert.QueryCondition.PromQL) } - if !strings.HasPrefix(withoutSubject(alert.QueryCondition.PromQL), "max_over_time(") { - t.Fatalf("%s is not the plain windowed delta: %s", id, alert.QueryCondition.PromQL) + // The windowed delta stays the whole statement, and it is summed + // by host_name: the signal-event streams carry a start_time label + // that changes with each counter run, so an unaggregated delta + // measures one run instead of the host and splits the alert into + // one dispatch per run (2026-09-02: sixteen failures arrived as + // three series of 4, 4 and 3). Summing by host keeps the subject + // the notification prints and cannot collapse it to a scalar. + body := withoutSubject(alert.QueryCondition.PromQL) + if !strings.HasPrefix(body, "sum by (host_name) (max_over_time(") { + t.Fatalf("%s is not the windowed delta summed by host: %s", id, alert.QueryCondition.PromQL) + } + if !strings.Contains(body, "- min_over_time(") { + t.Fatalf("%s lost the min_over_time half of its delta: %s", id, alert.QueryCondition.PromQL) } } if !found { @@ -543,3 +554,29 @@ func TestEveryPromQLAlertCarriesASubjectLabel(t *testing.T) { t.Fatalf("expected both grouped and fleet-wide alerts, got %d grouped and %d fleet", grouped, scalar) } } + +// Every rule that reads an OTel signal-event stream must aggregate by +// host_name. Those streams carry start_time, flag and instrumentation +// labels that change with each counter run, so an unaggregated expression +// returns one series per run: the count is wrong and the alert dispatches +// once per series. Found on 2026-09-02 when one incident of sixteen failed +// evaluations arrived as three identical pages. +func TestSignalEventRulesAggregateThePipelineLabelsAway(t *testing.T) { + bundle, err := Load("../../config/observability-rules.yaml") + if err != nil { + t.Fatal(err) + } + checked := 0 + for _, rule := range bundle.Rules { + if !strings.Contains(rule.Expression, "signal_events") { + continue + } + checked++ + if !strings.Contains(rule.Expression, "by (host_name)") { + t.Fatalf("rule %q reads a signal-event stream without aggregating by host_name: %s", rule.ID, rule.Expression) + } + } + if checked < 3 { + t.Fatalf("only %d signal-event rules were checked; the walk proves nothing", checked) + } +}