From a9507db8bd4690ae0032c04c49cddd84248e90cd Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Wed, 2 Sep 2026 20:00:58 +0500 Subject: [PATCH] fix(alerts): the signal-event rules count the host, not the data point The OTel signal-event streams carry a start_time label that changes with each counter run, so an expression that does not aggregate returns one series per run rather than one per host. Three rules read those streams and none aggregated. Measured on 2026-09-02: sixteen failed alert evaluations in two minutes reached the channel as three series of 4, 4 and 3 -- three identical pages for one incident, re-sent every eleven minutes for the hour the window remembers. The same arithmetic silently under-counts: three failures spread across three runs sum to one each and never cross the threshold, and in audit_suppression_burst and kernel_workqueue_hog a counter that restarts hides half its increase. All three now sum by (host_name), which is also the label the notification prints as its subject. A walking test refuses a signal-event rule that does not aggregate, and the sustained-delta test asserts the property rather than the literal prefix it used to match. Claude-Session: https://claude.ai/code/session_0128syXKxAGCfJGRDxUUNQXp --- config/observability-rules.yaml | 19 +++++++++-- internal/observabilityrules/rules_test.go | 41 +++++++++++++++++++++-- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/config/observability-rules.yaml b/config/observability-rules.yaml index e9a0994..c14de1f 100644 --- a/config/observability-rules.yaml +++ b/config/observability-rules.yaml @@ -2,6 +2,19 @@ schema_version: 2 backend: openobserve organization: default rules: + # Every rule that reads an OTel signal-event stream must aggregate the host + # out of the pipeline's own labels. Those streams carry a `start_time` label + # that changes with each counter run, so an unaggregated expression returns + # one series per run, not one per host: the sum below counted a single run + # instead of the hour, and the alert dispatched once per series. + # + # Measured on 2026-09-02: sixteen failed evaluations in two minutes arrived + # as three series of 4, 4 and 3 -- three identical pages for one incident, + # re-sent every eleven minutes for the hour the window remembers, while + # three failures spread across three runs would have summed to one each and + # never crossed the threshold at all. The same shape sits in + # audit_suppression_burst and kernel_workqueue_hog, where a counter that + # restarts hides half the increase. - id: alert_evaluation_failed severity: page query_language: promql @@ -32,7 +45,7 @@ rules: # one failure would have subtracted a post-restart minimum from a # pre-restart maximum and paged the on-call over nothing. Summing deltas # counts exactly and survives a restart. - expression: sum_over_time(gha_fleet_observability_signal_events{signal_class="alert_evaluation_failed",aggregation_temporality="AGGREGATION_TEMPORALITY_DELTA"}[1h]) + expression: sum by (host_name) (sum_over_time(gha_fleet_observability_signal_events{signal_class="alert_evaluation_failed",aggregation_temporality="AGGREGATION_TEMPORALITY_DELTA"}[1h])) operator: ">" threshold: 2 evaluation_seconds: 60 @@ -59,7 +72,7 @@ rules: # # A window with no events now returns an empty result, which is the honest # answer to "how many events", evaluates without error, and does not fire. - expression: max_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]) + expression: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m])) operator: ">" threshold: 30 # No outer subquery: evaluation and hold are equal, so the renderer emits @@ -412,7 +425,7 @@ rules: query_language: promql stream_name: gha_fleet_host_signal_events # Same `or vector(0)` removal as audit_suppression_burst; see there. - expression: max_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) + expression: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h])) operator: ">" threshold: 10 # No outer subquery: evaluation and hold are equal, so the renderer emits diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index b487478..65cbe44 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -117,8 +117,19 @@ func TestHostSignalSlowBurnsRemainVectorsForOpenObserveSubqueries(t *testing.T) if strings.Contains(alert.QueryCondition.PromQL, "min_over_time((max_over_time(") { t.Fatalf("%s still wraps its windowed delta in a subquery a quiet window cannot satisfy: %s", id, alert.QueryCondition.PromQL) } - if !strings.HasPrefix(withoutSubject(alert.QueryCondition.PromQL), "max_over_time(") { - t.Fatalf("%s is not the plain windowed delta: %s", id, alert.QueryCondition.PromQL) + // The windowed delta stays the whole statement, and it is summed + // by host_name: the signal-event streams carry a start_time label + // that changes with each counter run, so an unaggregated delta + // measures one run instead of the host and splits the alert into + // one dispatch per run (2026-09-02: sixteen failures arrived as + // three series of 4, 4 and 3). Summing by host keeps the subject + // the notification prints and cannot collapse it to a scalar. + body := withoutSubject(alert.QueryCondition.PromQL) + if !strings.HasPrefix(body, "sum by (host_name) (max_over_time(") { + t.Fatalf("%s is not the windowed delta summed by host: %s", id, alert.QueryCondition.PromQL) + } + if !strings.Contains(body, "- min_over_time(") { + t.Fatalf("%s lost the min_over_time half of its delta: %s", id, alert.QueryCondition.PromQL) } } if !found { @@ -543,3 +554,29 @@ func TestEveryPromQLAlertCarriesASubjectLabel(t *testing.T) { t.Fatalf("expected both grouped and fleet-wide alerts, got %d grouped and %d fleet", grouped, scalar) } } + +// Every rule that reads an OTel signal-event stream must aggregate by +// host_name. Those streams carry start_time, flag and instrumentation +// labels that change with each counter run, so an unaggregated expression +// returns one series per run: the count is wrong and the alert dispatches +// once per series. Found on 2026-09-02 when one incident of sixteen failed +// evaluations arrived as three identical pages. +func TestSignalEventRulesAggregateThePipelineLabelsAway(t *testing.T) { + bundle, err := Load("../../config/observability-rules.yaml") + if err != nil { + t.Fatal(err) + } + checked := 0 + for _, rule := range bundle.Rules { + if !strings.Contains(rule.Expression, "signal_events") { + continue + } + checked++ + if !strings.Contains(rule.Expression, "by (host_name)") { + t.Fatalf("rule %q reads a signal-event stream without aggregating by host_name: %s", rule.ID, rule.Expression) + } + } + if checked < 3 { + t.Fatalf("only %d signal-event rules were checked; the walk proves nothing", checked) + } +}