fix(alerts): the signal-event rules count the host, not the data point

config/observability-rules.yaml

@@ -2,6 +2,19 @@ schema_version: 2
 backend: openobserve
 organization: default
 rules:
+  # Every rule that reads an OTel signal-event stream must aggregate the host
+  # out of the pipeline's own labels. Those streams carry a `start_time` label
+  # that changes with each counter run, so an unaggregated expression returns
+  # one series per run, not one per host: the sum below counted a single run
+  # instead of the hour, and the alert dispatched once per series.
+  #
+  # Measured on 2026-09-02: sixteen failed evaluations in two minutes arrived
+  # as three series of 4, 4 and 3 -- three identical pages for one incident,
+  # re-sent every eleven minutes for the hour the window remembers, while
+  # three failures spread across three runs would have summed to one each and
+  # never crossed the threshold at all. The same shape sits in
+  # audit_suppression_burst and kernel_workqueue_hog, where a counter that
+  # restarts hides half the increase.
   - id: alert_evaluation_failed
     severity: page
     query_language: promql
@@ -32,7 +45,7 @@ rules:
     # one failure would have subtracted a post-restart minimum from a
     # pre-restart maximum and paged the on-call over nothing. Summing deltas
     # counts exactly and survives a restart.
-    expression: sum_over_time(gha_fleet_observability_signal_events{signal_class="alert_evaluation_failed",aggregation_temporality="AGGREGATION_TEMPORALITY_DELTA"}[1h])
+    expression: sum by (host_name) (sum_over_time(gha_fleet_observability_signal_events{signal_class="alert_evaluation_failed",aggregation_temporality="AGGREGATION_TEMPORALITY_DELTA"}[1h]))
     operator: ">"
     threshold: 2
     evaluation_seconds: 60
@@ -59,7 +72,7 @@ rules:
     #
     # A window with no events now returns an empty result, which is the honest
     # answer to "how many events", evaluates without error, and does not fire.
-    expression: max_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m])
+    expression: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]))
     operator: ">"
     threshold: 30
     # No outer subquery: evaluation and hold are equal, so the renderer emits
@@ -412,7 +425,7 @@ rules:
     query_language: promql
     stream_name: gha_fleet_host_signal_events
     # Same `or vector(0)` removal as audit_suppression_burst; see there.
-    expression: max_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h])
+    expression: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
     operator: ">"
     threshold: 10
     # No outer subquery: evaluation and hold are equal, so the renderer emits

internal/observabilityrules/rules_test.go

@@ -117,8 +117,19 @@ func TestHostSignalSlowBurnsRemainVectorsForOpenObserveSubqueries(t *testing.T)
 			if strings.Contains(alert.QueryCondition.PromQL, "min_over_time((max_over_time(") {
 				t.Fatalf("%s still wraps its windowed delta in a subquery a quiet window cannot satisfy: %s", id, alert.QueryCondition.PromQL)
 			}
-			if !strings.HasPrefix(withoutSubject(alert.QueryCondition.PromQL), "max_over_time(") {
-				t.Fatalf("%s is not the plain windowed delta: %s", id, alert.QueryCondition.PromQL)
+			// The windowed delta stays the whole statement, and it is summed
+			// by host_name: the signal-event streams carry a start_time label
+			// that changes with each counter run, so an unaggregated delta
+			// measures one run instead of the host and splits the alert into
+			// one dispatch per run (2026-09-02: sixteen failures arrived as
+			// three series of 4, 4 and 3). Summing by host keeps the subject
+			// the notification prints and cannot collapse it to a scalar.
+			body := withoutSubject(alert.QueryCondition.PromQL)
+			if !strings.HasPrefix(body, "sum by (host_name) (max_over_time(") {
+				t.Fatalf("%s is not the windowed delta summed by host: %s", id, alert.QueryCondition.PromQL)
+			}
+			if !strings.Contains(body, "- min_over_time(") {
+				t.Fatalf("%s lost the min_over_time half of its delta: %s", id, alert.QueryCondition.PromQL)
 			}
 		}
 		if !found {
@@ -543,3 +554,29 @@ func TestEveryPromQLAlertCarriesASubjectLabel(t *testing.T) {
 		t.Fatalf("expected both grouped and fleet-wide alerts, got %d grouped and %d fleet", grouped, scalar)
 	}
 }
+
+// Every rule that reads an OTel signal-event stream must aggregate by
+// host_name. Those streams carry start_time, flag and instrumentation
+// labels that change with each counter run, so an unaggregated expression
+// returns one series per run: the count is wrong and the alert dispatches
+// once per series. Found on 2026-09-02 when one incident of sixteen failed
+// evaluations arrived as three identical pages.
+func TestSignalEventRulesAggregateThePipelineLabelsAway(t *testing.T) {
+	bundle, err := Load("../../config/observability-rules.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	checked := 0
+	for _, rule := range bundle.Rules {
+		if !strings.Contains(rule.Expression, "signal_events") {
+			continue
+		}
+		checked++
+		if !strings.Contains(rule.Expression, "by (host_name)") {
+			t.Fatalf("rule %q reads a signal-event stream without aggregating by host_name: %s", rule.ID, rule.Expression)
+		}
+	}
+	if checked < 3 {
+		t.Fatalf("only %d signal-event rules were checked; the walk proves nothing", checked)
+	}
+}