diff --git a/config/observability-rules.yaml b/config/observability-rules.yaml index 0cb04a7..ad46b2e 100644 --- a/config/observability-rules.yaml +++ b/config/observability-rules.yaml @@ -162,7 +162,10 @@ rules: severity: page query_language: promql stream_name: gha_fleet_diagnostic_export_consecutive_failures - expression: max by (host_name) (gha_fleet_diagnostic_export_consecutive_failures) + # One exporter series exists, already labeled with its host. Aggregating it + # before the sustained range turns an empty input into a scalar on + # OpenObserve v1 and makes the subquery fail instead of reporting healthy. + expression: gha_fleet_diagnostic_export_consecutive_failures operator: ">" threshold: 0 evaluation_seconds: 30 diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index bc55bc4..4539152 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -68,7 +68,8 @@ func TestDiagnosticExporterPageRequiresSustainedFailure(t *testing.T) { } for _, alert := range rendered.Alerts { if alert.Name == rule.ID && (rule.HoldSecs != 180 || alert.TriggerCondition.Threshold != 1 || - !strings.HasPrefix(withoutSubject(alert.QueryCondition.PromQL), "min_over_time(")) { + !strings.HasPrefix(withoutSubject(alert.QueryCondition.PromQL), + "min_over_time((gha_fleet_diagnostic_export_consecutive_failures)")) { t.Fatalf("diagnostic rule=%#v alert=%#v", rule, alert) } }