From 3cadc4c01b3e1a51dfd30cdbd6a979c5c78e56e0 Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Wed, 2 Sep 2026 20:58:10 +0500 Subject: [PATCH] fix(alerts): the rules no longer evaluate in one herd OpenObserve snaps an aligned alert's next run to the previous interval boundary for its frequency, so with alignment on every one-minute rule evaluates at :00 and every ten-minute rule at :00, :10, :20 -- thirty-four alerts issuing their searches and persisting their group states in the same millisecond. Measured on 2026-09-02 on the live services host: the searches queued behind each other until some exceeded the PromQL load-data timeout, which is what raised alert_evaluation_failed sixteen times in two minutes; and the state writes queued behind SQLite's single writer until they were refused -- "could not persist group states ...: database is locked", 2386 times in one day, about 190 an hour, peaking on the ten-minute boundaries. A scheduler that cannot persist that it already notified notifies again on the next tick, which is why the channel repeated the same page every ten minutes for hours on 2026-09-01. Nothing in these rules depends on clock alignment: every expression takes its own range relative to evaluation time. A test refuses an aligned alert. Claude-Session: https://claude.ai/code/session_0128syXKxAGCfJGRDxUUNQXp --- internal/observabilityrules/openobserve.go | 21 ++++++++++++++- .../openobserve_subject_test.go | 26 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/internal/observabilityrules/openobserve.go b/internal/observabilityrules/openobserve.go index 7ce7a89e..7de0dfff 100644 --- a/internal/observabilityrules/openobserve.go +++ b/internal/observabilityrules/openobserve.go @@ -93,9 +93,28 @@ func RenderOpenObserve(bundle Bundle, destination string, enable bool) (OpenObse periodMinutes := max((rule.HoldSecs+59)/60, frequencyMinutes) streamType := "metrics" var query OpenObserveQuery + // AlignTime false: every alert keeps its own schedule instead of being + // snapped to the wall clock. + // + // OpenObserve aligns an alert's next run to the previous interval + // boundary for its frequency (TriggerCondition::get_aligned_next_trigger_time), + // so with alignment on, every one-minute rule evaluates at :00 and every + // ten-minute rule at :00, :10, :20 -- thirty-four alerts issuing their + // searches and persisting their group states in the same millisecond. + // Measured on 2026-09-02: the searches queued behind each other until + // some exceeded the PromQL load-data timeout (that is what raised + // alert_evaluation_failed), and the state writes queued behind the + // single SQLite writer until they were refused -- + // "could not persist group states ...: database is locked", 2386 times + // in one day. A scheduler that cannot persist that it already notified + // notifies again on the next tick, which is why the channel repeated + // the same page every ten minutes for hours. + // + // Nothing in these rules depends on clock alignment: every expression + // takes its own range relative to evaluation time. trigger := OpenObserveTrigger{ Period: periodMinutes, Frequency: frequencyMinutes, FrequencyType: "minutes", - Silence: silence, Timezone: "UTC", AlignTime: true, + Silence: silence, Timezone: "UTC", AlignTime: false, } switch rule.QueryLanguage { case "sql": diff --git a/internal/observabilityrules/openobserve_subject_test.go b/internal/observabilityrules/openobserve_subject_test.go index 8f88d3ec..0f216385 100644 --- a/internal/observabilityrules/openobserve_subject_test.go +++ b/internal/observabilityrules/openobserve_subject_test.go @@ -129,3 +129,29 @@ func TestNotificationBodySurvivesEncoding(t *testing.T) { t.Fatalf("placeholders did not survive encoding: %s", encoded) } } + +// Thirty-four alerts must not evaluate in the same millisecond. OpenObserve +// snaps an aligned alert's next run to its frequency boundary, so alignment +// puts every rule of a given frequency on the same tick: on 2026-09-02 the +// searches queued until some exceeded the PromQL load-data timeout and the +// group-state writes queued behind the single SQLite writer until they were +// refused 2386 times, and a scheduler that cannot persist that it notified +// notifies again. +func TestRenderedAlertsDoNotShareOneSchedulingTick(t *testing.T) { + bundle, err := Load("../../config/observability-rules.yaml") + if err != nil { + t.Fatal(err) + } + rendered, err := RenderOpenObserve(bundle, "fleet_oncall", true) + if err != nil { + t.Fatal(err) + } + if len(rendered.Alerts) < 20 { + t.Fatalf("only %d alerts rendered; the check proves nothing", len(rendered.Alerts)) + } + for _, alert := range rendered.Alerts { + if alert.TriggerCondition.AlignTime { + t.Fatalf("alert %q is aligned to the wall clock and will evaluate in the herd", alert.Name) + } + } +}