fix(alerts): the rules no longer evaluate in one herd

internal/observabilityrules/openobserve.go

@@ -93,9 +93,28 @@ func RenderOpenObserve(bundle Bundle, destination string, enable bool) (OpenObse
 		periodMinutes := max((rule.HoldSecs+59)/60, frequencyMinutes)
 		streamType := "metrics"
 		var query OpenObserveQuery
+		// AlignTime false: every alert keeps its own schedule instead of being
+		// snapped to the wall clock.
+		//
+		// OpenObserve aligns an alert's next run to the previous interval
+		// boundary for its frequency (TriggerCondition::get_aligned_next_trigger_time),
+		// so with alignment on, every one-minute rule evaluates at :00 and every
+		// ten-minute rule at :00, :10, :20 -- thirty-four alerts issuing their
+		// searches and persisting their group states in the same millisecond.
+		// Measured on 2026-09-02: the searches queued behind each other until
+		// some exceeded the PromQL load-data timeout (that is what raised
+		// alert_evaluation_failed), and the state writes queued behind the
+		// single SQLite writer until they were refused --
+		// "could not persist group states ...: database is locked", 2386 times
+		// in one day. A scheduler that cannot persist that it already notified
+		// notifies again on the next tick, which is why the channel repeated
+		// the same page every ten minutes for hours.
+		//
+		// Nothing in these rules depends on clock alignment: every expression
+		// takes its own range relative to evaluation time.
 		trigger := OpenObserveTrigger{
 			Period: periodMinutes, Frequency: frequencyMinutes, FrequencyType: "minutes",
-			Silence: silence, Timezone: "UTC", AlignTime: true,
+			Silence: silence, Timezone: "UTC", AlignTime: false,
 		}
 		switch rule.QueryLanguage {
 		case "sql":

internal/observabilityrules/openobserve_subject_test.go

@@ -129,3 +129,29 @@ func TestNotificationBodySurvivesEncoding(t *testing.T) {
 		t.Fatalf("placeholders did not survive encoding: %s", encoded)
 	}
 }
+
+// Thirty-four alerts must not evaluate in the same millisecond. OpenObserve
+// snaps an aligned alert's next run to its frequency boundary, so alignment
+// puts every rule of a given frequency on the same tick: on 2026-09-02 the
+// searches queued until some exceeded the PromQL load-data timeout and the
+// group-state writes queued behind the single SQLite writer until they were
+// refused 2386 times, and a scheduler that cannot persist that it notified
+// notifies again.
+func TestRenderedAlertsDoNotShareOneSchedulingTick(t *testing.T) {
+	bundle, err := Load("../../config/observability-rules.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	rendered, err := RenderOpenObserve(bundle, "fleet_oncall", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rendered.Alerts) < 20 {
+		t.Fatalf("only %d alerts rendered; the check proves nothing", len(rendered.Alerts))
+	}
+	for _, alert := range rendered.Alerts {
+		if alert.TriggerCondition.AlignTime {
+			t.Fatalf("alert %q is aligned to the wall clock and will evaluate in the herd", alert.Name)
+		}
+	}
+}