feat(alerts): silences match an anomaly incident's service and environment

apps/ai/src/mcp/tools/get-incident-timeline.ts

@@ -125,6 +125,7 @@ export function registerGetIncidentTimelineTool(server: McpToolRegistrar) {
 					ruleName: i.ruleName,
 					groupKey: i.groupKey,
 					...(i.groupLabel === undefined ? undefined : { groupLabel: i.groupLabel }),
+					...(i.labels === undefined ? undefined : { labels: i.labels }),
 					signalType: i.signalType,
 					severity: i.severity,
 					status: i.status,

apps/ai/src/mcp/tools/list-alert-incidents.ts

@@ -123,6 +123,7 @@ export function registerListAlertIncidentsTool(server: McpToolRegistrar) {
 						? undefined
 						: { anomalyIncidentId: i.anomalyIncidentId }),
 					...(i.groupLabel === undefined ? undefined : { groupLabel: i.groupLabel }),
+					...(i.labels === undefined ? undefined : { labels: i.labels }),
 				})),
 				total: incidents.length,
 				openCount,

apps/api/src/routes/v2/alert-incidents.http.ts

@@ -48,6 +48,7 @@ const toV2Incident = (doc: AlertIncidentDocument): V2AlertIncident => ({
 	error_issue_id: doc.errorIssueId,
 	...(doc.anomalyIncidentId === undefined ? undefined : { anomaly_incident_id: doc.anomalyIncidentId }),
 	...(doc.groupLabel === undefined ? undefined : { group_label: doc.groupLabel }),
+	...(doc.labels === undefined ? undefined : { labels: doc.labels }),
 	silence_id: doc.silenceId,
 	silenced_at: timestampOrNull(doc.silencedAt),
 })

apps/api/src/routes/v2/alerts.http.test.ts

@@ -1209,6 +1209,42 @@ describe("v2 alerts over HTTP", () => {
 		await harness.dispose()
 	})
 
+	it("lists an incident's labels, and omits them when none are recorded", async () => {
+		const harness = makeHarness()
+		const key = await harness.bootstrapKey(["alerts:write"])
+		await createWebhookAndRule(harness, key.secret)
+		await executeSql(
+			harness.testDb,
+			`insert into alert_incidents (
+				id, org_id, rule_id, incident_key, rule_name, group_key, signal_type, severity, status,
+				comparator, threshold, first_triggered_at, last_triggered_at, resolved_at, dedupe_key,
+				labels_json, created_at, updated_at
+			)
+			select v.id, rules.org_id, rules.id, v.id, 'Checkout error rate', '__total__', 'error_rate',
+				'critical', v.status, 'gt', 0.05, now(), now(), v.resolved_at, v.id, v.labels, now(), now()
+			from (values
+				('00000000-0000-4000-8000-0000000000b1', 'open', null::timestamptz,
+					'{"service":"checkout","environment":"production"}'::jsonb),
+				('00000000-0000-4000-8000-0000000000b2', 'resolved', now(), null::jsonb)
+			) as v(id, status, resolved_at, labels)
+			cross join lateral (
+				select id, org_id from alert_rules where org_id = 'org_alerts_e2e' limit 1
+			) as rules`,
+		)
+		const list = await harness.request("GET", "/v2/alerts/incidents", key.secret)
+		expect(list.status).toBe(200)
+		const labelsByKey = Arr.map(list.body.data, (incident: { dedupe_key: string; labels?: unknown }) => [
+			incident.dedupe_key,
+			incident.labels,
+		])
+		expect(labelsByKey).toEqual(
+			expect.arrayContaining([
+				["00000000-0000-4000-8000-0000000000b1", { service: "checkout", environment: "production" }],
+				["00000000-0000-4000-8000-0000000000b2", undefined],
+			]),
+		)
+	})
+
 	it("acknowledges and unacknowledges an open incident and lists its events", async () => {
 		const harness = makeHarness()
 		const key = await harness.bootstrapKey(["alerts:write"])

apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json

@@ -390,6 +390,27 @@
           "id": {
             "$ref": "#/components/schemas/_maple_AlertIncidentId"
           },
+          "labels": {
+            "additionalProperties": false,
+            "description": "The service and environment the incident is about, for rules whose `group_key` does not say (today `anomaly` rules). Alert silences match `services` and `environments` against these first. Absent when not recorded.",
+            "properties": {
+              "environment": {
+                "description": "The deployment environment the incident is about.",
+                "examples": [
+                  "production"
+                ],
+                "type": "string"
+              },
+              "service": {
+                "description": "The service the incident is about.",
+                "examples": [
+                  "checkout"
+                ],
+                "type": "string"
+              }
+            },
+            "type": "object"
+          },
           "last_delivered_event_type": {
             "$ref": "#/components/schemas/_maple_AlertEventType",
             "description": "The most recent notification event delivered for this incident (`trigger`, `resolve`, `renotify`, `test`), or `null`."

packages/backend/src/services/alerts/AlertReadModelsService.ts

@@ -194,6 +194,7 @@ export const rowToIncidentDocument = (row: AlertIncidentRow) =>
 		errorIssueId: row.errorIssueId != null ? decodeErrorIssueIdSync(row.errorIssueId) : null,
 		...(row.signalType === "anomaly" ? { anomalyIncidentId: row.anomalyIncidentId } : undefined),
 		...(row.groupLabel === null ? undefined : { groupLabel: row.groupLabel }),
+		...(row.labelsJson === null ? undefined : { labels: row.labelsJson }),
 		silenceId: row.silenceId,
 		silencedAt: toIso(row.silencedAt),
 	})

packages/backend/src/services/alerts/AlertsService.anomaly-silences.test.ts

@@ -0,0 +1,218 @@
+import { afterEach, assert, describe, expect, it } from "@effect/vitest"
+import * as PG from "@maple-dev/effect-orm/postgres"
+import { AnomalyIncidents } from "@maple/db/tables"
+import {
+	AlertRuleUpsertRequest,
+	AlertSilenceCreateRequest,
+	AnomalyIncidentId,
+	IsoDateTimeString,
+	type AlertSilenceMatchers,
+} from "@maple/domain/http"
+import { Array as Arr, Clock, Duration, Effect, Layer, Schema } from "effect"
+import { TestClock } from "effect/testing"
+import { Database } from "@maple/backend/platform/DatabaseLive"
+import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite"
+import { AlertsService, type AlertsServiceApi } from "./AlertsService"
+import { AlertSilencesService } from "./AlertSilencesService"
+import { AnomalyAlertSource } from "./anomaly/AnomalyAlertSource"
+import {
+	DEFAULT_CLOCK_EPOCH_MS,
+	adminRoles,
+	asOrgId,
+	asUserId,
+	createWebhookDestination,
+	makeLayer,
+	makeWarehouseStub,
+} from "./alerts-service-test-support"
+
+const trackedDbs: TestDb[] = []
+afterEach(() => cleanupTestDbs(trackedDbs))
+
+const orgId = asOrgId("org_anomaly_silences")
+const userId = asUserId("user_anomaly_silences")
+const anomalyId = Schema.decodeSync(AnomalyIncidentId)("00000000-0000-4000-8000-00000000b001")
+const iso = Schema.decodeUnknownSync(IsoDateTimeString)
+
+const harness = () => {
+	const testDb = createTestDb(trackedDbs)
+	const sent: Array<string> = []
+	const fetchImpl: typeof fetch = async (_input, init) => {
+		sent.push(new Headers(init?.headers).get("x-maple-event-type") ?? "")
+		return new Response("ok", { status: 200 })
+	}
+	const layer = Layer.mergeAll(
+		makeLayer(testDb, makeWarehouseStub({}), { fetch: fetchImpl }),
+		AnomalyAlertSource.layer.pipe(Layer.provide(testDb.layer)),
+		testDb.layer,
+	)
+	return { sent, layer }
+}
+
+/** An open critical error-rate anomaly in checkout (production). */
+const insertCheckoutAnomaly = Effect.gen(function* () {
+	const database = yield* Database
+	const now = yield* Clock.currentTimeMillis
+	yield* database.execute((db) =>
+		db.run(
+			PG.insertInto(AnomalyIncidents).values({
+				id: anomalyId,
+				orgId,
+				detectorKey: "error_rate:production:checkout",
+				signalType: "error_rate",
+				serviceName: "checkout",
+				deploymentEnv: "production",
+				fingerprintHash: null,
+				errorIssueId: null,
+				status: "open",
+				severity: "critical",
+				openedValue: 0.042,
+				baselineMedian: 0.003,
+				baselineSigma: 0.001,
+				thresholdValue: 0.006,
+				lastObservedValue: 0.042,
+				firstTriggeredAt: now,
+				lastTriggeredAt: now,
+				resolvedAt: null,
+				resolveReason: null,
+				dedupeKey: `dedupe-${anomalyId}`,
+				createdAt: now,
+				updatedAt: now,
+			}),
+		),
+	)
+}).pipe(Effect.orDie)
+
+/**
+ * An anomaly rule over every service in production, whose critical tier pages a webhook. Only Maple
+ * creates the fully unscoped default, so the environment scope stands in for it here.
+ */
+const createUnscopedAnomalyRule = (alerts: AlertsServiceApi) =>
+	Effect.gen(function* () {
+		const destination = yield* createWebhookDestination(alerts, orgId, userId)
+		return yield* alerts.createRule(
+			orgId,
+			userId,
+			adminRoles,
+			new AlertRuleUpsertRequest({
+				name: "Production anomalies",
+				enabled: true,
+				signalType: "anomaly",
+				environments: ["production"],
+				comparator: "gte",
+				windowMinutes: 5,
+				thresholds: [
+					{ severity: "warning", value: 1, upper: null, recoveryValue: null, destinationIds: [] },
+					{
+						severity: "critical",
+						value: 2,
+						upper: null,
+						recoveryValue: null,
+						destinationIds: [destination.id],
+					},
+				],
+			}),
+		)
+	})
+
+const silence = (matchers: AlertSilenceMatchers) =>
+	Effect.gen(function* () {
+		const silences = yield* AlertSilencesService
+		const now = yield* Clock.currentTimeMillis
+		return yield* silences.create(
+			orgId,
+			userId,
+			adminRoles,
+			new AlertSilenceCreateRequest({
+				reason: "checkout maintenance",
+				matchers,
+				endsAt: iso(new Date(now + 60 * 60_000).toISOString()),
+			}),
+		)
+	})
+
+const ticks = (alerts: AlertsServiceApi, count: number) =>
+	Effect.forEach(
+		Arr.range(1, count),
+		() =>
+			Effect.gen(function* () {
+				yield* TestClock.adjust(Duration.minutes(1))
+				yield* alerts.runSchedulerTick()
+			}),
+		{ discard: true },
+	)
+
+const deliveries = (alerts: AlertsServiceApi) =>
+	alerts
+		.listDeliveryEvents(orgId)
+		.pipe(
+			Effect.map((page) =>
+				Arr.map(Arr.reverse(page.events), (event) => `${event.eventType}:${event.status}`),
+			),
+		)
+
+describe("alert silences on anomaly rules", () => {
+	it.effect("a service silence holds back the page for an anomaly in that service", () => {
+		const { sent, layer } = harness()
+		return Effect.gen(function* () {
+			yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS)
+			const alerts = yield* AlertsService
+			yield* createUnscopedAnomalyRule(alerts)
+			yield* insertCheckoutAnomaly
+			const created = yield* silence({ services: ["checkout"] })
+
+			yield* ticks(alerts, 1)
+
+			const [incident] = (yield* alerts.listIncidents(orgId)).incidents
+			expect(incident).toMatchObject({
+				status: "open",
+				severity: "critical",
+				labels: { service: "checkout", environment: "production" },
+				silenceId: created.id,
+			})
+			assert.isNotNull(incident?.silencedAt)
+			assert.deepStrictEqual(yield* deliveries(alerts), ["trigger:silenced"])
+			assert.deepStrictEqual(sent, [])
+		}).pipe(Effect.provide(layer))
+	})
+
+	it.effect("a silence on another service leaves the anomaly paging", () => {
+		const { sent, layer } = harness()
+		return Effect.gen(function* () {
+			yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS)
+			const alerts = yield* AlertsService
+			yield* createUnscopedAnomalyRule(alerts)
+			yield* insertCheckoutAnomaly
+			yield* silence({ services: ["payments"] })
+
+			yield* ticks(alerts, 1)
+
+			const [incident] = (yield* alerts.listIncidents(orgId)).incidents
+			assert.isNull(incident?.silenceId)
+			assert.deepStrictEqual(yield* deliveries(alerts), ["trigger:success"])
+			assert.deepStrictEqual(sent, ["trigger"])
+		}).pipe(Effect.provide(layer))
+	})
+
+	it.effect("sends the held-back page once when the silence ends while the anomaly is open", () => {
+		const { sent, layer } = harness()
+		return Effect.gen(function* () {
+			yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS)
+			const alerts = yield* AlertsService
+			const silences = yield* AlertSilencesService
+			yield* createUnscopedAnomalyRule(alerts)
+			yield* insertCheckoutAnomaly
+			const created = yield* silence({ services: ["checkout"] })
+			yield* ticks(alerts, 2)
+			assert.deepStrictEqual(sent, [])
+
+			yield* silences.end(orgId, userId, adminRoles, created.id)
+			yield* ticks(alerts, 3)
+
+			const [incident] = (yield* alerts.listIncidents(orgId)).incidents
+			assert.deepStrictEqual(yield* deliveries(alerts), ["trigger:silenced", "trigger:success"])
+			assert.deepStrictEqual(sent, ["trigger"])
+			assert.isNull(incident?.silencedAt)
+			assert.strictEqual(incident?.silenceId, created.id)
+		}).pipe(Effect.provide(layer))
+	})
+})