feat(alerts): escalation chains that page step by step until someone acknowledges

apps/ai/src/mcp/dispatcher.ts

@@ -12,6 +12,7 @@ import { AlertsService } from "@maple/backend/services/alerts/AlertsService"
 import { AlertReadModelsService } from "@maple/backend/services/alerts/AlertReadModelsService"
 import { AlertIncidentAckService } from "@maple/backend/services/alerts/AlertIncidentAckService"
 import { AlertRulesService } from "@maple/backend/services/alerts/AlertRulesService"
+import { EscalationChainsService } from "@maple/backend/services/alerts/EscalationChainsService"
 import { AlertSilencesService } from "@maple/backend/services/alerts/AlertSilencesService"
 
 import { DashboardPersistenceService } from "@maple/backend/services/dashboards/DashboardPersistenceService"
@@ -274,6 +275,7 @@ const McpRuntimeServicesLive = Layer.mergeAll(
 	AlertReadModelsService.layer,
 	AlertIncidentAckService.layer,
 	AlertRulesService.layer,
+	EscalationChainsService.layer,
 	AlertSilencesService.layer,
 	AlertsService.layer,
 	AuditLogService.layer,

apps/ai/src/mcp/lib/alert-rules.ts

@@ -140,10 +140,34 @@ export const toAlertRuleRow = (rule: AlertRuleDocument): typeof AlertRuleRow.Typ
 	evaluationIntervalMinutes: rule.evaluationIntervalMinutes,
 	slowedIntervalMinutes: rule.slowedIntervalMinutes,
 	destinationIds: [...rule.destinationIds],
+	...escalationChainsOf(rule),
 	createdAt: rule.createdAt,
 	updatedAt: rule.updatedAt,
 })
 
+/** The chains a rule's tiers start, keyed by severity; absent when no tier names one. */
+const escalationChainsOf = (rule: Pick<AlertRuleDocument, "thresholds">) => {
+	const chains = Arr.flatMap(rule.thresholds, (tier) =>
+		tier.escalationChainId == null ? [] : [{ severity: tier.severity, chainId: tier.escalationChainId }],
+	)
+	return chains.length === 0 ? undefined : { escalationChains: chains }
+}
+
+/** e.g. `critical: Checkout on-call (3f2a…)`, or the bare id when the name was not looked up. */
+export const formatEscalationChains = (
+	chains: (typeof AlertRuleRow.Type)["escalationChains"],
+): string | undefined =>
+	chains === undefined
+		? undefined
+		: Arr.join(
+				Arr.map(chains, (c) =>
+					c.name == null
+						? `${c.severity}: ${c.chainId}`
+						: `${c.severity}: ${c.name} (${c.chainId})`,
+				),
+				"; ",
+			)
+
 interface RuleCadence {
 	readonly windowMinutes: number
 	readonly evaluationIntervalMinutes: number | null
@@ -329,6 +353,7 @@ export const renderRuleWrite = (
 				...ruleConditionFields(rule),
 				["Enabled", rule.enabled ? "Yes" : "No"],
 				["Destinations", rule.destinationIds.length],
+				["Escalation", formatEscalationChains(rule.escalationChains)],
 				[
 					"Environments",
 					rule.environments.length > 0 && rule.signalType !== "anomaly"

apps/ai/src/mcp/lib/escalation-chains.ts

@@ -0,0 +1,212 @@
+/** Pieces the escalation chain MCP tools share: rows, step text, step input and error mapping. */
+import { Array as Arr, Effect, Option, Schema, SchemaIssue } from "effect"
+import type {
+	AlertDestinationDocument,
+	AlertForbiddenError,
+	OrgId,
+	AlertPersistenceError,
+	AlertValidationError,
+	EscalationChainDocument,
+	EscalationChainInUseError,
+	EscalationChainNotFoundError,
+} from "@maple/domain/http"
+import { encodePublicId, PublicIdPrefixes } from "@maple/domain/http/v2"
+import type { EscalationChainRow } from "@maple/domain/mcp-outputs"
+import { AlertsService } from "@maple/backend/services/alerts/AlertsService"
+import { McpInvalidInputError } from "../tools/types"
+import { toMcpHttpError } from "./map-http-error"
+import * as P from "./params"
+import { doc } from "./tool-doc"
+
+type ChainRow = typeof EscalationChainRow.Type
+type DestinationRef = Pick<AlertDestinationDocument, "id" | "name" | "type">
+
+/** The steps a tool takes: each the destinations it notifies and the minutes it waits. */
+export const StepsInput = Schema.Array(
+	Schema.Struct({
+		destination_ids: Schema.Array(Schema.String).annotate({
+			description: "Destinations this step notifies, from list_alert_destinations (UUID or dest_…)",
+		}),
+		wait_minutes: Schema.Number.annotate({
+			description: "Minutes to wait for an acknowledgement before the next step, 1 to 1440",
+		}),
+	}),
+)
+
+export const stepsParameterDescription =
+	'Steps in order, each notifying its destinations then waiting for an ack before the next, e.g. [{"destination_ids":["dest_…"],"wait_minutes":10},{"destination_ids":["dest_…"],"wait_minutes":15}]. At most 10 steps of 10 destinations'
+
+/** The steps as the service schema takes them, destination ids decoded to internal ids. */
+export const toStepsRequest = (steps: typeof StepsInput.Type) =>
+	Effect.forEach(steps, (step) =>
+		Effect.map(
+			Effect.forEach(step.destination_ids, P.decodeResourceId("alertDestination", "steps")),
+			(ids) => ({
+				targets: Arr.map(ids, (destinationId) => ({ type: "destination" as const, destinationId })),
+				waitMinutes: step.wait_minutes,
+			}),
+		),
+	)
+
+/** Destination names for the rendered steps; a failed read only drops the names. */
+export const destinationDirectory = (orgId: OrgId) =>
+	Effect.gen(function* () {
+		const alerts = yield* AlertsService
+		return yield* alerts.listDestinations(orgId).pipe(
+			Effect.map((response): ReadonlyArray<DestinationRef> => response.destinations),
+			Effect.orElseSucceed((): ReadonlyArray<DestinationRef> => []),
+		)
+	})
+
+export const toEscalationChainRow = (
+	chain: EscalationChainDocument,
+	destinations: ReadonlyArray<DestinationRef>,
+): ChainRow => ({
+	id: chain.id,
+	publicId: encodePublicId(PublicIdPrefixes.escalationChain, chain.id),
+	name: chain.name,
+	description: chain.description,
+	steps: Arr.map(chain.steps, (step) => ({
+		targets: Arr.map(step.targets, (target) => {
+			const known = Option.getOrUndefined(
+				Arr.findFirst(destinations, (d) => d.id === target.destinationId),
+			)
+			return {
+				destinationId: target.destinationId,
+				destinationPublicId: encodePublicId(PublicIdPrefixes.alertDestination, target.destinationId),
+				name: known?.name ?? null,
+				type: known?.type ?? null,
+			}
+		}),
+		waitMinutes: step.waitMinutes,
+	})),
+	repeatCount: chain.repeatCount,
+	ruleIds: [...chain.ruleIds],
+	createdAt: chain.createdAt,
+	updatedAt: chain.updatedAt,
+})
+
+const formatWait = (minutes: number): string =>
+	minutes < 60
+		? `${minutes}m`
+		: minutes % 60 === 0
+			? `${minutes / 60}h`
+			: `${Math.floor(minutes / 60)}h ${minutes % 60}m`
+
+const targetLabel = (target: ChainRow["steps"][number]["targets"][number]): string =>
+	target.name === null
+		? `destination ${target.destinationPublicId}`
+		: target.type === null
+			? `${target.name} (${target.destinationPublicId})`
+			: `${target.name} (${target.type}, ${target.destinationPublicId})`
+
+/** One line per step, e.g. `1. notify On-call (pagerduty, dest_…), then wait 10m for an ack`. */
+const formatSteps = (chain: ChainRow): string =>
+	Arr.join(
+		Arr.map(
+			chain.steps,
+			(step, index) =>
+				`${index + 1}. notify ${Arr.join(Arr.map(step.targets, targetLabel), ", ")}, then wait ${formatWait(step.waitMinutes)} for an ack`,
+		),
+		"\n",
+	)
+
+const formatRepeats = (repeatCount: number): string =>
+	repeatCount === 0 ? "runs once" : `repeats ${repeatCount} more time${repeatCount === 1 ? "" : "s"}`
+
+export const chainBlocks = (chain: ChainRow) => [
+	doc.fields([
+		["ID", chain.id],
+		["REST id", chain.publicId],
+		["Name", chain.name],
+		["Description", chain.description ?? undefined],
+		["Passes", formatRepeats(chain.repeatCount)],
+		["Used by rules", chain.ruleIds.length > 0 ? Arr.join(chain.ruleIds, ", ") : "none"],
+	]),
+	doc.text(formatSteps(chain)),
+]
+
+export const chainNotFound = (chainId: string) =>
+	new McpInvalidInputError({
+		message: `Escalation chain not found: ${chainId}. Use list_escalation_chains to find chain IDs.`,
+		parameter: "chain_id",
+	})
+
+const formatIssues = SchemaIssue.makeFormatterStandardSchemaV1()
+
+/** A service request key as the tool spells it: a parameter, or a key inside `steps`. */
+const toolKey = (segment: PropertyKey): string => {
+	const key = String(segment)
+	switch (key) {
+		case "repeatCount":
+			return "repeat_count"
+		case "waitMinutes":
+			return "wait_minutes"
+		case "targets":
+			return "destination_ids"
+		default:
+			return key
+	}
+}
+
+/** A request that failed to decode, reported against the parameter its first issue is under. */
+export const invalidChainRequest = (error: Schema.SchemaError): McpInvalidInputError => {
+	const first = formatIssues(error.issue).issues[0]
+	const path = Arr.map(first?.path ?? [], (segment) =>
+		typeof segment === "object" && segment !== null ? segment.key : segment,
+	)
+	const [root] = path
+	const location = Arr.join(
+		Arr.map(path, (segment, index) =>
+			typeof segment === "number" ? `[${segment}]` : `${index === 0 ? "" : "."}${toolKey(segment)}`,
+		),
+		"",
+	)
+	const message = first?.message ?? String(error)
+	return new McpInvalidInputError({
+		message: location === "" ? `Invalid escalation chain: ${message}` : `${location}: ${message}`,
+		parameter: root === undefined ? "steps" : toolKey(root),
+	})
+}
+
+const failWith = (pipeName: string) => (error: AlertForbiddenError | AlertPersistenceError) =>
+	Effect.fail(toMcpHttpError(pipeName)(error))
+
+const invalidChain = (error: AlertValidationError) =>
+	Effect.fail(
+		new McpInvalidInputError({
+			message: `Invalid escalation chain: ${error.message}`,
+			parameter: "steps",
+		}),
+	)
+
+export const createChainErrors = {
+	"@maple/http/errors/AlertValidationError": invalidChain,
+	"@maple/http/errors/AlertForbiddenError": failWith("create_escalation_chain"),
+	"@maple/http/errors/AlertPersistenceError": failWith("create_escalation_chain"),
+}
+
+export const updateChainErrors = {
+	"@maple/http/errors/AlertValidationError": invalidChain,
+	"@maple/http/errors/EscalationChainNotFoundError": (error: EscalationChainNotFoundError) =>
+		Effect.fail(chainNotFound(error.chainId)),
+	"@maple/http/errors/AlertForbiddenError": failWith("update_escalation_chain"),
+	"@maple/http/errors/AlertPersistenceError": failWith("update_escalation_chain"),
+}
+
+export const deleteChainErrors = {
+	"@maple/http/errors/EscalationChainNotFoundError": (error: EscalationChainNotFoundError) =>
+		Effect.fail(chainNotFound(error.chainId)),
+	"@maple/http/errors/EscalationChainInUseError": (error: EscalationChainInUseError) =>
+		Effect.fail(
+			new McpInvalidInputError({
+				message: `Escalation chain is still used by ${Arr.join(
+					Arr.map(error.ruleIds, (id, index) => `"${error.ruleNames[index] ?? id}" (${id})`),
+					", ",
+				)}. Detach it first with update_alert_rule escalation_chain_id="none" and escalation_chain_tier set to the tier that names it, so another tier's chain stays. A tier that pages only through the chain needs a destination first: add_destination_ids adds to a one-tier rule or a two-tier rule's warning tier; a critical tier gains one by replacing the rule's thresholds in the v2 API or the dashboard.`,
+				parameter: "chain_id",
+			}),
+		),
+	"@maple/http/errors/AlertForbiddenError": failWith("delete_escalation_chain"),
+	"@maple/http/errors/AlertPersistenceError": failWith("delete_escalation_chain"),
+}

apps/ai/src/mcp/lib/params.ts

@@ -25,6 +25,7 @@ import {
 	AlertSilenceId,
 	DashboardId,
 	ErrorIssueId,
+	EscalationChainId,
 } from "@maple/domain/primitives"
 import { WarehouseTimeInput, type WarehouseDateTime } from "@maple/query-engine"
 import { McpInvalidInputError } from "../tools/types"
@@ -158,6 +159,11 @@ const RESOURCE_IDS = {
 		schema: AlertDestinationId,
 		noun: "alert destination",
 	},
+	escalationChain: {
+		prefix: PublicIdPrefixes.escalationChain,
+		schema: EscalationChainId,
+		noun: "escalation chain",
+	},
 	alertSilence: { prefix: PublicIdPrefixes.alertSilence, schema: AlertSilenceId, noun: "alert silence" },
 	dashboard: { prefix: PublicIdPrefixes.dashboard, schema: DashboardId, noun: "dashboard" },
 	errorIssue: { prefix: PublicIdPrefixes.errorIssue, schema: ErrorIssueId, noun: "error issue" },

apps/ai/src/mcp/resources/instructions.ts

@@ -117,6 +117,7 @@ first occurrences ──> (candidate) ──> issue in \`triage\`
 - Infrastructure (hosts, Kubernetes pods/nodes/workloads, Docker containers): list_infra (no \`kind\` = overview of what reports, hottest first) -> inspect_infra (CPU/memory over the window with the peak time, a node's or workload's pods). diagnose_service shows the workload and pods a service runs on. Connected integrations add kinds: cloudflare_zones / cloudflare_workers (requests, 5xx, latency; inspect a zone with section=traffic|breakdown|security|dns), railway_services and planetscale_databases; list_integrations says which are connected and healthy. Resource metrics are also chartable: create_dashboard group_by \`resource.host.name\` / \`resource.k8s.pod.name\` on source=metrics
 - Deploy verification: service_deployments (versions per service, newest vs previous) -> compare_periods around the rollout. Route usage over up to 90 days: route_usage. Silence check before calling an outage: ingest_freshness. Volume per service: ingest_usage. Database query shapes by volume: db_query_volume
 - Alert management: list_alert_rules / list_alert_destinations -> get_alert_rule -> preview_alert_rule (dry-run before saving; a window with no data is a blind rule, not a healthy one) -> create_alert_rule / update_alert_rule / delete_alert_rule -> list_alert_incidents
+- Paging step by step until someone acknowledges: list_escalation_chains / create_escalation_chain (steps of destinations + wait_minutes) / update_escalation_chain / delete_escalation_chain; attach to a rule with create_alert_rule / update_alert_rule escalation_chain_id. The rule's own destinations still fire at once.
 - Quieting alerts for a window (maintenance, a known outage): create_alert_silence (matchers + duration_minutes or ends_at) / list_alert_silences / end_alert_silence. Incidents still open; only notifications are held back.
 - Uptime: list_uptime_monitors is every http/tcp/dns check and heartbeat with its status and last check; a failed check's trace id goes to inspect_trace. Admins manage them with create_uptime_monitor / update_uptime_monitor (enabled=false pauses) / delete_uptime_monitor. A down monitor opens an alert incident with signal type \`uptime\`; its rule is managed only through the monitor.
 - SLOs: list_slos is every SLO with its SLI, budget left and worst burn rate, burning ones first -> inspect_slo (each 5m/30m/1h/6h burn window, which tier breaches, alerting) -> get_incident_timeline rule_id=<its alert rule>. Admins and engineers manage them with create_slo / update_slo (alerting_enabled=false silences it) / delete_slo. An SLO's burn-rate rule (signal type \`slo_burn_rate\`) is managed only through the SLO.