fix: patterns can be deleted; a deleted observation leaves no trace in patterns

CHANGELOG.md

@@ -47,6 +47,15 @@ a decision with visible consequences.
   goes, the removed entity's vector is queued for deletion (the startup dedup left it in Qdrant),
   and a malformed tags or attributes value no longer stops startup.
 
+- **Patterns can be deleted, and a deleted observation leaves them.** There was no way to delete
+  a pattern, and deleting an observation left its id, its lineage links and every pattern built
+  from it. `DELETE /pattern/{id}` now deletes a pattern of the calling agent (404 for another
+  agent's) with its lineage links and a `pattern` deletion receipt. Deleting an observation
+  removes its lineage links, its id in other observations and in patterns, and a pattern left
+  with no observation (with its receipt). Recovery replay of `observation` and `pattern`
+  receipts removes the same set from a restored backup. A pattern that still has other
+  observations keeps its text.
+
 - **Lineage stays inside one agent.** `POST /lineage/links`, `GET /lineage/{type}/{id}`,
   `DELETE /lineage/links/{id}` and an episode's `evidence_ids` now answer 404 for an object
   of another agent (the `X-Agent-Id` header), as for a missing one. Before, they read and

services/api/app/routes/observation.py

@@ -14,6 +14,7 @@
     ObservationUpdateRequest,
 )
 from app.services.agent_config_service import get_or_create_config
+from app.services.observation_forgetting import erase_observation_dependents
 from app.services.observation_lifecycle import apply_session_context, enforce_budget, find_duplicate
 from app.services.pattern_promotion import promote_from_observations
 from app.services.vector_outbox import enqueue_delete, enqueue_upsert, reconcile_one
@@ -189,6 +190,7 @@ def delete_observation(observation_id: str, agent_id: str = Depends(get_agent_id
         from app.models.deletion_receipt import record
         record(db, agent_id, "observation", row.id)
         enqueue_delete(db, "observation", row.id, agent_id=agent_id)
+        erase_observation_dependents(db, row.id)
         db.delete(row)
         db.commit()
         indexing = reconcile_one("observation", observation_id, sessions=SessionLocal)

services/api/app/routes/pattern.py

@@ -3,6 +3,7 @@
 
 from app.core.agent import get_agent_id, resolve_agent_id
 from app.core.db import SessionLocal
+from app.models.deletion_receipt import record
 from app.models.observation import Observation
 from app.models.pattern import Pattern
 from app.schemas.pattern import (
@@ -13,6 +14,7 @@
     PatternSearchRequest,
     PatternUpdateRequest,
 )
+from app.services.observation_forgetting import erase_pattern
 from app.services.pattern_promotion import (
     DEPRECATE_AT_CONTRADICTIONS,
     clamp_confidence,
@@ -79,6 +81,19 @@ def create_pattern(payload: PatternCreateRequest, header_agent_id: str = Depends
     finally:
         db.close()
 
+@router.delete("/{pattern_id}")
+def delete_pattern(pattern_id: str, agent_id: str = Depends(get_agent_id)):
+    """Delete a pattern and its lineage links; it leaves a content-free receipt for replay."""
+    db = SessionLocal()
+    try:
+        row = _get_owned_pattern(db, pattern_id, agent_id)
+        record(db, agent_id, "pattern", row.id)
+        erase_pattern(db, row.id)
+        db.commit()
+        return {"status": "ok"}
+    finally:
+        db.close()
+
 @router.post("/search")
 def search_patterns(payload: PatternSearchRequest, header_agent_id: str = Depends(get_agent_id)):
     agent_id = resolve_agent_id(header_agent_id, payload.agent_id)

services/api/app/services/deletion_recovery.py

@@ -11,12 +11,14 @@
 from app.models.memory import Memory
 from app.models.object_link import ObjectLink
 from app.models.observation import Observation
+from app.models.pattern import Pattern
 from app.models.session_transcript import SessionTranscript
 from app.services import conversation_memory, listener_forgetting
 from app.services.entity_forgetting import erase_dependents
+from app.services.observation_forgetting import erase_observation_dependents, erase_pattern
 from sqlalchemy import select
 
-MODELS = {"memory": Memory, "entity": Entity, "observation": Observation,
+MODELS = {"memory": Memory, "entity": Entity, "observation": Observation, "pattern": Pattern,
           "episode": EpisodeObject, "link": ObjectLink, "transcript": SessionTranscript}
 
 
@@ -124,6 +126,11 @@ def replay(receipts, target_sessions):
             record(db, agent_id, kind, identity)
             if kind == "entity":
                 erase_dependents(db, identity)
+            if kind == "observation":
+                erase_observation_dependents(db, identity)
+            if kind == "pattern":
+                erase_pattern(db, identity)
+                row = None
             if row is not None:
                 if kind == "episode":
                     links = list(db.scalars(select(ObjectLink).where(

services/api/app/services/entity_forgetting.py

@@ -2,50 +2,20 @@
 its id in observation and pattern id lists. Used by DELETE /entity/{id} and by recovery replay,
 so an entity deleted before a restore leaves nothing behind after it either."""
 
-import json
-
-from app.models.deletion_receipt import record
 from app.models.entity import EntityEdge, EntityEvent, EntityHistory
-from app.models.object_link import ObjectLink
-from sqlalchemy import delete, or_, select, text
+from app.services.lineage_forgetting import erase_links, strip_id
+from sqlalchemy import delete, or_
 
 ID_LISTS = (("observations", "entity_ids_json"), ("patterns", "applies_to_entity_ids_json"))
 
 
-def _without(value: str | None, entity_id: str) -> list | None:
-    """The stored id list without `entity_id`, or None when it does not hold it (or is not a list)."""
-    try:
-        ids = json.loads(value) if value else []
-    except (TypeError, ValueError):
-        return None
-    if not isinstance(ids, list) or entity_id not in ids:
-        return None
-    return [i for i in ids if i != entity_id]
-
-
 def erase_dependents(db, entity_id: str) -> None:
     """Delete everything that refers to `entity_id`, inside the caller's transaction (no commit).
     Each removed lineage link leaves a content-free `link` receipt, as other deletes do."""
     db.execute(delete(EntityHistory).where(EntityHistory.entity_id == entity_id))
     db.execute(delete(EntityEvent).where(EntityEvent.entity_id == entity_id))
     db.execute(delete(EntityEdge).where(
         or_(EntityEdge.from_entity_id == entity_id, EntityEdge.to_entity_id == entity_id)))
-
-    links = db.scalars(select(ObjectLink).where(or_(
-        (ObjectLink.source_type == "entity") & (ObjectLink.source_id == entity_id),
-        (ObjectLink.target_type == "entity") & (ObjectLink.target_id == entity_id),
-    ))).all()
-    for link in links:
-        record(db, "", "link", link.id)
-        db.delete(link)
-
+    erase_links(db, "entity", entity_id)
     for table, column in ID_LISTS:
-        rows = db.execute(
-            text(f"SELECT id, {column} AS ids FROM {table} WHERE {column} LIKE :pat"),
-            {"pat": f"%{entity_id}%"},
-        ).fetchall()
-        for row in rows:
-            kept = _without(row.ids, entity_id)
-            if kept is not None:
-                db.execute(text(f"UPDATE {table} SET {column} = :v WHERE id = :id"),
-                           {"v": json.dumps(kept), "id": row.id})
+        strip_id(db, table, column, entity_id)

services/api/app/services/lineage_forgetting.py

@@ -0,0 +1,47 @@
+"""Shared steps of a delete: drop the lineage links that name an object (each leaving a
+content-free `link` receipt) and take its id out of other rows' JSON id lists."""
+
+import json
+
+from app.models.deletion_receipt import record
+from app.models.object_link import ObjectLink
+from sqlalchemy import or_, select, text
+
+
+def erase_links(db, kind: str, object_id: str) -> None:
+    """Delete every lineage link with `kind`/`object_id` at either end (no commit)."""
+    links = db.scalars(select(ObjectLink).where(or_(
+        (ObjectLink.source_type == kind) & (ObjectLink.source_id == object_id),
+        (ObjectLink.target_type == kind) & (ObjectLink.target_id == object_id),
+    ))).all()
+    for link in links:
+        record(db, "", "link", link.id)
+        db.delete(link)
+
+
+def _without(value: str | None, object_id: str) -> list | None:
+    """The stored id list without `object_id`, or None when it does not hold it (or is not a list)."""
+    try:
+        ids = json.loads(value) if value else []
+    except (TypeError, ValueError):
+        return None
+    if not isinstance(ids, list) or object_id not in ids:
+        return None
+    return [i for i in ids if i != object_id]
+
+
+def strip_id(db, table: str, column: str, object_id: str) -> list[tuple[str, list]]:
+    """Remove `object_id` from `table.column` (a JSON id list) in every row that holds it.
+    Returns (row id, ids left) for each changed row. `table` and `column` are constants."""
+    rows = db.execute(
+        text(f"SELECT id, {column} AS ids FROM {table} WHERE {column} LIKE :pat"),
+        {"pat": f"%{object_id}%"},
+    ).fetchall()
+    changed = []
+    for row in rows:
+        kept = _without(row.ids, object_id)
+        if kept is not None:
+            db.execute(text(f"UPDATE {table} SET {column} = :v WHERE id = :id"),
+                       {"v": json.dumps(kept), "id": row.id})
+            changed.append((row.id, kept))
+    return changed

services/api/app/services/observation_forgetting.py

@@ -0,0 +1,31 @@
+"""What goes with a deleted observation or pattern. Used by the DELETE routes and by recovery
+replay, so a delete made before a restore leaves nothing behind after it either.
+
+A pattern is built from observations: when the last observation it names is deleted, the
+pattern goes too (with its own receipt), so its derived text does not outlive its evidence.
+A pattern that still has other observations keeps its text."""
+
+from app.models.deletion_receipt import record
+from app.models.pattern import Pattern
+from app.services.lineage_forgetting import erase_links, strip_id
+
+
+def erase_pattern(db, pattern_id: str) -> None:
+    """Delete the pattern's lineage links and the row itself, if present (no commit).
+    The caller records the pattern's receipt."""
+    erase_links(db, "pattern", pattern_id)
+    row = db.get(Pattern, pattern_id)
+    if row is not None:
+        db.delete(row)
+
+
+def erase_observation_dependents(db, observation_id: str) -> None:
+    """Delete what refers to `observation_id` (no commit): its lineage links, its id in other
+    observations' related ids, and its id in patterns; a pattern left with no observation goes."""
+    erase_links(db, "observation", observation_id)
+    strip_id(db, "observations", "related_observation_ids_json", observation_id)
+    for pattern_id, left in strip_id(db, "patterns", "observation_ids_json", observation_id):
+        if not left:
+            pattern = db.get(Pattern, pattern_id)
+            record(db, pattern.agent_id if pattern else "", "pattern", pattern_id)
+            erase_pattern(db, pattern_id)

services/api/tests/test_pattern_forgetting.py

@@ -0,0 +1,93 @@
+"""A pattern can be deleted, and a deleted observation does not live on in patterns: its id
+leaves them, its lineage links go, and a pattern left with no observation goes with it. A
+restored backup replays the same deletes."""
+
+from datetime import UTC, datetime
+
+from app.core.db import Base
+from app.models.object_link import ObjectLink
+from app.models.observation import Observation
+from app.models.pattern import Pattern
+from app.services import deletion_recovery as recovery
+from memory_stack import stack as stack
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import sessionmaker
+
+WHEN = datetime(2026, 1, 2, tzinfo=UTC)
+
+
+def _seed(db):
+    db.add_all([
+        Observation(id="obs_1", agent_id="owner", session_id="s", observed_at=WHEN, signal_type="mood",
+                    description="tired after calls with Maya"),
+        Observation(id="obs_2", agent_id="owner", session_id="s", observed_at=WHEN, signal_type="mood",
+                    description="tired on Mondays", related_observation_ids_json='["obs_1"]'),
+        Pattern(id="pat_only", agent_id="owner", pattern_name="calls with Maya",
+                description="tired after calls with Maya", observation_ids_json='["obs_1"]'),
+        Pattern(id="pat_both", agent_id="owner", pattern_name="tired", observation_ids_json='["obs_1", "obs_2"]'),
+        Pattern(id="pat_other", agent_id="owner", pattern_name="other", observation_ids_json='["obs_2"]'),
+        ObjectLink(id="lnk_1", source_type="observation", source_id="obs_1",
+                   target_type="pattern", target_id="pat_both", relationship="supports"),
+        ObjectLink(id="lnk_2", source_type="observation", source_id="obs_2",
+                   target_type="pattern", target_id="pat_other", relationship="supports"),
+    ])
+    db.commit()
+
+
+def _assert_forgotten(db):
+    assert db.get(Observation, "obs_1") is None
+    assert db.get(Pattern, "pat_only") is None  # its only observation is gone
+    assert db.get(Pattern, "pat_both").observation_ids_json == '["obs_2"]'
+    assert db.get(Observation, "obs_2").related_observation_ids_json == "[]"
+    assert db.get(Pattern, "pat_other") is None  # deleted by hand
+    assert db.scalars(select(ObjectLink)).all() == []
+
+
+def _forget(stack):
+    headers = stack.admin_headers()
+    assert stack.http.delete("/observation/obs_1", headers=headers).status_code == 200
+    assert stack.http.delete("/pattern/pat_other", headers=headers).status_code == 200
+
+
+def test_deleting_an_observation_and_a_pattern_leaves_nothing_that_names_them(stack):
+    with stack.sessions() as db:
+        _seed(db)
+
+    _forget(stack)
+
+    with stack.sessions() as db:
+        _assert_forgotten(db)
+    assert "calls with Maya" not in stack.stored_text()
+    assert stack.http.delete("/pattern/pat_other", headers=stack.admin_headers()).status_code == 404
+
+
+def test_a_pattern_of_another_agent_is_not_found(stack):
+    with stack.sessions() as db:
+        _seed(db)
+
+    other = {**stack.admin_headers(), "X-Agent-Id": "helper"}
+
+    assert stack.http.delete("/pattern/pat_other", headers=other).status_code == 404
+    with stack.sessions() as db:
+        assert db.get(Pattern, "pat_other") is not None
+
+
+def test_a_restored_backup_replays_the_observation_and_pattern_deletes(stack, tmp_path):
+    with stack.sessions() as db:
+        _seed(db)
+    _forget(stack)
+    with stack.sessions() as db:
+        receipts = recovery.evidence(db)
+    assert {("owner", "pattern", "pat_only"), ("owner", "pattern", "pat_other")} <= set(receipts)
+
+    engine = create_engine(f"sqlite:///{tmp_path / 'restored.db'}")
+    Base.metadata.create_all(engine)
+    restored = sessionmaker(engine, autoflush=False, expire_on_commit=False)
+    with restored() as db:
+        _seed(db)
+
+    recovery.replay(receipts, restored)
+
+    with restored() as db:
+        _assert_forgotten(db)
+    engine.dispose()