diff --git a/CHANGELOG.md b/CHANGELOG.md index 85e427d..0efed81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,15 @@ a decision with visible consequences. goes, the removed entity's vector is queued for deletion (the startup dedup left it in Qdrant), and a malformed tags or attributes value no longer stops startup. +- **Patterns can be deleted, and a deleted observation leaves them.** There was no way to delete + a pattern, and deleting an observation left its id, its lineage links and every pattern built + from it. `DELETE /pattern/{id}` now deletes a pattern of the calling agent (404 for another + agent's) with its lineage links and a `pattern` deletion receipt. Deleting an observation + removes its lineage links, its id in other observations and in patterns, and a pattern left + with no observation (with its receipt). Recovery replay of `observation` and `pattern` + receipts removes the same set from a restored backup. A pattern that still has other + observations keeps its text. + - **Lineage stays inside one agent.** `POST /lineage/links`, `GET /lineage/{type}/{id}`, `DELETE /lineage/links/{id}` and an episode's `evidence_ids` now answer 404 for an object of another agent (the `X-Agent-Id` header), as for a missing one. Before, they read and diff --git a/services/api/app/routes/observation.py b/services/api/app/routes/observation.py index bc0fe8e..fc534b9 100644 --- a/services/api/app/routes/observation.py +++ b/services/api/app/routes/observation.py @@ -14,6 +14,7 @@ ObservationUpdateRequest, ) from app.services.agent_config_service import get_or_create_config +from app.services.observation_forgetting import erase_observation_dependents from app.services.observation_lifecycle import apply_session_context, enforce_budget, find_duplicate from app.services.pattern_promotion import promote_from_observations from app.services.vector_outbox import enqueue_delete, enqueue_upsert, reconcile_one @@ -189,6 +190,7 @@ def delete_observation(observation_id: str, agent_id: str = Depends(get_agent_id from app.models.deletion_receipt import record record(db, agent_id, "observation", row.id) enqueue_delete(db, "observation", row.id, agent_id=agent_id) + erase_observation_dependents(db, row.id) db.delete(row) db.commit() indexing = reconcile_one("observation", observation_id, sessions=SessionLocal) diff --git a/services/api/app/routes/pattern.py b/services/api/app/routes/pattern.py index 4c89e50..79683ee 100644 --- a/services/api/app/routes/pattern.py +++ b/services/api/app/routes/pattern.py @@ -3,6 +3,7 @@ from app.core.agent import get_agent_id, resolve_agent_id from app.core.db import SessionLocal +from app.models.deletion_receipt import record from app.models.observation import Observation from app.models.pattern import Pattern from app.schemas.pattern import ( @@ -13,6 +14,7 @@ PatternSearchRequest, PatternUpdateRequest, ) +from app.services.observation_forgetting import erase_pattern from app.services.pattern_promotion import ( DEPRECATE_AT_CONTRADICTIONS, clamp_confidence, @@ -79,6 +81,19 @@ def create_pattern(payload: PatternCreateRequest, header_agent_id: str = Depends finally: db.close() +@router.delete("/{pattern_id}") +def delete_pattern(pattern_id: str, agent_id: str = Depends(get_agent_id)): + """Delete a pattern and its lineage links; it leaves a content-free receipt for replay.""" + db = SessionLocal() + try: + row = _get_owned_pattern(db, pattern_id, agent_id) + record(db, agent_id, "pattern", row.id) + erase_pattern(db, row.id) + db.commit() + return {"status": "ok"} + finally: + db.close() + @router.post("/search") def search_patterns(payload: PatternSearchRequest, header_agent_id: str = Depends(get_agent_id)): agent_id = resolve_agent_id(header_agent_id, payload.agent_id) diff --git a/services/api/app/services/deletion_recovery.py b/services/api/app/services/deletion_recovery.py index 16f5ec2..691f147 100644 --- a/services/api/app/services/deletion_recovery.py +++ b/services/api/app/services/deletion_recovery.py @@ -11,12 +11,14 @@ from app.models.memory import Memory from app.models.object_link import ObjectLink from app.models.observation import Observation +from app.models.pattern import Pattern from app.models.session_transcript import SessionTranscript from app.services import conversation_memory, listener_forgetting from app.services.entity_forgetting import erase_dependents +from app.services.observation_forgetting import erase_observation_dependents, erase_pattern from sqlalchemy import select -MODELS = {"memory": Memory, "entity": Entity, "observation": Observation, +MODELS = {"memory": Memory, "entity": Entity, "observation": Observation, "pattern": Pattern, "episode": EpisodeObject, "link": ObjectLink, "transcript": SessionTranscript} @@ -124,6 +126,11 @@ def replay(receipts, target_sessions): record(db, agent_id, kind, identity) if kind == "entity": erase_dependents(db, identity) + if kind == "observation": + erase_observation_dependents(db, identity) + if kind == "pattern": + erase_pattern(db, identity) + row = None if row is not None: if kind == "episode": links = list(db.scalars(select(ObjectLink).where( diff --git a/services/api/app/services/entity_forgetting.py b/services/api/app/services/entity_forgetting.py index a1c9fde..d86436b 100644 --- a/services/api/app/services/entity_forgetting.py +++ b/services/api/app/services/entity_forgetting.py @@ -2,27 +2,13 @@ its id in observation and pattern id lists. Used by DELETE /entity/{id} and by recovery replay, so an entity deleted before a restore leaves nothing behind after it either.""" -import json - -from app.models.deletion_receipt import record from app.models.entity import EntityEdge, EntityEvent, EntityHistory -from app.models.object_link import ObjectLink -from sqlalchemy import delete, or_, select, text +from app.services.lineage_forgetting import erase_links, strip_id +from sqlalchemy import delete, or_ ID_LISTS = (("observations", "entity_ids_json"), ("patterns", "applies_to_entity_ids_json")) -def _without(value: str | None, entity_id: str) -> list | None: - """The stored id list without `entity_id`, or None when it does not hold it (or is not a list).""" - try: - ids = json.loads(value) if value else [] - except (TypeError, ValueError): - return None - if not isinstance(ids, list) or entity_id not in ids: - return None - return [i for i in ids if i != entity_id] - - def erase_dependents(db, entity_id: str) -> None: """Delete everything that refers to `entity_id`, inside the caller's transaction (no commit). Each removed lineage link leaves a content-free `link` receipt, as other deletes do.""" @@ -30,22 +16,6 @@ def erase_dependents(db, entity_id: str) -> None: db.execute(delete(EntityEvent).where(EntityEvent.entity_id == entity_id)) db.execute(delete(EntityEdge).where( or_(EntityEdge.from_entity_id == entity_id, EntityEdge.to_entity_id == entity_id))) - - links = db.scalars(select(ObjectLink).where(or_( - (ObjectLink.source_type == "entity") & (ObjectLink.source_id == entity_id), - (ObjectLink.target_type == "entity") & (ObjectLink.target_id == entity_id), - ))).all() - for link in links: - record(db, "", "link", link.id) - db.delete(link) - + erase_links(db, "entity", entity_id) for table, column in ID_LISTS: - rows = db.execute( - text(f"SELECT id, {column} AS ids FROM {table} WHERE {column} LIKE :pat"), - {"pat": f"%{entity_id}%"}, - ).fetchall() - for row in rows: - kept = _without(row.ids, entity_id) - if kept is not None: - db.execute(text(f"UPDATE {table} SET {column} = :v WHERE id = :id"), - {"v": json.dumps(kept), "id": row.id}) + strip_id(db, table, column, entity_id) diff --git a/services/api/app/services/lineage_forgetting.py b/services/api/app/services/lineage_forgetting.py new file mode 100644 index 0000000..5c6fb7f --- /dev/null +++ b/services/api/app/services/lineage_forgetting.py @@ -0,0 +1,47 @@ +"""Shared steps of a delete: drop the lineage links that name an object (each leaving a +content-free `link` receipt) and take its id out of other rows' JSON id lists.""" + +import json + +from app.models.deletion_receipt import record +from app.models.object_link import ObjectLink +from sqlalchemy import or_, select, text + + +def erase_links(db, kind: str, object_id: str) -> None: + """Delete every lineage link with `kind`/`object_id` at either end (no commit).""" + links = db.scalars(select(ObjectLink).where(or_( + (ObjectLink.source_type == kind) & (ObjectLink.source_id == object_id), + (ObjectLink.target_type == kind) & (ObjectLink.target_id == object_id), + ))).all() + for link in links: + record(db, "", "link", link.id) + db.delete(link) + + +def _without(value: str | None, object_id: str) -> list | None: + """The stored id list without `object_id`, or None when it does not hold it (or is not a list).""" + try: + ids = json.loads(value) if value else [] + except (TypeError, ValueError): + return None + if not isinstance(ids, list) or object_id not in ids: + return None + return [i for i in ids if i != object_id] + + +def strip_id(db, table: str, column: str, object_id: str) -> list[tuple[str, list]]: + """Remove `object_id` from `table.column` (a JSON id list) in every row that holds it. + Returns (row id, ids left) for each changed row. `table` and `column` are constants.""" + rows = db.execute( + text(f"SELECT id, {column} AS ids FROM {table} WHERE {column} LIKE :pat"), + {"pat": f"%{object_id}%"}, + ).fetchall() + changed = [] + for row in rows: + kept = _without(row.ids, object_id) + if kept is not None: + db.execute(text(f"UPDATE {table} SET {column} = :v WHERE id = :id"), + {"v": json.dumps(kept), "id": row.id}) + changed.append((row.id, kept)) + return changed diff --git a/services/api/app/services/observation_forgetting.py b/services/api/app/services/observation_forgetting.py new file mode 100644 index 0000000..af12ddb --- /dev/null +++ b/services/api/app/services/observation_forgetting.py @@ -0,0 +1,31 @@ +"""What goes with a deleted observation or pattern. Used by the DELETE routes and by recovery +replay, so a delete made before a restore leaves nothing behind after it either. + +A pattern is built from observations: when the last observation it names is deleted, the +pattern goes too (with its own receipt), so its derived text does not outlive its evidence. +A pattern that still has other observations keeps its text.""" + +from app.models.deletion_receipt import record +from app.models.pattern import Pattern +from app.services.lineage_forgetting import erase_links, strip_id + + +def erase_pattern(db, pattern_id: str) -> None: + """Delete the pattern's lineage links and the row itself, if present (no commit). + The caller records the pattern's receipt.""" + erase_links(db, "pattern", pattern_id) + row = db.get(Pattern, pattern_id) + if row is not None: + db.delete(row) + + +def erase_observation_dependents(db, observation_id: str) -> None: + """Delete what refers to `observation_id` (no commit): its lineage links, its id in other + observations' related ids, and its id in patterns; a pattern left with no observation goes.""" + erase_links(db, "observation", observation_id) + strip_id(db, "observations", "related_observation_ids_json", observation_id) + for pattern_id, left in strip_id(db, "patterns", "observation_ids_json", observation_id): + if not left: + pattern = db.get(Pattern, pattern_id) + record(db, pattern.agent_id if pattern else "", "pattern", pattern_id) + erase_pattern(db, pattern_id) diff --git a/services/api/tests/test_pattern_forgetting.py b/services/api/tests/test_pattern_forgetting.py new file mode 100644 index 0000000..eb5ee6a --- /dev/null +++ b/services/api/tests/test_pattern_forgetting.py @@ -0,0 +1,93 @@ +"""A pattern can be deleted, and a deleted observation does not live on in patterns: its id +leaves them, its lineage links go, and a pattern left with no observation goes with it. A +restored backup replays the same deletes.""" + +from datetime import UTC, datetime + +from app.core.db import Base +from app.models.object_link import ObjectLink +from app.models.observation import Observation +from app.models.pattern import Pattern +from app.services import deletion_recovery as recovery +from memory_stack import stack as stack +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +WHEN = datetime(2026, 1, 2, tzinfo=UTC) + + +def _seed(db): + db.add_all([ + Observation(id="obs_1", agent_id="owner", session_id="s", observed_at=WHEN, signal_type="mood", + description="tired after calls with Maya"), + Observation(id="obs_2", agent_id="owner", session_id="s", observed_at=WHEN, signal_type="mood", + description="tired on Mondays", related_observation_ids_json='["obs_1"]'), + Pattern(id="pat_only", agent_id="owner", pattern_name="calls with Maya", + description="tired after calls with Maya", observation_ids_json='["obs_1"]'), + Pattern(id="pat_both", agent_id="owner", pattern_name="tired", observation_ids_json='["obs_1", "obs_2"]'), + Pattern(id="pat_other", agent_id="owner", pattern_name="other", observation_ids_json='["obs_2"]'), + ObjectLink(id="lnk_1", source_type="observation", source_id="obs_1", + target_type="pattern", target_id="pat_both", relationship="supports"), + ObjectLink(id="lnk_2", source_type="observation", source_id="obs_2", + target_type="pattern", target_id="pat_other", relationship="supports"), + ]) + db.commit() + + +def _assert_forgotten(db): + assert db.get(Observation, "obs_1") is None + assert db.get(Pattern, "pat_only") is None # its only observation is gone + assert db.get(Pattern, "pat_both").observation_ids_json == '["obs_2"]' + assert db.get(Observation, "obs_2").related_observation_ids_json == "[]" + assert db.get(Pattern, "pat_other") is None # deleted by hand + assert db.scalars(select(ObjectLink)).all() == [] + + +def _forget(stack): + headers = stack.admin_headers() + assert stack.http.delete("/observation/obs_1", headers=headers).status_code == 200 + assert stack.http.delete("/pattern/pat_other", headers=headers).status_code == 200 + + +def test_deleting_an_observation_and_a_pattern_leaves_nothing_that_names_them(stack): + with stack.sessions() as db: + _seed(db) + + _forget(stack) + + with stack.sessions() as db: + _assert_forgotten(db) + assert "calls with Maya" not in stack.stored_text() + assert stack.http.delete("/pattern/pat_other", headers=stack.admin_headers()).status_code == 404 + + +def test_a_pattern_of_another_agent_is_not_found(stack): + with stack.sessions() as db: + _seed(db) + + other = {**stack.admin_headers(), "X-Agent-Id": "helper"} + + assert stack.http.delete("/pattern/pat_other", headers=other).status_code == 404 + with stack.sessions() as db: + assert db.get(Pattern, "pat_other") is not None + + +def test_a_restored_backup_replays_the_observation_and_pattern_deletes(stack, tmp_path): + with stack.sessions() as db: + _seed(db) + _forget(stack) + with stack.sessions() as db: + receipts = recovery.evidence(db) + assert {("owner", "pattern", "pat_only"), ("owner", "pattern", "pat_other")} <= set(receipts) + + engine = create_engine(f"sqlite:///{tmp_path / 'restored.db'}") + Base.metadata.create_all(engine) + restored = sessionmaker(engine, autoflush=False, expire_on_commit=False) + with restored() as db: + _seed(db) + + recovery.replay(receipts, restored) + + with restored() as db: + _assert_forgotten(db) + engine.dispose()