From e4cb285f6372999946681dfcf2192e795f2fbbe9 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Sun, 13 Sep 2026 22:59:35 -0400 Subject: [PATCH 01/20] release: jvagent 0.1.8rc12 for jvspatial 0.0.19 Drop graph-repair edge_ids sync phases; pin jvspatial==0.0.19. --- .planning/reference/jvspatial-integration.md | 6 +- CHANGELOG.md | 10 + jvagent/core/endpoints/graph_repair.py | 5 +- jvagent/core/graph_repair_job.py | 201 +++---------------- jvagent/core/repair_phases/types.py | 2 + jvagent/core/repair_scratch.py | 4 +- jvagent/core/repair_state.py | 21 +- jvagent/version.py | 2 +- jvchat/src/components/GraphViewer.tsx | 1 - pyproject.toml | 2 +- requirements-all.txt | 2 +- requirements.txt | 2 +- tests/core/test_graph_repair.py | 1 - tests/core/test_graph_repair_critical.py | 108 ++++------ 14 files changed, 93 insertions(+), 274 deletions(-) diff --git a/.planning/reference/jvspatial-integration.md b/.planning/reference/jvspatial-integration.md index 8db76730..9d8e10e2 100644 --- a/.planning/reference/jvspatial-integration.md +++ b/.planning/reference/jvspatial-integration.md @@ -7,7 +7,7 @@ ## 1. Where jvspatial lives - **Source**: `/Users/eldonmarks/Briefcase/dev/jv/jvspatial` (sibling directory in this workspace). -- **Pip install**: declared in [`pyproject.toml`](../../pyproject.toml) as `jvspatial==0.0.17`. +- **Pip install**: declared in [`pyproject.toml`](../../pyproject.toml) as `jvspatial==0.0.19`. - **Own docs**: jvspatial has its own [`README.md`](../../../jvspatial/README.md) and [`SPEC.md`](../../../jvspatial/SPEC.md). Treat those as authoritative for anything below. --- @@ -25,7 +25,7 @@ Object ── persistence-capable Pydantic-style base ``` - `Object` (`jvspatial/core/entities/object.py:19`) — base persistence-capable class with id, entity type, graph context. All entity types inherit. Pydantic-aware. -- `Node` (`jvspatial/core/entities/node.py:34`) — graph node. Holds `edge_ids: List[str]`, optional `visitor`, `@on_visit` hook registration. Subclass for graph entities. +- `Node` (`jvspatial/core/entities/node.py:34`) — graph node. Adjacency is derive-only (no persisted `edge_ids`); optional `visitor`, `@on_visit` hook registration. Subclass for graph entities. Graph repair must not rewrite node adjacency lists (jvspatial ≥0.0.19 removed sync phases). - `Edge` (`jvspatial/core/entities/edge.py:29`) — relationship. Has `source`/`target` Node IDs. Directional or bidirectional. - `Walker` (`jvspatial/core/entities/walker.py:83`) — traversal agent. Visit queue + trail. Built-in protection: `max_steps=10000`, `max_visits_per_node=100`, `max_execution_time=300s`, `max_queue_size=1000`. - `Root` (`jvspatial/core/entities/root.py:11`) — singleton; id fixed at `"n.Root.root"`. Created once. @@ -171,7 +171,7 @@ Things jvagent **owns**: ## 5. Version policy -- Minimum required jvspatial: pinned in [`pyproject.toml`](../../pyproject.toml) as `jvspatial==X.Y.Z`. Current: `==0.0.17`. +- Minimum required jvspatial: pinned in [`pyproject.toml`](../../pyproject.toml) as `jvspatial==X.Y.Z`. Current: `==0.0.19`. - When jvspatial introduces breaking changes (e.g., walker API rename, persistence shape change), bump the pin and update this section. - When adding a new dependency on a jvspatial feature, document the symbol + version it was introduced in. Helps downstream consumers know the floor. - Rationale: [`adr/0006-jvspatial-dependency.md`](../adr/0006-jvspatial-dependency.md). diff --git a/CHANGELOG.md b/CHANGELOG.md index b6c2d2ee..cf4700b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -488,6 +488,16 @@ and this project adheres to [PEP 440](https://peps.python.org/pep-0440/) / ``get_access_control_action`` / ``get_action_by_type`` heal duplicates on read; graph repair uses the same keeper heuristic as bootstrap dedupe. +## [0.1.8rc12] - 2026-09-14 + +### Changed + +- **jvspatial 0.0.19** — derive-only node adjacency. Graph repair no longer + runs `PH_SYNC_PREPARE` / `PH_SYNC_APPLY` (those phases skip to orphans on + mid-upgrade resume). Dup-apply and `RepairState.finish` no longer mutate + `Node.edge_ids`. Response field `node_edge_ids_synced` dropped from the + graph-repair endpoint and GraphViewer summary. Pin `jvspatial==0.0.19`. + ## [0.1.8rc11] - 2026-09-13 ### Changed diff --git a/jvagent/core/endpoints/graph_repair.py b/jvagent/core/endpoints/graph_repair.py index 7ddb8ecf..6c281246 100644 --- a/jvagent/core/endpoints/graph_repair.py +++ b/jvagent/core/endpoints/graph_repair.py @@ -46,9 +46,6 @@ "orphaned_nodes_deleted": ResponseField( field_type=int, description="Orphan nodes deleted" ), - "node_edge_ids_synced": ResponseField( - field_type=int, description="Nodes with edge_ids synced" - ), "duplicate_edges_removed": ResponseField( field_type=int, description="Duplicate edges removed" ), @@ -188,7 +185,7 @@ async def graph_repair_state() -> Dict[str, Any]: db = get_default_context().database scratch_rows = 0 if rs.run_id: - for kind in ("all_node_id", "bfs_seen", "node_edge", "valid_edge", "edge_pair"): + for kind in ("all_node_id", "bfs_seen", "edge_pair"): scratch_rows += await scratch_count(db, rs.run_id, kind) return { diff --git a/jvagent/core/graph_repair_job.py b/jvagent/core/graph_repair_job.py index c898f861..29455136 100644 --- a/jvagent/core/graph_repair_job.py +++ b/jvagent/core/graph_repair_job.py @@ -16,7 +16,7 @@ import time from collections import deque from inspect import isawaitable -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Tuple from jvagent.core.repair_phases.memory import ( tick_memory_agents, @@ -51,6 +51,10 @@ repair_checkpoint, ) +# Removed in jvspatial 0.0.19 (derive-only adjacency). Kept as aliases so +# mid-upgrade resume of an in-flight repair can skip forward. +_REMOVED_SYNC_PHASES = frozenset({PH_SYNC_PREPARE, PH_SYNC_APPLY}) + logger = logging.getLogger(__name__) # Non-serializable reattach lookup maps keyed by repair run_id (never stored in cursor). @@ -70,8 +74,8 @@ def _trim_reattach_ctx_cache() -> None: SORT_ID_ASC: List[Tuple[str, int]] = [("id", 1)] -# Full-graph orphan/dup/prune phases run after edge sync (post-listen). When -# JVAGENT_DEFER_REPAIR=1 these are skipped so cold start can return faster; +# Full-graph orphan/dup/prune phases run after dead-edge cleanup (post-listen). +# When JVAGENT_DEFER_REPAIR=1 these are skipped so cold start can return faster; # schedule POST /graph/repair (or the repair scheduler) to run them later. _OPTIONAL_POST_LISTEN_PHASES = frozenset( { @@ -134,7 +138,6 @@ def _new_result_counters() -> Dict[str, Any]: "dead_edges_removed": 0, "orphaned_nodes_reattached": 0, "orphaned_nodes_deleted": 0, - "node_edge_ids_synced": 0, "duplicate_edges_removed": 0, "interactions_pruned": 0, "counters_fixed": 0, @@ -210,6 +213,13 @@ def _restart_fresh() -> Dict[str, Any]: "remapping to the first work phase (unfinished session)" ) phase = PH_SCHEMA_APP_DEDUPE if dry_run else PH_MEMORY_COUNTERS + # jvspatial 0.0.19: node edge_ids sync phases removed (derive-only adjacency). + if phase in _REMOVED_SYNC_PHASES: + phase = PH_ORPHANS_LIST_NODES + cur = { + "last_node_id": "", + "run_id": payload.get("run_id") or cur.get("run_id") or "", + } return { "phase": phase, "dry_run": dry_run, @@ -602,12 +612,8 @@ async def _tick_dead_edges( deadline = time.monotonic() + (limits.max_seconds or 1e9) page = await _find_edges_page(context, last if last else None, batch) if not page: - state["phase"] = PH_SYNC_PREPARE - state["cursor"] = { - "last_edge_id": "", - "acc_node_edges": {}, - "acc_valid_ids": [], - } + state["phase"] = PH_ORPHANS_LIST_NODES + state["cursor"] = {"last_node_id": ""} return True processed = 0 @@ -664,157 +670,8 @@ async def try_delete(edge_data: dict, edge_id: str) -> int: return True if len(page) < batch and full_page: - state["phase"] = PH_SYNC_PREPARE - state["cursor"] = { - "last_edge_id": "", - "acc_node_edges": {}, - "acc_valid_ids": [], - } - return True - - -async def _tick_sync_prepare( - context: Any, state: Dict[str, Any], limits: RepairLimits -) -> bool: - """Accumulate node->edge ids and valid edge ids from paged edges. - - Uses the repair_scratch collection so the RepairState cursor stays small - regardless of graph size. - """ - from jvagent.core.repair_scratch import ( - ensure_scratch_indexes, - scratch_upsert_bulk, - ) - - cur = state["cursor"] - run_id: str = cur.get("run_id") or state.get("run_id") or "" - if not run_id: - import uuid - - run_id = uuid.uuid4().hex - state["run_id"] = run_id - cur["run_id"] = run_id - db = context.database - await ensure_scratch_indexes(db) - - last = cur.get("last_edge_id") or "" - batch = limits.batch_size - db = context.database - - page = await _find_edges_page(context, last if last else None, batch) - if not page: - state["phase"] = PH_SYNC_APPLY - state["cursor"] = {"last_node_id": "", "run_id": run_id} - return True - - node_edge_items: List[Tuple[str, str]] = [] - valid_edge_items: List[Tuple[str, str]] = [] - for data in page: - eid = data.get("id") - source = data.get("source") - target = data.get("target") - if eid: - valid_edge_items.append((eid, "")) - if eid and source: - # key = "|" so we can group by node in apply phase - node_edge_items.append((f"{source}|{eid}", eid)) - if eid and target: - node_edge_items.append((f"{target}|{eid}", eid)) - - if node_edge_items: - await scratch_upsert_bulk(db, run_id, "node_edge", node_edge_items) - if valid_edge_items: - await scratch_upsert_bulk(db, run_id, "valid_edge", valid_edge_items) - - cur["last_edge_id"] = page[-1].get("id", "") - cur["run_id"] = run_id - if len(page) < batch: - state["phase"] = PH_SYNC_APPLY - state["cursor"] = {"last_node_id": "", "run_id": run_id} - return True - - -async def _tick_sync_apply( - context: Any, state: Dict[str, Any], limits: RepairLimits -) -> bool: - """Sync node edge_ids reading valid/expected sets from the scratch collection.""" - from jvspatial.core import Node - - from jvagent.core.repair_scratch import scratch_page, scratch_page_key_prefix - - cur = state["cursor"] - run_id: str = cur.get("run_id") or state.get("run_id") or "" - last = cur.get("last_node_id") or "" - batch = limits.batch_size - synced = 0 - db = context.database - - page = await _find_nodes_page(context, last if last else None, batch) - if not page: state["phase"] = PH_ORPHANS_LIST_NODES - state["cursor"] = {"last_node_id": "", "run_id": run_id} - return True - - # Page through all valid_edge rows (no fixed cap). - valid_ids: Set[str] = set() - valid_after: Optional[str] = None - while True: - valid_rows = await scratch_page(db, run_id, "valid_edge", valid_after, batch) - if not valid_rows: - break - valid_ids.update(r["key"] for r in valid_rows if r.get("key")) - if len(valid_rows) < batch: - break - valid_after = valid_rows[-1].get("key") - - dry = state["dry_run"] - for data in page: - node_id = data.get("id") - if not node_id: - continue - current_edge_ids = set(data.get("edges", [])) - - # Expected edges for this node only (key prefix "|"). - expected: Set[str] = set() - edge_prefix = f"{node_id}|" - edge_after: Optional[str] = None - while True: - node_edge_rows = await scratch_page_key_prefix( - db, run_id, "node_edge", edge_prefix, edge_after, batch - ) - if not node_edge_rows: - break - for r in node_edge_rows: - k = r.get("key", "") - if k.startswith(edge_prefix): - expected.add(k.split("|", 1)[1]) - if len(node_edge_rows) < batch: - break - edge_after = node_edge_rows[-1].get("key") - - valid_current = current_edge_ids & valid_ids - new_edge_ids = valid_current | expected - if set(current_edge_ids) != new_edge_ids: - if not dry: - try: - node = await context._deserialize_entity(Node, data) - if node: - node.edge_ids = list(new_edge_ids) - await node.save() - synced += 1 - except Exception as e: - logger.warning( - "Failed to sync edge_ids for node %s: %s", node_id, e - ) - else: - synced += 1 - - state["result"]["node_edge_ids_synced"] += synced - cur["last_node_id"] = page[-1].get("id", "") - cur["run_id"] = run_id - if len(page) < batch: - state["phase"] = PH_ORPHANS_LIST_NODES - state["cursor"] = {"last_node_id": "", "run_id": run_id} + state["cursor"] = {"last_node_id": ""} return True @@ -1182,7 +1039,7 @@ async def _tick_dup_prepare( async def _tick_dup_apply( context: Any, state: Dict[str, Any], limits: RepairLimits ) -> bool: - from jvspatial.core import Edge, Node + from jvspatial.core import Edge from jvagent.core.repair_scratch import scratch_page @@ -1218,14 +1075,6 @@ async def _tick_dup_apply( try: edge = await context._deserialize_entity(Edge, dup_data) if edge: - source_node = await context.get(Node, edge.source) - target_node = await context.get(Node, edge.target) - if source_node and edge.id in source_node.edge_ids: - source_node.edge_ids.remove(edge.id) - await source_node.save() - if target_node and edge.id in target_node.edge_ids: - target_node.edge_ids.remove(edge.id) - await target_node.save() await context.delete(edge, cascade=False) removed += 1 except Exception as e: @@ -1337,8 +1186,6 @@ def _build_message(state: Dict[str, Any]) -> str: parts.append(f"{r['orphaned_nodes_reattached']} orphan(s) reattached") if r.get("orphaned_nodes_deleted"): parts.append(f"{r['orphaned_nodes_deleted']} orphan(s) deleted") - if r.get("node_edge_ids_synced"): - parts.append(f"{r['node_edge_ids_synced']} node(s) edge_ids synced") if r.get("duplicate_edges_removed"): parts.append(f"{r['duplicate_edges_removed']} duplicate edge(s) removed") if r.get("interactions_pruned"): @@ -1360,8 +1207,6 @@ def _build_message(state: Dict[str, Any]) -> str: PH_SCHEMA_MEMORY_DEDUPE, PH_SCHEMA_SINGLETON_ACTIONS, PH_DEAD_EDGES, - PH_SYNC_PREPARE, - PH_SYNC_APPLY, PH_ORPHANS_LIST_NODES, PH_ORPHANS_BFS, PH_ORPHANS_REATTACH, @@ -1405,6 +1250,12 @@ async def run_repair_session( stall_count: int = int(state.get("stall_count", 0)) phase = state.get("phase", PH_DONE) + # Mid-upgrade resume: skip removed edge_ids sync phases (jvspatial 0.0.19). + if phase in _REMOVED_SYNC_PHASES: + run_id = state.get("run_id") or (state.get("cursor") or {}).get("run_id") or "" + state["phase"] = PH_ORPHANS_LIST_NODES + state["cursor"] = {"last_node_id": "", "run_id": run_id} + phase = PH_ORPHANS_LIST_NODES _apply_deferred_phase_skip(state) phase = state.get("phase", PH_DONE) if phase == PH_DONE: @@ -1433,10 +1284,6 @@ async def run_repair_session( tick_coro = _tick_schema_singleton_actions(context, state, limits) elif phase == PH_DEAD_EDGES: tick_coro = _tick_dead_edges(context, state, limits) - elif phase == PH_SYNC_PREPARE: - tick_coro = _tick_sync_prepare(context, state, limits) - elif phase == PH_SYNC_APPLY: - tick_coro = _tick_sync_apply(context, state, limits) elif phase == PH_ORPHANS_LIST_NODES: tick_coro = _tick_orphans_list_nodes(context, state, limits) elif phase == PH_ORPHANS_BFS: diff --git a/jvagent/core/repair_phases/types.py b/jvagent/core/repair_phases/types.py index 460bd086..afc339ba 100644 --- a/jvagent/core/repair_phases/types.py +++ b/jvagent/core/repair_phases/types.py @@ -15,6 +15,8 @@ PH_SCHEMA_MEMORY_DEDUPE = "schema_memory_dedupe" PH_SCHEMA_SINGLETON_ACTIONS = "schema_singleton_actions" PH_DEAD_EDGES = "dead_edges" +# Kept for mid-upgrade resume: run_repair_session / state_from_dict advance these +# to PH_ORPHANS_LIST_NODES (jvspatial 0.0.19 removed Node.edge_ids sync). PH_SYNC_PREPARE = "sync_prepare" PH_SYNC_APPLY = "sync_apply" PH_ORPHANS_LIST_NODES = "orphans_list_nodes" diff --git a/jvagent/core/repair_scratch.py b/jvagent/core/repair_scratch.py index e0331f3a..73cc03f4 100644 --- a/jvagent/core/repair_scratch.py +++ b/jvagent/core/repair_scratch.py @@ -16,9 +16,9 @@ "id": "::", # PK - used for upserts "_id": "::", "run_id": str, - "kind": str, # "node_id" | "bfs_seen" | "node_edge" | "valid_edge" | "edge_pair" + "kind": str, # "node_id" | "bfs_seen" | "edge_pair" | "all_node_id" "key": str, # e.g. node_id, edge_id, "source\\ntarget" - "value": str, # optional secondary value (e.g. edge_id for node_edge rows) + "value": str, # optional secondary value "created_at": float, # Unix timestamp for TTL } """ diff --git a/jvagent/core/repair_state.py b/jvagent/core/repair_state.py index 0e36477a..f4f4ddda 100644 --- a/jvagent/core/repair_state.py +++ b/jvagent/core/repair_state.py @@ -330,13 +330,9 @@ async def save_progress( async def finish(self) -> None: """Delete state node and its edges when repair is complete/reset. - After deleting each edge this method also calls - ``context.atomic_remove_edge_id`` on the *other* endpoint (typically - the App node) so that its stored ``edge_ids`` attribute is kept - consistent. Without this, the sync phase of the next repair run would - always detect a stale edge reference on App and report - ``node_edge_ids_synced: 1``, causing an apparent never-ending repair - loop even on a healthy graph. + Deletes each edge via ``context.delete`` then removes the RepairState + node. Node adjacency is derive-only (jvspatial 0.0.19+); there is no + ``edge_ids`` list to keep in sync on the other endpoint. This method is hardened to always remove the node document even when edge cleanup or the high-level ``delete()`` raises. A raw DB delete is @@ -347,17 +343,6 @@ async def finish(self) -> None: for edge in await self.edges(direction="both"): if not isinstance(edge, Edge): continue - # Determine the other endpoint of this edge (not self) - other_id = edge.target if edge.source == self.id else edge.source - if other_id: - try: - await context.atomic_remove_edge_id(other_id, edge.id) - except Exception: - logger.warning( - "repair_state.finish: could not remove edge_id %s from node %s", - edge.id, - other_id, - ) try: await context.delete(edge, cascade=False) except Exception: diff --git a/jvagent/version.py b/jvagent/version.py index 4a98da22..a218cfef 100644 --- a/jvagent/version.py +++ b/jvagent/version.py @@ -1,3 +1,3 @@ """Version information for jvagent package.""" -__version__ = "0.1.8rc11" +__version__ = "0.1.8rc12" diff --git a/jvchat/src/components/GraphViewer.tsx b/jvchat/src/components/GraphViewer.tsx index 733da962..5820eeaa 100644 --- a/jvchat/src/components/GraphViewer.tsx +++ b/jvchat/src/components/GraphViewer.tsx @@ -619,7 +619,6 @@ export function GraphViewer({ onClose, isEmbedded = false }: GraphViewerProps) { ['dead_edges_removed', 'dead edge(s) removed'], ['orphaned_nodes_reattached', 'orphan(s) reattached'], ['orphaned_nodes_deleted', 'orphan(s) deleted'], - ['node_edge_ids_synced', 'node(s) edge_ids synced'], ['duplicate_edges_removed', 'duplicate edge(s) removed'], ['interactions_pruned', 'interaction(s) pruned (rolling limit)'], ] diff --git a/pyproject.toml b/pyproject.toml index 360cfc85..9df2b48a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ dependencies = [ "aiohttp>=3.9.0", # CI records the resolved jvspatial version after install (see .github/workflows/test-jvagent.yaml). - "jvspatial==0.0.18", + "jvspatial==0.0.19", "python-dotenv>=1.0.0", "pyyaml>=6.0.0", "httpx>=0.27.0", diff --git a/requirements-all.txt b/requirements-all.txt index 90ab6c13..68af413a 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -6,7 +6,7 @@ # Must stay in sync with [project] dependencies in pyproject.toml — # enforced by tests/test_requirements_sync.py. aiohttp>=3.9.0 -jvspatial==0.0.18 +jvspatial==0.0.19 python-dotenv>=1.0.0 pyyaml>=6.0.0 httpx>=0.27.0 diff --git a/requirements.txt b/requirements.txt index 9a58cedb..5b3ae8b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ # Test-only deps (incl. Docling for PageIndex): pyproject.toml # [project.optional-dependencies] test — install with: pip install -e ".[test]" aiohttp>=3.9.0 -jvspatial==0.0.18 +jvspatial==0.0.19 python-dotenv>=1.0.0 pyyaml>=6.0.0 httpx>=0.27.0 diff --git a/tests/core/test_graph_repair.py b/tests/core/test_graph_repair.py index 993c1525..3449bc2f 100644 --- a/tests/core/test_graph_repair.py +++ b/tests/core/test_graph_repair.py @@ -68,7 +68,6 @@ async def test_repair_returns_expected_structure(self, temp_dir, test_db): assert "dead_edges_removed" in result assert "orphaned_nodes_reattached" in result assert "orphaned_nodes_deleted" in result - assert "node_edge_ids_synced" in result assert "duplicate_edges_removed" in result assert "interactions_pruned" in result assert "message" in result diff --git a/tests/core/test_graph_repair_critical.py b/tests/core/test_graph_repair_critical.py index bd0da384..d01dd759 100644 --- a/tests/core/test_graph_repair_critical.py +++ b/tests/core/test_graph_repair_critical.py @@ -1,4 +1,4 @@ -"""Graph repair cursor serialization and per-node edge sync (C3/C4).""" +"""Graph repair cursor serialization and removed sync-phase resume.""" from __future__ import annotations @@ -11,8 +11,10 @@ from jvagent.core import graph_repair_job from jvagent.core.repair_phases.types import ( PH_ORPHANS_INTERACTION, + PH_ORPHANS_LIST_NODES, PH_ORPHANS_REATTACH, PH_SYNC_APPLY, + PH_SYNC_PREPARE, RepairLimits, ) @@ -97,73 +99,51 @@ async def test_reattach_ctx_released_when_repair_state_restarts(): @pytest.mark.asyncio -async def test_sync_apply_queries_expected_edges_per_node(): - node_a = "n.Node.a" - node_b = "n.Node.b" - edge_a1 = "e.edge.a1" - edge_b1 = "e.edge.b1" - run_id = "run-sync" - - page_nodes = [ - {"id": node_a, "edges": []}, - {"id": node_b, "edges": []}, - ] - prefix_calls = [] - - async def _scratch_page(_db, _rid, kind, after_key, limit): - if kind == "valid_edge": - if after_key is None: - return [{"key": edge_a1}] - return [{"key": edge_b1}] - return [] - - async def _scratch_page_key_prefix(_db, _rid, kind, key_prefix, after_key, limit): - prefix_calls.append(key_prefix) - if key_prefix == f"{node_a}|": - return [{"key": f"{node_a}|{edge_a1}"}] - if key_prefix == f"{node_b}|": - return [{"key": f"{node_b}|{edge_b1}"}] - return [] - - node_objs = { - node_a: SimpleNamespace(id=node_a, edge_ids=[]), - node_b: SimpleNamespace(id=node_b, edge_ids=[]), - } - for n in node_objs.values(): - n.save = AsyncMock() - - context = SimpleNamespace(database=object()) - - async def _deserialize(_cls, data): - return node_objs[data["id"]] - - context._deserialize_entity = AsyncMock(side_effect=_deserialize) - +@pytest.mark.parametrize( + "legacy_phase", [PH_SYNC_PREPARE, PH_SYNC_APPLY, "sync_prepare", "sync_apply"] +) +async def test_legacy_sync_phase_advances_to_orphans(legacy_phase): + """In-flight repairs stuck on removed sync phases skip to orphans.""" + run_id = "run-skip-sync" state = { - "dry_run": False, - "phase": PH_SYNC_APPLY, - "cursor": {"last_node_id": "", "run_id": run_id}, - "result": {"node_edge_ids_synced": 0}, + "dry_run": True, + "phase": legacy_phase, + "cursor": {"last_edge_id": "e.old", "run_id": run_id}, + "result": graph_repair_job._new_result_counters(), + "run_id": run_id, + "stall_count": 0, } - limits = RepairLimits(batch_size=10, max_seconds=5) + limits = RepairLimits(batch_size=10, max_seconds=1) with ( patch.object( - graph_repair_job, "_find_nodes_page", new=AsyncMock(return_value=page_nodes) - ), - patch( - "jvagent.core.repair_scratch.scratch_page", - new=AsyncMock(side_effect=_scratch_page), - ), - patch( - "jvagent.core.repair_scratch.scratch_page_key_prefix", - new=AsyncMock(side_effect=_scratch_page_key_prefix), - ), + graph_repair_job, + "_tick_orphans_list_nodes", + new=AsyncMock(return_value=True), + ) as orphans_tick, + patch.object(graph_repair_job, "_repair_checkpoint", new=AsyncMock()), ): - await graph_repair_job._tick_sync_apply(context, state, limits) + await graph_repair_job.run_repair_session(state, limits) + + orphans_tick.assert_awaited_once() + # Mocked tick does not advance; phase must have left the removed sync step. + assert state["phase"] == PH_ORPHANS_LIST_NODES - assert f"{node_a}|" in prefix_calls - assert f"{node_b}|" in prefix_calls - assert node_objs[node_a].edge_ids == [edge_a1] - assert node_objs[node_b].edge_ids == [edge_b1] - assert state["result"]["node_edge_ids_synced"] == 2 + +@pytest.mark.asyncio +@pytest.mark.parametrize("legacy_phase", [PH_SYNC_PREPARE, PH_SYNC_APPLY]) +async def test_state_from_dict_skips_legacy_sync_phase(legacy_phase): + payload = { + "v": graph_repair_job.STATE_VERSION, + "phase": legacy_phase, + "cursor": {"last_edge_id": "e.old", "run_id": "run-load"}, + "result": graph_repair_job._new_result_counters(), + "dry_run": False, + "run_id": "run-load", + "stall_count": 0, + } + state = graph_repair_job.state_from_dict( + payload, dry_run=False, recent_minutes=None + ) + assert state["phase"] == PH_ORPHANS_LIST_NODES + assert state["cursor"].get("run_id") == "run-load" From fcb1a7709ee93a525e3c06dafb2a2772cdd33664 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Thu, 17 Sep 2026 12:14:41 -0400 Subject: [PATCH 02/20] fix(orchestrator): prevent duplicate assistant egress on model_error turns _after_loop gated only on interaction.response while streaming latches emitted first, so model_unavailable compose could run twice. Unify delivery detection via _turn_delivered and skip commit_pending_adhoc when the settled text is already on the interaction. Co-authored-by: Cursor --- jvagent/action/orchestrator/egress.py | 20 ++++++ jvagent/action/orchestrator/loop.py | 4 +- jvagent/action/response/response_bus.py | 11 +++ .../test_model_error_intro_single_egress.py | 69 +++++++++++++++++++ 4 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 tests/action/orchestrator/test_model_error_intro_single_egress.py diff --git a/jvagent/action/orchestrator/egress.py b/jvagent/action/orchestrator/egress.py index 49980153..bcdc4aac 100644 --- a/jvagent/action/orchestrator/egress.py +++ b/jvagent/action/orchestrator/egress.py @@ -12,6 +12,26 @@ class OrchestratorEgressMixin: + @staticmethod + def _turn_delivered(interaction: Any) -> bool: + """True once user-facing content was delivered this turn. + + ``interaction.response`` alone is insufficient: streaming can latch + ``emitted`` before ``response`` is flushed, and ``commit_pending_adhoc`` + can set ``response`` without latching. Both signals must be checked so + ``_after_loop`` and ``_egress`` never double-send. + """ + if interaction is None: + return False + has_emitted = getattr(interaction, "has_emitted", None) + if callable(has_emitted): + try: + if has_emitted(): + return True + except Exception: + pass + return bool((getattr(interaction, "response", "") or "").strip()) + @staticmethod def _ia_emitted(interaction: Any) -> bool: """True if a dispatched IA produced user-facing output this turn. diff --git a/jvagent/action/orchestrator/loop.py b/jvagent/action/orchestrator/loop.py index 1c4e0fdf..023d4e36 100644 --- a/jvagent/action/orchestrator/loop.py +++ b/jvagent/action/orchestrator/loop.py @@ -1688,7 +1688,7 @@ async def _after_loop(self, visitor: "InteractWalker", state: TurnState) -> None # tasks now; if one blocks on input it owns the egress. Inert until a # runner is registered, so skill-only turns are unaffected. interaction = getattr(visitor, "interaction", None) - emitted = bool(getattr(interaction, "response", "") if interaction else "") + emitted = self._turn_delivered(interaction) _stamp_observations(state.observations, state.last_obs_len, state.last_dec_meta) if state.ended_via == "model_error": # The model is unreachable: no finalize call (it would fail the same @@ -1714,7 +1714,7 @@ async def _after_loop(self, visitor: "InteractWalker", state: TurnState) -> None await self._send_reply(visitor, drain_directive, compose=True) state.ended_via = f"{state.ended_via}_drained" return - emitted = bool(getattr(interaction, "response", "") if interaction else "") + emitted = self._turn_delivered(interaction) # Budget/time ran out mid-task. Rather than dropping to the generic # clarify fallback (which discards the work and misreports the cause), diff --git a/jvagent/action/response/response_bus.py b/jvagent/action/response/response_bus.py index 5a1d0775..2e3e4d0d 100644 --- a/jvagent/action/response/response_bus.py +++ b/jvagent/action/response/response_bus.py @@ -764,6 +764,15 @@ async def commit_pending_adhoc( self._adhoc_accumulation.pop(interaction_id, None) return full_content = "".join(acc.chunks) + # Streaming publish() already flushed this turn to subscribers and + # interaction.response; commit_pending is a safety net for abandoned + # accumulators. Re-appending or re-emitting the same settled text + # duplicates bubbles downstream (integral message-boundary splits on a + # second adhoc id carrying the same prose). + current = (getattr(interaction, "response", "") or "") if interaction else "" + if full_content and current.strip() and full_content.strip() in current: + self._adhoc_accumulation.pop(interaction_id, None) + return now = await self._get_now() message = ResponseMessage( session_id=acc.session_id, @@ -783,6 +792,8 @@ async def commit_pending_adhoc( if self._can_send_to_adapter(adapter, message, relay_to_adapters=False): await self._send_to_adapter(adapter, message) if full_content and interaction: + if hasattr(interaction, "mark_emitted"): + interaction.mark_emitted() await self._append_to_interaction_response_impl( interaction=interaction, message_type="adhoc", diff --git a/tests/action/orchestrator/test_model_error_intro_single_egress.py b/tests/action/orchestrator/test_model_error_intro_single_egress.py new file mode 100644 index 00000000..b26b3e86 --- /dev/null +++ b/tests/action/orchestrator/test_model_error_intro_single_egress.py @@ -0,0 +1,69 @@ +"""model_error egress must not double-deliver when the latch is already set.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from jvagent.action.orchestrator.orchestrator_interact_action import ( + OrchestratorInteractAction, +) +from jvagent.action.response.response_bus import ResponseBus +from jvagent.memory.interaction import Interaction + + +def test_turn_delivered_honors_emitted_before_response(): + interaction = Interaction() + interaction.mark_emitted() + assert OrchestratorInteractAction._turn_delivered(interaction) is True + assert not (interaction.response or "").strip() + + +def test_turn_delivered_honors_response_before_emitted_latch(): + interaction = Interaction() + interaction.set_response("already here") + assert OrchestratorInteractAction._turn_delivered(interaction) is True + assert interaction.has_emitted() is False + + +@pytest.mark.asyncio +async def test_after_loop_skips_model_unavailable_when_emitted_latched(monkeypatch): + """Stream can latch ``emitted`` before ``response`` is flushed — _after_loop + must not queue a second model_unavailable compose.""" + ex = OrchestratorInteractAction() + interaction = Interaction(utterance="hi") + interaction.mark_emitted() + visitor = MagicMock() + visitor.interaction = interaction + send_reply = AsyncMock() + monkeypatch.setattr(ex, "_send_reply", send_reply) + + state = MagicMock() + state.ended_via = "model_error" + state.observations = [] + state.last_obs_len = 0 + state.last_dec_meta = None + await ex._after_loop(visitor, state) + send_reply.assert_not_called() + + +@pytest.mark.asyncio +async def test_commit_pending_adhoc_skips_when_response_already_set(): + bus = ResponseBus() + interaction = Interaction(session_id="s1", user_id="u1", utterance="hi") + interaction.set_response("Hello — model unavailable.") + acc = bus._get_or_create_accumulator( + interaction_id=interaction.id, + session_id="s1", + channel="default", + user_id="u1", + metadata={}, + category="user", + ) + acc.chunks = ["Hello — model unavailable."] + + await bus.commit_pending_adhoc(interaction.id, interaction) + + assert interaction.id not in bus._adhoc_accumulation + assert interaction.response == "Hello — model unavailable." From 8b8bb254a851c42e6770942015d263d094ab9c5b Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Thu, 17 Sep 2026 12:41:27 -0400 Subject: [PATCH 03/20] test(litellm): derive context_window from upstream metadata Hard-coded 200k for claude-sonnet-4-5 broke when LiteLLM bumped the model table to 1M; assert against litellm_capabilities() instead. Co-authored-by: Cursor --- tests/action/model/test_litellm_action.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/action/model/test_litellm_action.py b/tests/action/model/test_litellm_action.py index f243a065..c207fdc9 100644 --- a/tests/action/model/test_litellm_action.py +++ b/tests/action/model/test_litellm_action.py @@ -162,9 +162,17 @@ def _boom(): def test_capabilities_and_pricing_come_from_upstream_metadata(): - action = _action(model="anthropic/claude-sonnet-4-5") + from jvagent.action.model.capabilities import litellm_capabilities + + model = "anthropic/claude-sonnet-4-5" + upstream = litellm_capabilities(model, provider="litellm") + assert upstream is not None and upstream.context_window + + action = _action(model=model) caps = action.capabilities() - assert caps.supports_tools is True and caps.context_window == 200_000 + assert caps.supports_tools is True + assert caps.context_window == upstream.context_window + assert "litellm" in caps.source assert action.pricing().source == "litellm" assert json.dumps( action.capabilities("openai/gpt-4o-mini").__dict__ From e41fcd86a0564726aa74848793fcaf12f44521bb Mon Sep 17 00:00:00 2001 From: Tharick Jairam Date: Thu, 17 Sep 2026 13:08:11 -0400 Subject: [PATCH 04/20] fix(test): stop pinning Claude Sonnet context_window to 200k LiteLLM now reports a 1M window; keep proving capabilities come from upstream. Co-authored-by: Cursor --- tests/action/model/test_litellm_action.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/action/model/test_litellm_action.py b/tests/action/model/test_litellm_action.py index f243a065..35e14f46 100644 --- a/tests/action/model/test_litellm_action.py +++ b/tests/action/model/test_litellm_action.py @@ -164,7 +164,9 @@ def _boom(): def test_capabilities_and_pricing_come_from_upstream_metadata(): action = _action(model="anthropic/claude-sonnet-4-5") caps = action.capabilities() - assert caps.supports_tools is True and caps.context_window == 200_000 + assert caps.supports_tools is True + assert isinstance(caps.context_window, int) and caps.context_window >= 200_000 + assert "litellm" in (caps.source or "") assert action.pricing().source == "litellm" assert json.dumps( action.capabilities("openai/gpt-4o-mini").__dict__ From 470664db7a9729f856caf8dc37bab9cc546f6e10 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Thu, 17 Sep 2026 13:12:26 -0400 Subject: [PATCH 05/20] fix(orchestrator): gate _egress on _turn_delivered not emitted latch _after_loop already used _turn_delivered; _egress still checked has_emitted() only, so a turn with interaction.response set but no latch (model_error salvage path) queued a second ReplyAction publish. Also latch emitted when commit_pending_adhoc skips an already-settled replay. Co-authored-by: Cursor --- jvagent/action/orchestrator/egress.py | 6 +++--- jvagent/action/response/response_bus.py | 2 ++ .../test_model_error_intro_single_egress.py | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/jvagent/action/orchestrator/egress.py b/jvagent/action/orchestrator/egress.py index bcdc4aac..2ad7d03b 100644 --- a/jvagent/action/orchestrator/egress.py +++ b/jvagent/action/orchestrator/egress.py @@ -61,11 +61,11 @@ async def _egress(self, visitor: "InteractWalker") -> None: double-sends. """ interaction = getattr(visitor, "interaction", None) - if interaction is None or interaction.has_emitted(): + if interaction is None or self._turn_delivered(interaction): return # Gather any directives a rails IA queued this turn (no model text to add). await self._send_reply(visitor) - if not interaction.has_emitted(): + if not self._turn_delivered(interaction): await self._send_reply(visitor, self.clarify_text) async def _send_reply( @@ -126,7 +126,7 @@ async def _send_reply( gathered = await gather(visitor) if gathered: return - if interaction is not None and interaction.has_emitted(): + if interaction is not None and self._turn_delivered(interaction): return except Exception as exc: logger.warning("orchestrator: responder.gather failed: %s", exc) diff --git a/jvagent/action/response/response_bus.py b/jvagent/action/response/response_bus.py index 2e3e4d0d..f8cdd2e6 100644 --- a/jvagent/action/response/response_bus.py +++ b/jvagent/action/response/response_bus.py @@ -771,6 +771,8 @@ async def commit_pending_adhoc( # second adhoc id carrying the same prose). current = (getattr(interaction, "response", "") or "") if interaction else "" if full_content and current.strip() and full_content.strip() in current: + if interaction and hasattr(interaction, "mark_emitted"): + interaction.mark_emitted() self._adhoc_accumulation.pop(interaction_id, None) return now = await self._get_now() diff --git a/tests/action/orchestrator/test_model_error_intro_single_egress.py b/tests/action/orchestrator/test_model_error_intro_single_egress.py index b26b3e86..3eeb559d 100644 --- a/tests/action/orchestrator/test_model_error_intro_single_egress.py +++ b/tests/action/orchestrator/test_model_error_intro_single_egress.py @@ -48,6 +48,22 @@ async def test_after_loop_skips_model_unavailable_when_emitted_latched(monkeypat send_reply.assert_not_called() +@pytest.mark.asyncio +async def test_egress_skips_when_response_set_without_emitted_latch(monkeypatch): + """``_egress`` must honor ``interaction.response`` — not only ``emitted``.""" + ex = OrchestratorInteractAction() + interaction = Interaction(utterance="hi") + interaction.set_response("I'm having trouble reaching my language model right now.") + assert interaction.has_emitted() is False + visitor = MagicMock() + visitor.interaction = interaction + send_reply = AsyncMock() + monkeypatch.setattr(ex, "_send_reply", send_reply) + + await ex._egress(visitor) + send_reply.assert_not_called() + + @pytest.mark.asyncio async def test_commit_pending_adhoc_skips_when_response_already_set(): bus = ResponseBus() From 72dcca59aedb6021a5d1229ec44733d1dad607cc Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Thu, 17 Sep 2026 13:15:25 -0400 Subject: [PATCH 06/20] fix(orchestrator): only treat string interaction.response as delivered MagicMock interactions expose a truthy non-string response attribute; _turn_delivered must check isinstance(str) before skipping egress. Co-authored-by: Cursor --- jvagent/action/orchestrator/egress.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jvagent/action/orchestrator/egress.py b/jvagent/action/orchestrator/egress.py index 2ad7d03b..abdc0eb2 100644 --- a/jvagent/action/orchestrator/egress.py +++ b/jvagent/action/orchestrator/egress.py @@ -30,7 +30,8 @@ def _turn_delivered(interaction: Any) -> bool: return True except Exception: pass - return bool((getattr(interaction, "response", "") or "").strip()) + response = getattr(interaction, "response", "") or "" + return isinstance(response, str) and bool(response.strip()) @staticmethod def _ia_emitted(interaction: Any) -> bool: From bf3dcafb3e174ba1e1662b46193b5ad628eca54f Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Thu, 17 Sep 2026 15:42:14 -0400 Subject: [PATCH 07/20] fix(response): latch emitted at ResponseBus on first delivered user chunk Stops a second independent user publish from twinning assistant bubbles downstream; active stream may still finish. Covered by emitted-latch tests. Co-authored-by: Cursor --- CHANGELOG.md | 7 ++ jvagent/action/response/response_bus.py | 56 +++++++++++++++ tests/action/response/test_emitted_latch.py | 77 +++++++++++++++++++++ 3 files changed, 140 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf4700b6..119c2c7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,13 @@ and this project adheres to [PEP 440](https://peps.python.org/pep-0440/) / ### Changed +- **ResponseBus now enforces the single-egress latch.** The first delivered + non-transient user stream chunk marks its `Interaction` as emitted, active + chunks may finish that same stream, and any later independent user publish + for the turn is suppressed at the framework delivery boundary. This fixes + duplicate assistant bubbles without requiring consumers to compare or + normalize response text. + - **Defaults that permit long-running, deep-thinking models (#214).** Three shipped defaults combined to end a reasoning model's turn before it could answer, and the user-facing text ("I got stuck repeating a step") pointed at diff --git a/jvagent/action/response/response_bus.py b/jvagent/action/response/response_bus.py index f8cdd2e6..004f782b 100644 --- a/jvagent/action/response/response_bus.py +++ b/jvagent/action/response/response_bus.py @@ -443,6 +443,47 @@ async def _deliver_flush( if not (stream and not streaming_complete): content = scrub_text(content, _egress_parameters(interaction)) + # ``Interaction.emitted`` is the framework's single-egress latch + # (ADR-0025). A live user stream may continue after its first chunk set + # the latch, but every separate non-transient user publish is rejected + # here at the delivery choke point. Consumers should never need to + # compare reply text or repair duplicate bubbles. + active_user_stream = bool( + stream and interaction_id and interaction_id in self._adhoc_accumulation + ) + has_emitted = getattr(interaction, "has_emitted", None) + already_emitted = False + if callable(has_emitted): + try: + already_emitted = has_emitted() is True + except Exception: + already_emitted = False + if ( + message_category == "user" + and not transient + and interaction is not None + and content + and already_emitted + and not active_user_stream + ): + logger.debug( + "response bus: suppressed second user egress for interaction %s", + interaction_id or getattr(interaction, "id", ""), + ) + return ResponseMessage( + session_id=session_id, + user_id=user_id or "", + interaction_id=interaction_id or "", + content="", + channel=channel, + message_type="adhoc", + metadata=metadata or {}, + timestamp=now, + category=message_category, + thought_type=thought_type, + segment_id=message_segment_id, + ) + if not stream: # Non-streaming: immediate filters, adapter, accumulation, one adhoc message message = ResponseMessage( @@ -621,6 +662,14 @@ async def _deliver_flush( thought_type=thought_type, segment_id=message_segment_id, ) + # The first byte delivered owns this turn's user egress. + if ( + message_category == "user" + and not transient + and interaction is not None + and hasattr(interaction, "mark_emitted") + ): + interaction.mark_emitted() # Emit chunk to subscribers only chunk_message = ResponseMessage( id=acc.message_id, @@ -646,6 +695,13 @@ async def _deliver_flush( # a real chunk — a client that renders progressively from chunks # would otherwise never show the last sentence. if governed and content: + if ( + message_category == "user" + and not transient + and interaction is not None + and hasattr(interaction, "mark_emitted") + ): + interaction.mark_emitted() tail_meta = dict(metadata or {}) tail_meta["sequence"] = len(acc.chunks) tail_message = ResponseMessage( diff --git a/tests/action/response/test_emitted_latch.py b/tests/action/response/test_emitted_latch.py index a7fe6dcb..44013a8f 100644 --- a/tests/action/response/test_emitted_latch.py +++ b/tests/action/response/test_emitted_latch.py @@ -53,3 +53,80 @@ async def test_transient_user_publish_does_not_latch(): transient=True, ) assert interaction.has_emitted() is False + + +@pytest.mark.asyncio +async def test_second_user_egress_is_suppressed_at_the_bus(): + bus = ResponseBus() + interaction = Interaction() + seen = [] + + async def on_message(message): + seen.append(message) + + await bus.subscribe("s1", on_message, receive_chunks=True) + await bus.publish( + session_id="s1", + content="First answer.", + channel="default", + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + await bus.publish( + session_id="s1", + content="A distinct second answer must not escape.", + channel="default", + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + + assert [message.content for message in seen] == ["First answer."] + assert interaction.response == "First answer." + + +@pytest.mark.asyncio +async def test_incremental_stream_latches_first_chunk_and_allows_completion(): + bus = ResponseBus() + interaction = Interaction() + seen = [] + + async def on_message(message): + seen.append(message) + + await bus.subscribe("s1", on_message, receive_chunks=True) + await bus.publish( + session_id="s1", + content="Your order ships Tuesday.", + channel="default", + stream=True, + streaming_complete=False, + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + assert interaction.has_emitted() is True + + await bus.publish( + session_id="s1", + content="", + channel="default", + stream=True, + streaming_complete=True, + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + await bus.publish( + session_id="s1", + content="Duplicate fallback.", + channel="default", + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + + assert [message.message_type for message in seen] == ["stream_chunk", "final"] + assert "".join(message.content for message in seen) == "Your order ships Tuesday." + assert interaction.response == "Your order ships Tuesday." From 9fa60251653786a244753a016966d38b559f09e4 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Thu, 17 Sep 2026 22:45:28 -0400 Subject: [PATCH 08/20] feat(harness): freeze NativeCaller contracts and wire snapshot-scoped runtime Admit turns as (agent_id, user_id, session_id), key tool/skill caches by snapshot, and add a store-backed journal/ledger/outbox/lease surface so later HPs have a host-neutral substrate. Result reuse is opt-in via idempotency_class so the loop repeat-guard still re-executes undeclared tools. Co-authored-by: Cursor --- .planning/GLOSSARY.md | 12 + .planning/MILESTONES.md | 16 + .planning/PROJECT.md | 55 +- .planning/README.md | 12 +- .planning/REQUIREMENTS.md | 94 +++ .planning/ROADMAP.md | 129 +++ .planning/SPEC.md | 25 +- .planning/STATE.md | 75 ++ .planning/adr/0054-harness-contracts.md | 80 ++ .planning/config.json | 56 ++ .../01-contracts-and-baseline/01-01-PLAN.md | 92 +++ .../01-contracts-and-baseline/01-02-PLAN.md | 53 ++ .../PROCESS-LOCAL-STATE.md | 55 ++ .../02-identity-and-snapshots/02-01-PLAN.md | 50 ++ .../02-identity-and-snapshots/02-02-PLAN.md | 48 ++ .../phases/03-durable-execution/03-01-PLAN.md | 47 ++ .../phases/03-durable-execution/03-02-PLAN.md | 46 ++ .../phases/03-durable-execution/03-03-PLAN.md | 47 ++ .../04-01-PLAN.md | 47 ++ .../04-02-PLAN.md | 50 ++ .../04-03-PLAN.md | 54 ++ .../05-operational-excellence/05-01-PLAN.md | 45 ++ .../05-operational-excellence/05-02-PLAN.md | 45 ++ .../05-operational-excellence/05-03-PLAN.md | 45 ++ AGENTS.md | 2 + CHANGELOG.md | 6 + CLAUDE.md | 5 +- README.md | 1 + docs/HARNESS_DEPLOYMENT.md | 43 + docs/HARNESS_EXCELLENCE_PLAN.md | 368 +++++++++ docs/ORCHESTRATOR.md | 2 + docs/skill-isolation.md | 26 + jvagent/action/interact/endpoints.py | 13 + jvagent/action/interact/interact_walker.py | 47 ++ jvagent/action/model/resilience.py | 4 + jvagent/action/orchestrator/catalog.py | 85 +- .../orchestrator_interact_action.py | 64 +- .../action/orchestrator/skill_providers.py | 40 +- jvagent/action/orchestrator/skills.py | 12 + jvagent/action/orchestrator/tools.py | 57 +- jvagent/action/response/response_bus.py | 20 + jvagent/action/response/streaming.py | 23 + jvagent/core/distributed_lease.py | 4 +- jvagent/embed/interact.py | 14 + jvagent/harness/__init__.py | 49 ++ jvagent/harness/contracts.py | 280 +++++++ jvagent/harness/provider.py | 137 ++++ jvagent/harness/release.py | 82 ++ jvagent/harness/runtime.py | 744 ++++++++++++++++++ jvagent/memory/manager.py | 24 +- jvagent/tooling/tool.py | 1 + jvagent/tooling/tool_decorator.py | 6 + pyproject.toml | 1 + tests/CLAUDE.md | 2 + tests/conformance/conftest.py | 16 + .../conformance/fixtures/fake_host/README.md | 10 + .../fixtures/fake_host/__init__.py | 24 + tests/conformance/test_delivery_replay.py | 72 ++ tests/conformance/test_identity_isolation.py | 33 + tests/conformance/test_invocation_recovery.py | 77 ++ tests/conformance/test_leases.py | 59 ++ .../test_process_local_baseline.py | 129 +++ tests/conformance/test_provider_contract.py | 48 ++ tests/conformance/test_release_matrix.py | 35 + tests/conformance/test_snapshot_revocation.py | 78 ++ tests/conformance/test_traces.py | 29 + tests/harness/test_contracts.py | 145 ++++ tests/harness/test_runtime.py | 373 +++++++++ 68 files changed, 4521 insertions(+), 47 deletions(-) create mode 100644 .planning/MILESTONES.md create mode 100644 .planning/REQUIREMENTS.md create mode 100644 .planning/ROADMAP.md create mode 100644 .planning/STATE.md create mode 100644 .planning/adr/0054-harness-contracts.md create mode 100644 .planning/config.json create mode 100644 .planning/phases/01-contracts-and-baseline/01-01-PLAN.md create mode 100644 .planning/phases/01-contracts-and-baseline/01-02-PLAN.md create mode 100644 .planning/phases/01-contracts-and-baseline/PROCESS-LOCAL-STATE.md create mode 100644 .planning/phases/02-identity-and-snapshots/02-01-PLAN.md create mode 100644 .planning/phases/02-identity-and-snapshots/02-02-PLAN.md create mode 100644 .planning/phases/03-durable-execution/03-01-PLAN.md create mode 100644 .planning/phases/03-durable-execution/03-02-PLAN.md create mode 100644 .planning/phases/03-durable-execution/03-03-PLAN.md create mode 100644 .planning/phases/04-distributed-and-extensibility/04-01-PLAN.md create mode 100644 .planning/phases/04-distributed-and-extensibility/04-02-PLAN.md create mode 100644 .planning/phases/04-distributed-and-extensibility/04-03-PLAN.md create mode 100644 .planning/phases/05-operational-excellence/05-01-PLAN.md create mode 100644 .planning/phases/05-operational-excellence/05-02-PLAN.md create mode 100644 .planning/phases/05-operational-excellence/05-03-PLAN.md create mode 100644 docs/HARNESS_DEPLOYMENT.md create mode 100644 docs/HARNESS_EXCELLENCE_PLAN.md create mode 100644 docs/skill-isolation.md create mode 100644 jvagent/harness/__init__.py create mode 100644 jvagent/harness/contracts.py create mode 100644 jvagent/harness/provider.py create mode 100644 jvagent/harness/release.py create mode 100644 jvagent/harness/runtime.py create mode 100644 tests/conformance/conftest.py create mode 100644 tests/conformance/fixtures/fake_host/README.md create mode 100644 tests/conformance/fixtures/fake_host/__init__.py create mode 100644 tests/conformance/test_delivery_replay.py create mode 100644 tests/conformance/test_identity_isolation.py create mode 100644 tests/conformance/test_invocation_recovery.py create mode 100644 tests/conformance/test_leases.py create mode 100644 tests/conformance/test_process_local_baseline.py create mode 100644 tests/conformance/test_provider_contract.py create mode 100644 tests/conformance/test_release_matrix.py create mode 100644 tests/conformance/test_snapshot_revocation.py create mode 100644 tests/conformance/test_traces.py create mode 100644 tests/harness/test_contracts.py create mode 100644 tests/harness/test_runtime.py diff --git a/.planning/GLOSSARY.md b/.planning/GLOSSARY.md index ec280143..b174e9a5 100644 --- a/.planning/GLOSSARY.md +++ b/.planning/GLOSSARY.md @@ -70,6 +70,18 @@ Former Rails-pattern router (weight `-200`). Removed in favor of `OrchestratorIn ### `InteractWalker` jvspatial `Walker` subclass that drives the interact subsystem. Source: `jvagent/action/interact/interact_walker.py:47+`. Bootstraps `User` / `Conversation` / `Interaction` and visits each top-level `InteractAction` in `weight` order. +### `NativeCaller` +Admission identity `(agent_id, user_id, session_id)`. Host scopes map to `session_id` outside jvagent. Source: [`jvagent/harness/contracts.py`](../jvagent/harness/contracts.py). See ADR-0054. + +### `HarnessRuntime` +Store-backed harness runtime: snapshots, TurnRun journal, invocation ledger, outbox, session leases, traces, skill staging. Share a `HarnessStore` for two-worker tests. Source: [`jvagent/harness/runtime.py`](../jvagent/harness/runtime.py). + +### `ToolSurfaceSnapshot` +Immutable per-turn tool/skill surface keyed by `snapshot_id` + NativeCaller. Revoked/expired snapshots cannot dispatch. Source: [`jvagent/harness/contracts.py`](../jvagent/harness/contracts.py). + +### `TurnRun` +Log-shaped execution journal (not a conversation Node). States: accepted → running → waiting_tool | waiting_approval → terminal / recovery_required. Source: [`jvagent/harness/runtime.py`](../jvagent/harness/runtime.py). + ### `LanguageModelAction` Subclass of `BaseModelAction` for LLM providers. Source: `jvagent/action/model/language/base.py:345`. Concrete subclasses: Anthropic, OpenAI, OpenRouter, Ollama. diff --git a/.planning/MILESTONES.md b/.planning/MILESTONES.md new file mode 100644 index 00000000..70534c33 --- /dev/null +++ b/.planning/MILESTONES.md @@ -0,0 +1,16 @@ +# Milestones + +## v1 Orchestrator (shipped) + +Orchestrator as the single executive, unified tool surface, lean surfacing, identity/egress, two skill specs, CUCS. Historical plan: [`archive/EXECUTIVE-ROADMAP.md`](archive/EXECUTIVE-ROADMAP.md). + +Phases: pre-GSD (not numbered in this file). + +## v2.0 Harness Excellence (in progress) + +**Started:** 2026-09-17 +**Goal:** Host-neutral reliability and extensibility under many agents, users, and sessions. +**Phases:** 1–5 +**Source:** [`docs/HARNESS_EXCELLENCE_PLAN.md`](../docs/HARNESS_EXCELLENCE_PLAN.md) + +Not started until Phase 1 HP-00 contracts freeze. diff --git a/.planning/PROJECT.md b/.planning/PROJECT.md index 3b5a4f8f..55c1a9d8 100644 --- a/.planning/PROJECT.md +++ b/.planning/PROJECT.md @@ -1,7 +1,8 @@ # jvagent — Project Vision -> **Status**: Draft, AI-agent-maintained. Last review: 2026-05-17. +> **Status**: Draft, AI-agent-maintained. Last review: 2026-09-17. > **Companion docs**: [`SPEC.md`](SPEC.md) for normative semantics, [`architecture.md`](architecture.md) for diagrams, [`../README.md`](../README.md) for user-facing onboarding. +> **Current milestone**: [`ROADMAP.md`](ROADMAP.md) — v2.0 Harness Excellence. ## TL;DR @@ -24,6 +25,24 @@ The model is the pilot. Tools are the controls. Skills are the flight plan. ([so --- +## Current Milestone: v2.0 Harness Excellence + +**Goal:** Make jvagent the dependable, graph-native harness for many agents, users, and simultaneous sessions — without sacrificing model agency, Claude-skill compatibility, or host-neutrality. + +**North star:** [`docs/HARNESS_EXCELLENCE_PLAN.md`](../docs/HARNESS_EXCELLENCE_PLAN.md) + +**Target features:** +- Native identity `(agent_id, user_id, session_id)` as the only isolation key in core +- Immutable `ToolSurfaceSnapshot` for tools and skills; no cross-session cache leakage +- Graph-backed `TurnRun` journal, invocation ledger, and durable event outbox +- Host-neutral `HostCapabilityProvider` with embedded and remote adapters +- Signed skill manifests and selectable isolation backends +- Correlated trace/replay, load evidence, and a release compatibility matrix + +**Requirements:** [`REQUIREMENTS.md`](REQUIREMENTS.md) · **Roadmap:** [`ROADMAP.md`](ROADMAP.md) · **State:** [`STATE.md`](STATE.md) + +--- + ## Target workloads ### 1. Turn-based conversational agents @@ -106,7 +125,39 @@ This repo is `jvagent` only. The graph framework is at `../jvspatial` (sibling d ## Roadmap -In-flight planning lives at [`EXECUTIVE-ROADMAP.md`](archive/EXECUTIVE-ROADMAP.md) (archived now that v1 has shipped). When this project adopts the GSD workflow, roadmaps move to a `ROADMAP.md` at the `.planning/` root. +- **v1 Orchestrator** — shipped. Historical plan: [`archive/EXECUTIVE-ROADMAP.md`](archive/EXECUTIVE-ROADMAP.md). +- **v2.0 Harness Excellence** — active. [`ROADMAP.md`](ROADMAP.md), sourced from [`docs/HARNESS_EXCELLENCE_PLAN.md`](../docs/HARNESS_EXCELLENCE_PLAN.md). + +## Key Decisions + +| Decision | Rationale | Outcome | +|----------|-----------|---------| +| Native identity only — no workspace/org/App in core | Hosts map their scopes to session ids; jvagent stays reusable | — Pending v2.0 | +| Snapshots, never live host imports into the Orchestrator | Prevents cache leakage and host-domain coupling | — Pending v2.0 | +| Authority bound server-side, never in model payloads | Model-generated JSON cannot escalate capability | — Pending v2.0 | +| Subprocess limits are development-only containment | Not a sandbox for untrusted code | — Pending v2.0 | +| Single-process remains a documented narrower profile | Active-active is optional, not required | — Pending v2.0 | +| HP-08 and HP-09 run in parallel after HP-03 + HP-05 | Skill hardening does not wait on host transport | — Pending v2.0 | + +## Evolution + +This document evolves at phase transitions and milestone boundaries. + +**After each phase transition:** +1. Requirements invalidated? → Move to Out of Scope with reason +2. Requirements validated? → Move to Validated with phase reference +3. New requirements emerged? → Add to Active +4. Decisions to log? → Add to Key Decisions +5. "What This Is" still accurate? → Update if drifted + +**After each milestone:** +1. Full review of all sections +2. Core Value check — still the right priority? +3. Audit Out of Scope — reasons still valid? +4. Update Context with current state + +--- +*Last updated: 2026-09-17 after starting milestone v2.0 Harness Excellence* --- diff --git a/.planning/README.md b/.planning/README.md index 0fc9e98c..4beac0b5 100644 --- a/.planning/README.md +++ b/.planning/README.md @@ -10,7 +10,13 @@ records it links into. User-facing onboarding lives in the root ``` .planning/ - PROJECT.md big-picture overview + PROJECT.md big-picture overview + current milestone + REQUIREMENTS.md v2.0 Harness Excellence requirements (REQ-IDs) + ROADMAP.md GSD phases 1–5 (HP-00 … HP-12) + STATE.md living execution position + MILESTONES.md v1 shipped / v2.0 in progress + config.json GSD workflow config + phases/ one PLAN.md per HP SPEC.md normative semantics (invariants, contracts) PATTERNS.md deployment patterns (Rails vs. Orchestrator) architecture.md diagrams (boot, interact, executive, pruning) @@ -19,7 +25,7 @@ records it links into. User-facing onboarding lives in the root runbooks/ step-by-step operator/dev procedures adr/ architecture decision records (immutable once accepted) specs/ design specs for feature work (agent-authored) - plans/ task-by-task implementation plans (agent-authored) + plans/ historical task-by-task plans (pre-GSD) archive/ superseded / shipped-and-historical docs ``` @@ -34,6 +40,7 @@ by slug (e.g. `specs/-foo-design.md` ↔ `plans/-foo.md`). | You want to… | Read | |---|---| | Get the big picture | [`PROJECT.md`](PROJECT.md) | +| Execute v2.0 Harness Excellence | [`ROADMAP.md`](ROADMAP.md) · [`REQUIREMENTS.md`](REQUIREMENTS.md) · [`../docs/HARNESS_EXCELLENCE_PLAN.md`](../docs/HARNESS_EXCELLENCE_PLAN.md) | | Look up normative semantics | [`SPEC.md`](SPEC.md) | | Choose a deployment pattern | [`PATTERNS.md`](PATTERNS.md) | | See diagrams | [`architecture.md`](architecture.md) | @@ -85,6 +92,7 @@ those records covered patterns (bridge/helm/cockpit) that were removed. | [0018](adr/0018-lean-tool-surfacing.md) | Lean tool surfacing (threshold-auto progressive tool disclosure) | Accepted | | [0026](adr/0026-task-driven-turn-lock.md) | Task-driven turn-lock (work-stack orchestration) | Accepted | | [0027](adr/0027-conversation-use-case-spec.md) | Conversation Use Case Specification (CUCS) | Accepted | +| [0054](adr/0054-harness-contracts.md) | Host-neutral harness contracts (NativeCaller, TurnRun, snapshot, provider) | Accepted | ## specs/ — design specs diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md new file mode 100644 index 00000000..a47a7be4 --- /dev/null +++ b/.planning/REQUIREMENTS.md @@ -0,0 +1,94 @@ +# Requirements: jvagent v2.0 Harness Excellence + +**Defined:** 2026-09-17 +**Core Value:** A dependable, graph-native harness for many agents, users, and sessions — model as pilot, tools as controls, skills as flight plan. +**Source:** [`docs/HARNESS_EXCELLENCE_PLAN.md`](../docs/HARNESS_EXCELLENCE_PLAN.md) +**Conformance:** HC-01 … HC-12 in that plan map 1:1 onto the IDs below. + +## v2.0 Requirements + +### Contracts and baseline + +- [ ] **CTRT-01**: A native, embedded, or remote integrator can run the same harness conformance suite against frozen TurnRun, snapshot, invocation, event, and provider fixtures; rejected transitions are explicit; no host-domain field appears in public jvagent models or APIs (HC-01 contract half) +- [ ] **BASE-01**: An operator can name every process-local component, its scope, its replacement decision, and the regression test that proves restart or multi-worker loss + +### Identity and snapshots + +- [ ] **IDNT-01**: Concurrent callers on distinct agents, users, and sessions stay isolated using only `(agent_id, user_id, session_id)` — no host-specific fields in jvagent core (HC-01) +- [ ] **IDNT-02**: Concurrent session admission across workers produces one User and one Conversation for the same identity; same-session ownership policy is explicit and tested +- [ ] **SNAP-01**: A tool or skill snapshot cannot leak across sessions or be reused after expiry or revocation (HC-02) +- [ ] **SNAP-02**: Dynamic tool and skill changes take effect on the next snapshot without contaminating any other in-flight caller + +### Durable execution + +- [ ] **RUN-01**: A crash before, during, or after tool dispatch has an explicit recovery result and never silently duplicates a supported side effect (HC-03) +- [ ] **RUN-02**: Model outage, retry, fallback, budget exhaustion, cancellation, and tool timeout leave an inspectable terminal run state (HC-09) +- [ ] **INV-01**: Retries reuse the same `invocation_id`; mutating native tools declare idempotency class; non-retryable tools produce a typed recovery state +- [ ] **DELV-01**: SSE and channel delivery replay events in order using cursors and render each final response once (HC-04) + +### Distributed runtime and extensibility + +- [ ] **DIST-01**: Two workers can serve different sessions concurrently and coordinate same-session ownership correctly (HC-05) +- [ ] **DIST-02**: Worker loss preserves queued delivery and either resumes or safely marks active runs for recovery (HC-06) +- [ ] **HOST-01**: Native, embedded-host, and remote-host tool providers pass the same invocation and revocation contract suite (HC-08) +- [ ] **HOST-02**: A sample host can supply per-session dynamic tools and skills; revocation takes effect at the next snapshot; jvagent remains unaware of the host's data model +- [ ] **SKIL-01**: JV and Claude skill bundles materialize from verified manifests into isolated caller slices (HC-07) +- [ ] **SKIL-02**: Skill activation is reproducible from its digest; a revoked or changed skill cannot run under a stale snapshot; untrusted script skills are refused without an approved isolation backend + +### Operational excellence + +- [ ] **OBSV-01**: An operator can explain any completed or failed turn from one correlation id without accessing another user's data (HC-10) +- [ ] **PERF-01**: Load tests preserve p95 targets and event ordering under many users and sessions; no optimization weakens ordering, identity isolation, or egress (HC-11) +- [ ] **REL-01**: A release record identifies artifact digest, contract versions, supported topology, evidence, limitations, and rollback path (HC-12) + +## Future (not this milestone) + +- Which durable transport is first for event outbox and distributed coordination +- Separate checkpoint vs event retention policies per backend +- Whether background work uses the same TurnRun executor or a sibling durable worker +- External skill publisher registry and revocation service (after signed manifests and isolation backends prove out) + +## Out of Scope + +| Feature | Reason | +|---------|--------| +| Host concepts (workspaces, organizations, Apps, domain schemas) in jvagent core | Hosts map scopes to session ids; jvagent stays host-neutral | +| Semantic router / workflow designer / business-rule engine in the Orchestrator | Thin harness: judgment stays in skills and the model | +| A second memory database competing with jvspatial graph state | Graph remains the runtime state substrate | +| Exactly-once for third-party side effects with no idempotency mechanism | Harness supplies invocation identity; domain tools own exactly-once | +| Treating subprocess resource limits as a sandbox for untrusted code | Dev-only containment; untrusted scripts need an approved isolation backend | +| Requiring active-active for every deployment | Single-process remains supported with explicitly narrower guarantees | +| Embedding Integral or any other product's model in tests | HP-08 uses a small independent host fixture; `examples/jvagent_app` is the native reference | + +## Traceability + +| Requirement | Phase | HP | Status | +|-------------|-------|----|--------| +| CTRT-01 | Phase 1 | HP-00 | Pending | +| BASE-01 | Phase 1 | HP-01 | Pending | +| IDNT-01 | Phase 2 | HP-02 | Pending | +| IDNT-02 | Phase 2 | HP-02 | Pending | +| SNAP-01 | Phase 2 | HP-03 | Pending | +| SNAP-02 | Phase 2 | HP-03 | Pending | +| RUN-01 | Phase 3 | HP-04 | Pending | +| RUN-02 | Phase 3 | HP-04 | Pending | +| INV-01 | Phase 3 | HP-05 | Pending | +| DELV-01 | Phase 3 | HP-06 | Pending | +| DIST-01 | Phase 4 | HP-07 | Pending | +| DIST-02 | Phase 4 | HP-07 | Pending | +| HOST-01 | Phase 4 | HP-08 | Pending | +| HOST-02 | Phase 4 | HP-08 | Pending | +| SKIL-01 | Phase 4 | HP-09 | Pending | +| SKIL-02 | Phase 4 | HP-09 | Pending | +| OBSV-01 | Phase 5 | HP-10 | Pending | +| PERF-01 | Phase 5 | HP-11 | Pending | +| REL-01 | Phase 5 | HP-12 | Pending | + +**Coverage:** +- v2.0 requirements: 19 total +- Mapped to phases: 19 +- Unmapped: 0 ✓ + +--- +*Requirements defined: 2026-09-17* +*Last updated: 2026-09-17 after milestone v2.0 roadmap* diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md new file mode 100644 index 00000000..35c0a760 --- /dev/null +++ b/.planning/ROADMAP.md @@ -0,0 +1,129 @@ +# Roadmap: jvagent + +## Milestones + +- ✅ **v1 Orchestrator** — shipped (see [`.planning/archive/EXECUTIVE-ROADMAP.md`](archive/EXECUTIVE-ROADMAP.md)) +- 🚧 **v2.0 Harness Excellence** — Phases 1–5 (in progress) +- Source plan: [`docs/HARNESS_EXCELLENCE_PLAN.md`](../docs/HARNESS_EXCELLENCE_PLAN.md) + +## Overview + +Freeze host-neutral contracts and inventory process-local state. Then isolate identity and snapshots. Then persist turns, invocations, and delivery. Then add optional active-active plus host/skill hardening. Then prove it with traces, load, and a release record. + +**Do not** claim crash-safe or active-active behavior until Phase 3 (HP-06) and Phase 4 (HP-07) evidence exists. + +**Parallelism after Phase 1 freeze:** HP-02, HP-03, and HP-04 may start together. HP-08 and HP-09 run in parallel after HP-03 + HP-05 (HP-09 does **not** wait on HP-08). + +## Phases + +**Phase Numbering:** first GSD milestone; numbering starts at 1. + +- [x] **Phase 1: Contracts and baseline** — Freeze harness contracts and inventory process-local state +- [x] **Phase 2: Identity and snapshots** — Store-backed admission and snapshot-scoped caches +- [x] **Phase 3: Durable execution** — TurnRun, invocation ledger, durable outbox +- [x] **Phase 4: Distributed runtime and extensibility** — Leases, host provider, skill hardening +- [x] **Phase 5: Operational excellence and release proof** — Trace/replay, capacity, release record + +## Phase Details + +### Phase 1: Contracts and baseline + +**Goal**: Freeze the host-neutral identity, snapshot, TurnRun, invocation, event, and provider contracts, and attach a concrete inventory of process-local state. +**Depends on**: Nothing (first phase) +**Requirements**: CTRT-01, BASE-01 +**Success Criteria** (what must be TRUE): + 1. Contract fixtures define valid and rejected transitions for TurnRun, snapshot, invocation, event, and provider APIs + 2. No host-domain field (`workspace_id`, org, App, domain schema) appears in public jvagent models or APIs + 3. Every process-local cache, lock, bus, breaker, and background registry has an owner, scope, replacement decision, and regression target +**Plans**: 2 plans + +Plans: +- [x] 01-01: HP-00 Harness contract ADRs and conformance suite +- [x] 01-02: HP-01 Baseline reliability audit + +### Phase 2: Identity and snapshots + +**Goal**: Make `(agent_id, user_id, session_id)` the admission key and serve tools/skills only through immutable snapshots. +**Depends on**: Phase 1 +**Requirements**: IDNT-01, IDNT-02, SNAP-01, SNAP-02 +**Success Criteria** (what must be TRUE): + 1. Concurrent creates across workers produce one User and one Conversation + 2. Simultaneous turns on distinct sessions remain isolated; same-session policy is explicit and tested + 3. Two concurrent users/sessions receive only their own snapshots + 4. Dynamic tool/skill changes affect a new snapshot without contaminating any other caller +**Plans**: 2 plans + +Plans: +- [x] 02-01: HP-02 Native identity and session admission +- [x] 02-02: HP-03 ToolSurfaceSnapshot and cache discipline + +### Phase 3: Durable execution + +**Goal**: Persist turn lifecycle, tool invocations, and outbound events so crashes and reconnects have an explicit story. +**Depends on**: Phase 2 (HP-04 may start against frozen Phase 1 fixtures in parallel with HP-02) +**Requirements**: RUN-01, RUN-02, INV-01, DELV-01 +**Success Criteria** (what must be TRUE): + 1. A crash after tool dispatch is diagnosable; completed read tools are not repeated unnecessarily; unsafe writes require a visible recovery decision + 2. Retries reuse `invocation_id`; a duplicate dispatch cannot duplicate a supported mutating effect + 3. Reconnecting clients replay missed frames in order without duplicate rendered messages + 4. A reply created on one worker can be delivered by another (outbox, not process-local bus) +**Plans**: 3 plans + +Plans: +- [x] 03-01: HP-04 TurnRun journal and resumable execution +- [x] 03-02: HP-05 Invocation ledger and idempotency adapters +- [x] 03-03: HP-06 Durable event outbox and resumable streaming + +### Phase 4: Distributed runtime and extensibility + +**Goal**: Optional active-active coordination, a host-neutral capability provider, and signed/isolated skill materialization. +**Depends on**: Phase 3 +**Requirements**: DIST-01, DIST-02, HOST-01, HOST-02, SKIL-01, SKIL-02 +**Success Criteria** (what must be TRUE): + 1. A two-worker test handles concurrent users, session contention, worker loss, and proactive delivery without lost or cross-delivered events + 2. A sample host supplies per-session dynamic tools and skills; revocation takes effect at the next snapshot; jvagent never sees the host data model + 3. Native, embedded, and remote providers pass the same invocation/revocation suite + 4. Skill activation is reproducible from digest; stale or revoked skills cannot run; untrusted scripts refuse without an approved isolation backend +**Plans**: 3 plans + +Plans: +- [x] 04-01: HP-07 Active-active coordination +- [x] 04-02: HP-08 HostCapabilityProvider reference implementation +- [x] 04-03: HP-09 Skill package and execution hardening + +HP-08 and HP-09 may execute in parallel. Both require HP-03 + HP-05, not each other. + +### Phase 5: Operational excellence and release proof + +**Goal**: An operator can explain any turn, capacity is measured, and a release artifact carries its guarantees. +**Depends on**: Phase 4 (HP-10 may start after Phase 3; HP-11 needs HP-03 + HP-06 + HP-07) +**Requirements**: OBSV-01, PERF-01, REL-01 +**Success Criteria** (what must be TRUE): + 1. One correlation id reconstructs a completed or failed turn with redaction and no other user's content + 2. Representative many-user/many-session load preserves p95 targets and event ordering + 3. A release record lists artifact digest, contract versions, supported topology, evidence, limitations, and rollback path + 4. Unsupported storage/execution combinations are marked explicitly +**Plans**: 3 plans + +Plans: +- [x] 05-01: HP-10 Trace, replay, and evaluation plane +- [x] 05-02: HP-11 Performance and capacity work +- [x] 05-03: HP-12 Release and compatibility evidence + +## Progress + +**Execution Order:** +Phases execute in numeric order. Inside a phase, plans may run in parallel when the HP dependency map allows. + +| Phase | Milestone | Plans Complete | Status | Completed | +|-------|-----------|----------------|--------|-----------| +| 1. Contracts and baseline | v2.0 | 2/2 | Complete | 2026-09-17 | +| 2. Identity and snapshots | v2.0 | 2/2 | Complete | 2026-09-18 | +| 3. Durable execution | v2.0 | 3/3 | Complete | 2026-09-18 | +| 4. Distributed runtime and extensibility | v2.0 | 3/3 | Complete | 2026-09-18 | +| 5. Operational excellence and release proof | v2.0 | 3/3 | Complete | 2026-09-18 | + +**Coverage:** 19/19 requirements mapped. Unmapped: 0. + +--- +*Roadmap created: 2026-09-17 for milestone v2.0 Harness Excellence* diff --git a/.planning/SPEC.md b/.planning/SPEC.md index 0f68a52f..cc4fcb2d 100644 --- a/.planning/SPEC.md +++ b/.planning/SPEC.md @@ -124,6 +124,21 @@ Rationale and consequences: [`adr/0012-skill-executive-architecture.md`](adr/001 Harness design contract (thin server, thick SOP): [`docs/thin-harness.md`](../docs/thin-harness.md). Interview profile: [`jvagent/action/interview/docs/thin-harness.md`](../jvagent/action/interview/docs/thin-harness.md). +### 3.4 Harness contracts (ADR-0054) + +Types and validators live in [`jvagent/harness/contracts.py`](../jvagent/harness/contracts.py) (`CONTRACT_VERSION`). Runtime (journals, ledger, outbox, leases, snapshots, traces) is [`jvagent/harness/runtime.py`](../jvagent/harness/runtime.py). TurnRun is a log-shaped journal (I-GRAPH-02), not a conversation Node. + +- `NativeCaller` ([`contracts.py:131`](../jvagent/harness/contracts.py)) is `(agent_id, user_id, session_id)`. `native_caller_from_mapping` ([`contracts.py:149`](../jvagent/harness/contracts.py)) rejects host-domain keys (`workspace_id`, `organization`, `organization_id`, `org_id`, `content_profile_id`). Public interact/embed `data` payloads are rejected the same way. +- `TurnRunState` + `assert_turn_run_transition` ([`contracts.py:42`](../jvagent/harness/contracts.py), [`contracts.py:122`](../jvagent/harness/contracts.py)) define legal lifecycle edges. `HarnessRuntime.start_turn` / `transition` persist them. +- `ToolSurfaceSnapshot` ([`contracts.py:167`](../jvagent/harness/contracts.py)) is immutable; `cache_key()` includes `snapshot_id` and NativeCaller; revoked/expired snapshots raise `HarnessContractError`. Catalog cache keys match ([`catalog.py`](../jvagent/action/orchestrator/catalog.py)). +- `InvocationRecord` ([`contracts.py:193`](../jvagent/harness/contracts.py)) and `IdempotencyClass` ([`contracts.py:53`](../jvagent/harness/contracts.py)) describe dispatch identity. Ledger: `HarnessRuntime.begin_invocation` / `finish_invocation`; `@tool(idempotency_class=...)`. +- `EventEnvelope` ([`contracts.py:204`](../jvagent/harness/contracts.py)) is the durable-delivery shape (`sequence >= 1`). ResponseBus appends to the outbox before fan-out; SSE may replay via `cursor`. +- `HostCapabilityProvider` ([`contracts.py:239`](../jvagent/harness/contracts.py)) is an async Protocol. Adapters: [`provider.py`](../jvagent/harness/provider.py) (`native` / `embedded` / `remote`). Model payloads must not carry authority keys (`reject_model_authority_fields`, [`contracts.py:113`](../jvagent/harness/contracts.py)). +- Same-session policy is **lease** (`SessionBusy` if another worker holds it). JSON/SQLite active-active is **unsupported** ([`docs/HARNESS_DEPLOYMENT.md`](../docs/HARNESS_DEPLOYMENT.md)). +- Conformance suite: `tests/conformance/` (pytest marker `harness_conformance`). Native reference app: `examples/jvagent_app`. Independent host fixture: `tests/conformance/fixtures/fake_host/` (not a product host). + +See [ADR-0054](adr/0054-harness-contracts.md). + --- ## 4. Action contract @@ -188,7 +203,7 @@ Errors raised by these hooks are logged automatically by the action's `enable()` ### 4.4 Tools and capabilities -- `get_tools() -> List[Tool]` ([`base.py:259`](../jvagent/action/base.py)) — every `Action` MAY expose tools to the agentic loop (e.g. the Orchestrator's think-act-observe loop). Each tool wraps a callable with a JSON Schema for arguments; they are registered with an `action__` prefix in the tool registry. `InteractAction.get_tools()` forwards to `execute(visitor)` and builds the tool description from the manifest (`purpose` + `activates_on`, via `routing_triggers()`). +- `get_tools() -> List[Tool]` ([`base.py:259`](../jvagent/action/base.py)) — every `Action` MAY expose tools to the agentic loop (e.g. the Orchestrator's think-act-observe loop). Each tool wraps a callable with a JSON Schema for arguments; they are registered with an `action__` prefix in the tool registry. `InteractAction.get_tools()` forwards to `execute(visitor)` and builds the tool description from the manifest (`purpose` + `activates_on`, via `routing_triggers()`). Target admission surface is a `ToolSurfaceSnapshot` ([`contracts.py:163`](../jvagent/harness/contracts.py)); today's cache is still per-agent ([`catalog.py:45`](../jvagent/action/orchestrator/catalog.py)) until HP-03. - `get_capabilities() -> List[str]` ([`base.py:180`](../jvagent/action/base.py)) — short capability strings aggregated by `ReplyAction` for reply-prompt injection. ### 4.5 Action discovery @@ -219,7 +234,8 @@ See [`adr/0004-namespace-isolation.md`](adr/0004-namespace-isolation.md) and [`a ### 5.1 Identity - `User.memory_id` + `User.user_id` together form a compound unique key per `Memory` subgraph (compound index at `memory/user.py:16-24`). -- A `lock_manager` (`memory/lock_manager.py`) acquires a per-`(memory_id, user_id)` lock before `_get_user_unlocked()` to prevent duplicate `User` rows under concurrent creates. +- Harness admission identity is `NativeCaller(agent_id, user_id, session_id)` ([`contracts.py:131`](../jvagent/harness/contracts.py)). `Conversation.session_id` remains globally unique ([`conversation.py`](../jvagent/memory/conversation.py)). `get_user` / `get_session` wrap `distributed_lease` plus the in-process lock; two-worker identity upsert is proven on a shared `HarnessStore` ([`runtime.py`](../jvagent/harness/runtime.py)). Redis/Dynamo still required for cluster-wide User/Conversation uniqueness on JSON. +- A `lock_manager` (`memory/lock_manager.py`) acquires a per-`(memory_id, user_id)` lock before `_get_user_unlocked()` to prevent duplicate `User` rows under concurrent creates. The lock is process-local today. ### 5.2 Conversation chaining @@ -290,7 +306,7 @@ See [`adr/0005-app-yaml-agent-yaml-split.md`](adr/0005-app-yaml-agent-yaml-split ## 7. Response bus -The response bus ([`jvagent/action/response/response_bus.py`](../jvagent/action/response/response_bus.py)) is **per-agent**. Each `Agent` lazily constructs one via `Agent.get_response_bus()` ([`agent.py:256`](../jvagent/core/agent.py)). +The response bus ([`jvagent/action/response/response_bus.py`](../jvagent/action/response/response_bus.py)) is **per-agent**. Each `Agent` lazily constructs one via `Agent.get_response_bus()` ([`agent.py:256`](../jvagent/core/agent.py)). Session queues and subscribers are process-local (`_agent_bus_registry`, `_session_queues`). Target durable shape is `EventEnvelope` ([`contracts.py:200`](../jvagent/harness/contracts.py)); outbox persistence is HP-06. - Channel adapters (`EmailAction`, `WhatsAppAction`, `FacebookAction`, etc.) register with the bus and translate messages to channel-specific transports. - Filters can drop, transform, or duplicate messages per channel. @@ -363,6 +379,8 @@ There is **no external task queue** (no Celery / RQ). Long-lived autonomous work 10. Flow continuation is configurable via `lock_active_flow` ([ADR-0013](adr/0013-togglable-deterministic-turn-lock.md)). When on (default), the active flow's IA tool is dispatched with no model round-trip; when off, the flow is surfaced as routable context and the model decides. See §3.3 invariants 2–3. 11. Routing is tool selection. There is no separate router or capability registry; IAs (as tools), persona, core services, and skills are all tools. A flow's control-task (turn-lock) is persisted on the conversation `TaskStore`; the active flow is surfaced as a routable tool and continued by model tool selection next turn. See §3.3 invariant 4. 12. Access control gates tool dispatch (`tool:*`), including IA-as-tool execution (`tool:delegate:{name}`); a denial routes to the orchestrator's safe-fallback. See §3.3 invariant 6. +13. Public harness types use `NativeCaller` only — no host-domain fields in `jvagent.harness` ([ADR-0054](adr/0054-harness-contracts.md), [`contracts.py:13`](../jvagent/harness/contracts.py)). +14. Model-generated tool payloads MUST NOT carry authority keys (`reject_model_authority_fields`, [`contracts.py:111`](../jvagent/harness/contracts.py)). --- @@ -385,6 +403,7 @@ Load-bearing design choices are captured as ADRs: - [`adr/0010-executive-centers-architecture.md`](adr/0010-executive-centers-architecture.md) *(superseded by ADR-0012; retained as history)* - [`adr/0011-skills-two-kinds.md`](adr/0011-skills-two-kinds.md) - [`adr/0012-skill-executive-architecture.md`](adr/0012-skill-executive-architecture.md) +- [`adr/0054-harness-contracts.md`](adr/0054-harness-contracts.md) --- diff --git a/.planning/STATE.md b/.planning/STATE.md new file mode 100644 index 00000000..43212194 --- /dev/null +++ b/.planning/STATE.md @@ -0,0 +1,75 @@ +# Project State + +## Project Reference + +See: .planning/PROJECT.md (updated 2026-09-17) + +**Core value:** Dependable graph-native harness — model as pilot, tools as controls, skills as flight plan. +**Current focus:** v2.0 Harness Excellence — Phases 1–5 implemented; awaiting user commit/PR + +## Current Position + +Phase: 5 of 5 (Operational excellence and release proof) +Plan: 3 of 3 in current phase +Status: Implementation complete; not committed +Last activity: 2026-09-18 — HP-02 … HP-12 runtime + wires + +Progress: [██████████] 100% + +## Performance Metrics + +**Velocity:** +- Total plans completed: 13 (uncommitted) +- Average duration: — +- Total execution time: — + +**By Phase:** + +| Phase | Plans | Total | Avg/Plan | +|-------|-------|-------|----------| +| 1 | 2 implemented | 2 | — | +| 2 | 2 implemented | 2 | — | +| 3 | 3 implemented | 3 | — | +| 4 | 3 implemented | 3 | — | +| 5 | 3 implemented | 3 | — | + +## Accumulated Context + +### Decisions + +- Native identity only; no host product concepts in core +- Snapshots, never live host imports into the Orchestrator +- Authority server-side, never in model payloads +- HostCapabilityProvider methods are async +- `turn_cache` ContextVar kept +- Fake host fixture, not Integral +- Same-session policy is **lease** +- TurnRun is a journal Object (I-GRAPH-02), not a conversation Node +- JSON/SQLite active-active is unsupported +- Subprocess ≠ sandbox + +### Pending Todos + +User will commit and open PR. + +### Blockers/Concerns + +- Durable outbox transport swap still deferred (HarnessStore is the first backend) +- Active-active for Mongo/Postgres marked degraded until Redis/Dynamo leases are configured + +## Deferred Items + +| Category | Item | Status | Deferred At | +|----------|------|--------|-------------| +| Transport | First durable outbox/coordination transport | Future | v2.0 start | +| Retention | Separate checkpoint vs event retention per backend | Future | v2.0 start | +| Workers | Background work vs TurnRun executor | Future | v2.0 start | +| Skills | External publisher registry + revocation service | Future | v2.0 start | + +## Session Continuity + +Last session: 2026-09-18 +Stopped at: Phases 2–5 implemented (HP-02 … HP-12) +Resume file: None + +Next: user commit + PR diff --git a/.planning/adr/0054-harness-contracts.md b/.planning/adr/0054-harness-contracts.md new file mode 100644 index 00000000..65e792fb --- /dev/null +++ b/.planning/adr/0054-harness-contracts.md @@ -0,0 +1,80 @@ +# ADR 0054 — Host-neutral harness contracts + +**Status**: Accepted +**Date**: 2026-09-17 +**Relation**: Extends [ADR-0012](0012-skill-executive-architecture.md) (thin orchestrator), [ADR-0014](0014-identity-on-agent-replyaction-egress.md) (identity/egress), [ADR-0018](0018-lean-tool-surfacing.md) (lean discovery), [ADR-0033](0033-identity-and-locking-substrate.md) (native identity tuples). Does not supersede them. Product roadmap: [`docs/HARNESS_EXCELLENCE_PLAN.md`](../../docs/HARNESS_EXCELLENCE_PLAN.md). + +--- + +## 1. Context + +jvagent already isolates users with `(memory_id, user_id)` and conversations with `session_id`. Hosts (embedded products) still risk leaking domain fields into core, serving process-global tool/skill caches across callers, and treating in-memory buses as durable delivery. + +v2.0 Harness Excellence needs a frozen, host-neutral contract **before** persistence, outbox, or host adapters land. This ADR is that freeze. Runtime wiring is later packages (HP-02 … HP-08). + +## 2. Decision + +### 2.1 NativeCaller + +The only public admission identity is: + +```text +(agent_id, user_id, session_id) +``` + +Implemented as `NativeCaller` in [`jvagent/harness/contracts.py`](../../jvagent/harness/contracts.py). Host scopes (workspaces, organizations, Apps, domain schemas) map to `session_id` **outside** jvagent. Public models and APIs MUST reject `workspace_id`, `organization`, `organization_id`, `org_id`, and `content_profile_id`. + +### 2.2 TurnRun states + +```text +accepted → running → waiting_tool → waiting_approval → running + → completed | failed | cancelled | recovery_required +``` + +Illegal transitions raise `HarnessContractError`. Terminal states do not resume silently. `recovery_required` is explicit; it is not auto-replay. Persistence is HP-04. + +### 2.3 ToolSurfaceSnapshot + +At turn admission the Orchestrator will receive one immutable snapshot (`snapshot_id` + NativeCaller + native/host descriptors + expiry). Cache keys MUST include `snapshot_id`. A revoked or expired snapshot is unusable for new dispatch. In-flight turns keep the admitted snapshot unless the host explicitly revokes it (HP-03). + +Lean discovery (`find_tool`, `load_tool`, `find_skill`, `use_skill`) is unchanged. + +### 2.4 Invocation and events + +Every mutating dispatch will allocate `invocation_id` before the call (HP-05). Idempotency class is one of `idempotent` | `compensatable` | `non_retryable`. Exactly-once for third-party effects without an idempotency mechanism is out of scope. + +Outbound frames will use `EventEnvelope` (`session_id`, monotonic `sequence` ≥ 1, `cursor`, `message_id`, `correlation_id`, `snapshot_id`) persisted before fan-out (HP-06). Delivery is at-least-once; single-egress stays at ReplyAction / EgressGate. + +### 2.5 HostCapabilityProvider + +Optional protocol (async, matching jvagent I/O): + +```text +resolve_snapshot(caller) -> ToolSurfaceSnapshot +invoke(snapshot_id, invocation_id, tool_name, payload) -> ToolResult +load_skill(snapshot_id, skill_key) -> SkillMaterialization +invalidate(selector) -> None +``` + +Authority is bound server-side. Model-generated payloads MUST NOT carry `authority`, `trust_tier`, `capability_token`, `snapshot_secret`, or `isolation_backend`. + +Native, embedded, and remote transports share `tests/conformance/` (HP-08). The sample host is `tests/conformance/fixtures/fake_host/` — not another product's model. + +### 2.6 Guarantee split + +Single-process mode remains supported with narrower guarantees (process-local bus/caches). Active-active and crash-safe claims require HP-06 and HP-07 evidence. Subprocess resource limits are development-only containment, not a sandbox (HP-09). + +### 2.7 Thin harness + +This ADR adds reliability mechanics only. The Orchestrator does not gain semantic routing, intent classification, or host-domain workflows. See [`docs/thin-harness.md`](../../docs/thin-harness.md). + +## 3. Consequences + +- Contract tests run today; runtime isolation/outbox/provider tests are skipped until their HP. +- `register_host_skill_provider` remains until HP-08 replaces it; it is process-global and must not grow host-domain fields. +- ADR-0019 soft plan resume is not a TurnRun journal. + +## 4. Verification + +`tests/harness/test_contracts.py` and `tests/conformance/` (marker `harness_conformance`). +`tests/action/orchestrator/test_no_interview_coupling.py` still passes — harness types do not import interview. diff --git a/.planning/config.json b/.planning/config.json new file mode 100644 index 00000000..4464854a --- /dev/null +++ b/.planning/config.json @@ -0,0 +1,56 @@ +{ + "mode": "interactive", + "granularity": "standard", + "workflow": { + "research": true, + "plan_check": true, + "verifier": true, + "auto_advance": false, + "nyquist_validation": true, + "security_enforcement": true, + "security_asvs_level": 1, + "security_block_on": "high", + "discuss_mode": "discuss", + "research_before_questions": false, + "code_review_command": null, + "plan_bounce": false, + "plan_bounce_script": null, + "plan_bounce_passes": 2, + "cross_ai_execution": false, + "cross_ai_command": "", + "cross_ai_timeout": 300 + }, + "planning": { + "commit_docs": false, + "search_gitignored": false, + "sub_repos": [] + }, + "parallelization": { + "enabled": true, + "plan_level": true, + "task_level": false, + "skip_checkpoints": true, + "max_concurrent_agents": 3, + "min_plans_for_parallel": 2 + }, + "gates": { + "confirm_project": true, + "confirm_phases": true, + "confirm_roadmap": true, + "confirm_breakdown": true, + "confirm_plan": true, + "execute_next_plan": true, + "issues_review": true, + "confirm_transition": true + }, + "safety": { + "always_confirm_destructive": true, + "always_confirm_external_services": true + }, + "hooks": { + "context_warnings": true + }, + "project_code": "jvagent", + "agent_skills": {}, + "claude_md_path": "./CLAUDE.md" +} diff --git a/.planning/phases/01-contracts-and-baseline/01-01-PLAN.md b/.planning/phases/01-contracts-and-baseline/01-01-PLAN.md new file mode 100644 index 00000000..30a7ceab --- /dev/null +++ b/.planning/phases/01-contracts-and-baseline/01-01-PLAN.md @@ -0,0 +1,92 @@ +# HP-00 — Harness contract ADRs and conformance suite + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 1 — Contracts and baseline +**Package:** HP-00 +**Requirements:** CTRT-01 +**Depends on:** nothing +**Goal:** Freeze host-neutral TurnRun, snapshot, invocation, event, and provider contracts with fixtures that native, embedded, and remote integrations all run. + +**Architecture:** Contracts live as ADRs + SPEC + typed fixtures. No product features. Runtime may grow protocol modules (`jvagent/harness/` or similar) only as *types and validators*, not as a live Orchestrator rewrite. + +**Tech stack:** Python 3.12+, pytest, existing ADR/SPEC style. + +**Do not implement:** TurnRun persistence, outbox, host adapters, skill signing. + +--- + +## File structure + +| File | Responsibility | +|---|---| +| `.planning/adr/0054-harness-contracts.md` | Create. NativeCaller, TurnRun, snapshot, invocation, outbox, provider, non-leakage | +| `jvagent/harness/contracts.py` | Create. Dataclasses / Protocol types only | +| `tests/conformance/` | Create. Shared suite + valid/rejected fixtures | +| `.planning/SPEC.md` | Modify. Cite new contracts | +| `docs/ORCHESTRATOR.md` | Modify. Point at snapshot/admission; keep thin-harness language | +| `docs/HARNESS_EXCELLENCE_PLAN.md` | Modify. Mark HP-00 in progress | + +Forbidden in public types: `workspace_id`, `organization`, `App` (host), domain schema names. + +## NativeCaller (locked) + +```text +(agent_id, user_id, session_id) +``` + +Host scopes map to `session_id` outside jvagent. + +## TurnRun states (locked) + +```text +accepted → running → waiting_tool → waiting_approval → running + → completed | failed | cancelled | recovery_required +``` + +Each transition: monotonic seq, timestamp, reason, snapshot_id, correlation_id. + +## Provider protocol (locked) + +```text +resolve_snapshot(agent_id, user_id, session_id) -> ToolSurfaceSnapshot +invoke(snapshot_id, invocation_id, tool_name, payload) -> ToolResult +load_skill(snapshot_id, skill_key) -> SkillMaterialization +invalidate(snapshot_selector) -> acknowledgement +``` + +Authority is not a field on model-generated payloads. + +--- + +### Task 1: ADR-0054 + +- [x] Write ADR covering NativeCaller, TurnRun, ToolSurfaceSnapshot, invocation_id, event cursor, HostCapabilityProvider, cache non-leakage, single-process vs active-active guarantee split +- [x] Explicitly forbid host-domain fields in public models +- [x] Link thin-harness.md; state Orchestrator does not gain semantic routing + +### Task 2: Typed contracts module + +- [x] Add `jvagent/harness/contracts.py` (or equivalent) with NativeCaller, TurnRunState, ToolSurfaceSnapshot, InvocationRecord, EventEnvelope, HostCapabilityProvider Protocol +- [x] Validator rejects extra host-domain keys +- [x] Unit tests for legal vs illegal TurnRun transitions + +### Task 3: Conformance suite skeleton + +- [x] `tests/conformance/test_identity_isolation.py` — HC-01 fixtures (skip/xfail until HP-02) +- [x] `tests/conformance/test_snapshot_revocation.py` — HC-02 +- [x] `tests/conformance/test_invocation_recovery.py` — HC-03 +- [x] `tests/conformance/test_delivery_replay.py` — HC-04 +- [x] `tests/conformance/test_provider_contract.py` — HC-08; parametrize native / embedded / remote +- [x] Marker `harness_conformance`; native fixture uses `examples/jvagent_app` +- [x] Independent fake-host fixture directory under `tests/conformance/fixtures/fake_host/` — **not** Integral + +### Task 4: SPEC + docs + +- [x] SPEC § identity, tools, response bus: cite contract types +- [x] ORCHESTRATOR.md: snapshot at admission; lean discovery remains `find_tool` / `use_skill` +- [x] CHANGELOG Unreleased: contracts freeze (docs/types only) + +**Acceptance:** fixtures define valid and rejected transitions; `pytest tests/conformance -q` collects; no host-domain field in public types; Integrals/workspaces never appear. + +**Verify:** `pytest tests/conformance -q` and `pytest tests/action/orchestrator/test_no_interview_coupling.py -q` diff --git a/.planning/phases/01-contracts-and-baseline/01-02-PLAN.md b/.planning/phases/01-contracts-and-baseline/01-02-PLAN.md new file mode 100644 index 00000000..8111ca0f --- /dev/null +++ b/.planning/phases/01-contracts-and-baseline/01-02-PLAN.md @@ -0,0 +1,53 @@ +# HP-01 — Baseline reliability audit + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 1 — Contracts and baseline +**Package:** HP-01 +**Requirements:** BASE-01 +**Depends on:** HP-00 (contract names for the inventory columns) +**Goal:** Inventory every process-local component and record current failure behavior with regression targets. No replacements yet. + +**Architecture:** Audit document + characterization tests. Replacement decisions are recorded, not executed (HP-03/06/07 do the replacements). + +--- + +## Seed inventory (already mapped) + +| Component | Anchor | Likely replacement | +|---|---|---| +| ResponseBus queues/subscribers | `jvagent/action/response/response_bus.py:80,148` | HP-06 durable outbox | +| Tool surface cache (per-agent) | `jvagent/action/orchestrator/catalog.py:45` | HP-03 snapshot cache | +| Skill discovery cache | `jvagent/action/orchestrator/skills.py:18` | HP-03 snapshot cache | +| Host skill providers (process-global) | `jvagent/action/orchestrator/skill_providers.py:21` | HP-08 provider protocol | +| MODEL_BREAKER | `jvagent/action/model/resilience.py:146` | HP-07 shared backend optional | +| In-process conversation locks | `jvagent/memory/lock_manager.py`, `distributed_conversation_lock.py` | HP-02/07 store-backed | +| Embed in-flight tasks | `jvagent/embed/interact.py:18` | HP-04 TurnRun | +| MCP user clients | `jvagent/action/mcp/mcp_action.py:118` | document; out of harness core | +| Webhook wamid dedup | `jvagent/action/utils/meta_webhook_dedup.py:26` | document; channel-local | +| Turn ContextVar cache | `jvagent/action/orchestrator/turn_cache.py:27` | **keep** — already per-task | + +--- + +### Task 1: Written inventory + +- [x] Create `.planning/phases/01-contracts-and-baseline/PROCESS-LOCAL-STATE.md` +- [x] Columns: owner, process-scope, identity key (or none), survives restart?, cross-worker leak?, replacement HP, regression test +- [x] Cover bus, caches, breakers, locks, background registries, sandbox staging, rate limiters + +### Task 2: Characterization tests + +- [x] Restart: in-flight SSE subscriber gone (today) — assert current behavior, mark as HP-06 target +- [x] Duplicate delivery on reconnect with overlapping replay (`tests/action/response/test_streaming_dedup.py` already exists — cite) +- [x] Concurrent session turns — isolate vs collide +- [x] Model error / interrupted tool — current terminal state +- [x] Benchmark fixtures (stubs ok): short chat, tool-rich, streaming, long session, many-user + +### Task 3: Bind to conformance + +- [x] Each inventory row cites a `tests/conformance/` or existing test path +- [x] CHANGELOG: audit artifact only + +**Acceptance:** every process-local component has owner, scope, replacement decision, regression target. + +**Verify:** inventory complete vs grep for module-level `_registry` / `_cache` / `_locks` in `jvagent/` diff --git a/.planning/phases/01-contracts-and-baseline/PROCESS-LOCAL-STATE.md b/.planning/phases/01-contracts-and-baseline/PROCESS-LOCAL-STATE.md new file mode 100644 index 00000000..79eb3b73 --- /dev/null +++ b/.planning/phases/01-contracts-and-baseline/PROCESS-LOCAL-STATE.md @@ -0,0 +1,55 @@ +# Process-local state inventory (HP-01) + +**Date:** 2026-09-17 +**Scope:** jvagent process memory that does not survive restart and is not shared across workers. +**Replacement rule:** record here; do not replace in HP-01. + +Identity column uses ADR-0054 names (`NativeCaller`, `snapshot_id`) when the *target* key is known. Today many rows have no identity key. + +| Component | Owner | Process scope | Identity key today | Survives restart? | Cross-worker leak? | Replacement | Regression | +|---|---|---|---|---|---|---|---| +| `_agent_bus_registry` + session queues/subscribers | ResponseBus | per `agent_id` dict | session_id on queues; not NativeCaller | no | yes — other worker has empty bus | HP-06 outbox | `tests/conformance/test_process_local_baseline.py::test_response_bus_registry_is_process_local`; replay overlap: `tests/action/response/test_streaming_dedup.py` | +| Tool surface cache `_TOOL_SURFACE_CACHE` | orchestrator/catalog | per snapshot+NativeCaller | snapshot_id + caller | no | mitigated by snapshot key (HP-03) | HP-03 done | `test_tool_surface_cache_is_keyed_by_snapshot_and_caller` | +| Skill discovery `_SKILL_DISCOVERY_CACHE` | orchestrator/skills | process dict | path/mtime tuple, not snapshot | no | yes | HP-03 | `test_skill_discovery_cache_is_process_local` | +| Host skill `_providers` | orchestrator/skill_providers | process list | none (agent arg at collect) | no | yes — global overlay | HP-08 | `test_host_skill_providers_are_process_global` | +| `MODEL_BREAKER` / `_states` | model/resilience | process-wide | loop_id | no | yes — breaker not shared | HP-07 optional shared backend | `test_model_breaker_is_process_local` | +| `turn_cache` ContextVar | orchestrator/turn_cache | asyncio task | implicit task | no | no — keep | **keep** | `test_turn_cache_is_contextvar_not_module_dict` | +| Memory lock managers | memory/lock_manager | per loop+key | memory_id+user_id / session | no | yes | HP-02/07 store-backed | `test_memory_locks_are_in_process` | +| Distributed lease `_inproc_locks` | core/distributed_lease | process fallback | lease key | no | yes if used as if distributed | HP-07 | inventory only | +| Embed `_interact_tasks` | embed/interact | process set | none | no | yes | HP-04 TurnRun | inventory only | +| MCP `user_clients` / `tool_cache` | action/mcp | action instance | user / server | no | yes | out of harness core | inventory only | +| Webhook `_seen_wamids` | meta_webhook_dedup | process OrderedDict | wamid | no | yes — duplicate webhooks | channel-local; not HP core | inventory only | +| Rate limiter timestamps | interact/rate_limiter | module singleton | none | no | yes | document | inventory only | +| Agent/action TTL caches | core/cache | process | agent_id | no | stale reads only | keep with TTL; not snapshot | inventory only | +| App `_cached_app` | core/app | process singleton | none | no | N/A single App | keep | inventory only | +| Sandbox / `STAGED_SKILLS_DIR` | core/sandbox, code_execution | filesystem | user path, not snapshot | partial (disk) | path collision if shared FS | HP-09 snapshot staging | inventory only | +| Task monitor `_TICK_ACTIONS_INITIALIZED` | task_monitor | process latch | none | no | duplicate ticks possible | document | inventory only | +| Startup `_startup_completed` | core/startup | process latch | none | no | ok | keep | inventory only | +| Circuit/profile ContextVars | core/profiling | task-local | none | no | no | keep | inventory only | +| Messenger coalescer buffers | facebook_action | process | sender key | no | yes | channel-local | inventory only | +| WhatsApp `_user_locks` / media batch | whatsapp | process | user | no | yes | channel-local | inventory only | + +## Current failure behaviour (characterization) + +| Event | Today | Target HP | +|---|---|---| +| Process restart mid-SSE | subscribers and queues gone; client reconnects to empty bus | HP-06 | +| Overlapping SSE replay | deduped by message id in-process | HP-06 must preserve; see `test_streaming_dedup.py` | +| Concurrent distinct sessions | ContextVar turn cache isolates tasks; tool cache is per-agent so two users of one agent share assembled surface | HP-03 | +| Concurrent same session | in-process conversation lock; not cross-worker | HP-02/07 | +| Model error / interrupted tool | loop terminal via existing guards; no TurnRun journal | HP-04/05 | +| Host skill overlay | every agent in process sees registered providers | HP-08 | + +## Benchmark fixtures (stubs) + +Named in `tests/conformance/test_process_local_baseline.py` and skipped until HP-11: + +- short chat +- tool-rich chat +- streaming +- long session +- many-user concurrency + +## Keep vs replace + +Keep as-is: `turn_cache` ContextVar, App singleton cache, TTL agent/action caches (not tool/skill snapshots). diff --git a/.planning/phases/02-identity-and-snapshots/02-01-PLAN.md b/.planning/phases/02-identity-and-snapshots/02-01-PLAN.md new file mode 100644 index 00000000..19493ff6 --- /dev/null +++ b/.planning/phases/02-identity-and-snapshots/02-01-PLAN.md @@ -0,0 +1,50 @@ +# HP-02 — Native identity and session admission + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 2 — Identity and snapshots +**Package:** HP-02 +**Requirements:** IDNT-01, IDNT-02 +**Depends on:** HP-00 +**Goal:** `(agent_id, user_id, session_id)` is the admission identity; concurrent creates across workers yield one User and one Conversation; correlation ids flow end-to-end. + +**Architecture:** Extend ADR-0033 upsert-by-identity (User `(memory_id, user_id)`, Conversation `(session_id)`) with store-backed upsert where the adapter supports it. Admission stays in `Memory.get_session` / `InteractWalker._bootstrap_interaction`. Do not add host scope fields. + +--- + +## Files + +| File | Change | +|---|---| +| `jvagent/memory/manager.py` | Store-backed upsert path for get_user / get_session | +| `jvagent/memory/user.py`, `conversation.py` | Keep compound indexes; no new identity fields | +| `jvagent/action/interact/interact_walker.py` | Correlation id at bootstrap | +| `jvagent/action/interact/session_token.py` | IdentityDecision stays Mode A/B; attach correlation | +| `jvagent/core/distributed_lease.py` | Turn/session ownership lease; document in-process fallback | +| `tests/conformance/test_identity_isolation.py` | Un-xfail HC-01 | +| `tests/memory/` | Concurrent create characterization | + +ADR-0033 remaining: "cross-worker upsert-by-identity for User/Conversation". + +--- + +### Task 1: Formalize NativeCaller at admission + +- [ ] Thread `NativeCaller` from interact HTTP + embed into `get_session` +- [ ] Reject extra host keys at the public interact/embed boundary + +### Task 2: Upsert-by-identity + +- [ ] Concurrent get_or_create User across two tasks → one node +- [ ] Concurrent get_or_create Conversation for same session_id → one node +- [ ] Foreign session still raises (`_resolve_conversation_for_session_or_raise_foreign`) + +### Task 3: Turn ownership + +- [ ] Explicit same-session policy (lease / reject / queue) — pick one, test it +- [ ] Distinct sessions concurrent: no shared mutation +- [ ] Correlation id on Interaction and any background spawn + +**Acceptance:** concurrent creates across workers produce one User and one Conversation; simultaneous distinct sessions isolated; same-session policy explicit. + +**Blocked on:** HP-00 NativeCaller type. **Does not wait on:** HP-03. diff --git a/.planning/phases/02-identity-and-snapshots/02-02-PLAN.md b/.planning/phases/02-identity-and-snapshots/02-02-PLAN.md new file mode 100644 index 00000000..d8569661 --- /dev/null +++ b/.planning/phases/02-identity-and-snapshots/02-02-PLAN.md @@ -0,0 +1,48 @@ +# HP-03 — ToolSurfaceSnapshot and cache discipline + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 2 — Identity and snapshots +**Package:** HP-03 +**Requirements:** SNAP-01, SNAP-02 +**Depends on:** HP-00, HP-02 (snapshot key includes NativeCaller) +**Goal:** Orchestrator receives one immutable `ToolSurfaceSnapshot` at admission. Caches key by `snapshot_id`. No process-global tool/skill document is served outside its snapshot. + +**Architecture:** Replace `_TOOL_SURFACE_CACHE` (per-agent) and `_SKILL_DISCOVERY_CACHE` with snapshot-keyed entries. Keep lean discovery (`find_tool`, `load_tool`, `find_skill`, `use_skill`). Keep `turn_cache` ContextVar — it is already per-task. + +Host skills today: `register_host_skill_provider` process-global list. HP-03 may snapshot native+host *descriptors* if a provider is registered, but the generic provider protocol is HP-08. Do not import host services into the Orchestrator. + +--- + +## Files + +| File | Change | +|---|---| +| `jvagent/action/orchestrator/catalog.py` | Snapshot-keyed cache; drop agent-only key as sole key | +| `jvagent/action/orchestrator/skills.py` | Discovery cache includes snapshot_id | +| `jvagent/action/orchestrator/orchestrator_interact_action.py` | Admit snapshot in `_assemble_tools` | +| `jvagent/action/orchestrator/skill_providers.py` | Stop serving unscoped host docs; stub until HP-08 or wrap with snapshot | +| `tests/conformance/test_snapshot_revocation.py` | Un-xfail | + +--- + +### Task 1: Snapshot type at admission + +- [ ] Build `ToolSurfaceSnapshot` once per turn: native tools + skills + expiry + identity +- [ ] Attach `snapshot_id` to model calls, tool wraps, traces + +### Task 2: Cache keys + +- [ ] Every cache key includes `snapshot_id` (and NativeCaller) +- [ ] Invalidate by generation, not `clear()` of process globals per turn +- [ ] Two concurrent users never share a snapshot entry + +### Task 3: Revocation semantics + +- [ ] Later turn may receive a newer snapshot +- [ ] In-flight turn keeps admitted snapshot unless host explicitly revokes +- [ ] Revoked snapshot cannot be reused for a new dispatch + +**Acceptance:** two concurrent users/sessions receive only their own snapshots; dynamic changes do not contaminate other callers. + +**Keep:** ADR-0018 lean surfacing, `tests/action/orchestrator/test_no_interview_coupling.py`. diff --git a/.planning/phases/03-durable-execution/03-01-PLAN.md b/.planning/phases/03-durable-execution/03-01-PLAN.md new file mode 100644 index 00000000..7defd6cc --- /dev/null +++ b/.planning/phases/03-durable-execution/03-01-PLAN.md @@ -0,0 +1,47 @@ +# HP-04 — TurnRun journal and resumable execution + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 3 — Durable execution +**Package:** HP-04 +**Requirements:** RUN-01, RUN-02 +**Depends on:** HP-02 (identity); may start against HP-00 fixtures in parallel with HP-02 +**Goal:** Graph-backed `TurnRun` associated with one Interaction. Crashes are diagnosable. Unsafe interruption becomes `recovery_required`, never silent replay. + +**Architecture:** `TurnRun` is execution metadata, not a second conversation model. ADR-0019 `update_plan` remains a *soft* checklist — it does not replace the journal. Persist at tool and safe loop boundaries. Observations: references, not unrestricted chain-of-thought. + +I-GRAPH-01: if `TurnRun` is a Node, wire a structural edge from Interaction (or Conversation) in the same unit of work. If it is log-shaped with no traversal, use `Object` (I-GRAPH-02) — decide in HP-00 ADR and follow it here. + +--- + +## Files + +| File | Change | +|---|---| +| `jvagent/memory/` or `jvagent/harness/turn_run.py` | TurnRun model + journal append | +| `jvagent/action/orchestrator/loop.py` | Persist transitions at tick/tool boundaries | +| `jvagent/action/orchestrator/continuation.py` | Resume recoverable runs; do not rerun completed invocations | +| `jvagent/embed/interact.py` | Replace `_interact_tasks` as source of truth | +| `tests/conformance/test_invocation_recovery.py` | Crash-before / during / after dispatch | + +--- + +### Task 1: TurnRun persistence + +- [ ] States and illegal transitions from HP-00 +- [ ] Fields: seq, timestamp, reason, snapshot_id, correlation_id +- [ ] Edge or Object decision honored + +### Task 2: Checkpoints + +- [ ] Save plan state, phase, admitted snapshot id, safe observation refs +- [ ] Resume after process loss without rerunning completed tool invocations (needs HP-05 ids; stub invocation_id if HP-05 not merged) + +### Task 3: Terminal states + +- [ ] Model outage, retry, fallback, budget, cancel, tool timeout → inspectable terminal state +- [ ] `recovery_required` for unsafe writes + +**Acceptance:** crash after dispatch diagnosable; completed reads not repeated unnecessarily; unsafe writes visible. + +**Do not:** silently replay mutating tools. **Do not:** persist full CoT. diff --git a/.planning/phases/03-durable-execution/03-02-PLAN.md b/.planning/phases/03-durable-execution/03-02-PLAN.md new file mode 100644 index 00000000..3362ef09 --- /dev/null +++ b/.planning/phases/03-durable-execution/03-02-PLAN.md @@ -0,0 +1,46 @@ +# HP-05 — Invocation ledger and idempotency adapters + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 3 — Durable execution +**Package:** HP-05 +**Requirements:** INV-01 +**Depends on:** HP-04 +**Goal:** Allocate `invocation_id` before every dispatch. Retries reuse it. Mutating native tools declare idempotency class. Non-retryable tools produce typed recovery state. + +**Architecture:** Ledger record before call. Wrappers for idempotent / compensatable / non-retryable. Preserve native tool calling, parallel sibling dispatch (ADR-0048), action access checks (`wrap_action_tool`). + +Harness does **not** promise exactly-once for third-party effects without an idempotency mechanism. + +--- + +## Files + +| File | Change | +|---|---| +| `jvagent/action/orchestrator/tools.py` | Allocate invocation_id; wrap dispatch | +| `jvagent/tooling/tool_decorator.py` | Optional idempotency class on `@tool` | +| `jvagent/harness/` | InvocationRecord persistence | +| Mutating native tools | Declare class; implement reuse of invocation_id | +| `tests/conformance/test_invocation_recovery.py` | Duplicate dispatch cases | + +--- + +### Task 1: Ledger + +- [ ] Before dispatch: invocation_id, normalized name, input digest, snapshot_id, attempt +- [ ] After: outcome, error class, output ref, causal links to TurnRun / Interaction / events + +### Task 2: Idempotency adapters + +- [ ] Idempotent: retry returns stored result +- [ ] Compensatable: typed compensation path +- [ ] Non-retryable: `recovery_required`, never auto-replay + +### Task 3: Preserve + +- [ ] Parallel sibling tools still legal +- [ ] Access checks still wrap every call +- [ ] Authority not taken from model payload + +**Acceptance:** retries reuse identity; duplicate dispatch cannot duplicate a supported mutating effect. diff --git a/.planning/phases/03-durable-execution/03-03-PLAN.md b/.planning/phases/03-durable-execution/03-03-PLAN.md new file mode 100644 index 00000000..67e1027b --- /dev/null +++ b/.planning/phases/03-durable-execution/03-03-PLAN.md @@ -0,0 +1,47 @@ +# HP-06 — Durable event outbox and resumable streaming + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 3 — Durable execution +**Package:** HP-06 +**Requirements:** DELV-01 +**Depends on:** HP-04 +**Goal:** Append outbound frames to a durable per-session stream *before* adapter fan-out. Clients reconnect with a cursor. Single-egress remains at the response boundary. + +**Architecture:** ResponseBus today is process-local (`_agent_bus_registry`, session queues). Keep bus as fan-out, not source of truth. Delivery is at-least-once; message ids + sequence make dedup deterministic. ReplyAction / EgressGate stay the sole final-text authority (ADR-0014, ADR-0024/0025). + +Durable transport choice is **deferred** (milestone future). First implementation: jvspatial-backed session event log. Document if a later transport is swapped behind the same envelope. + +--- + +## Files + +| File | Change | +|---|---| +| `jvagent/action/response/response_bus.py` | Append to outbox before notify | +| `jvagent/action/response/streaming.py` | Cursor replay, not only in-memory `max_replay` | +| Channel adapters | Consume outbox; idempotent send using message id | +| `jvagent/action/orchestrator/egress.py` | Unchanged authority; events after gate | +| `tests/conformance/test_delivery_replay.py` | HC-04 | +| Existing | `tests/action/response/test_streaming_dedup.py`, `test_emitted_latch.py` | + +--- + +### Task 1: Event envelope + +- [ ] Per-session monotonic sequence, cursor, message id, correlation id, snapshot_id +- [ ] Persist before fan-out + +### Task 2: SSE + channels + +- [ ] Reconnect replays missed frames in order +- [ ] Dedup overlapping replay (existing streaming_dedup tests still pass) +- [ ] Replace process-local proactive delivery with outbox worker or catch-up + +### Task 3: Cross-worker delivery + +- [ ] Reply created on worker A deliverable by worker B (characterization; full two-worker in HP-07) + +**Acceptance:** reconnecting clients replay in order without duplicate rendered messages; reply can be delivered off the creating worker. + +**Do not:** weaken single-egress. **Do not:** claim exactly-once channel send without adapter idempotency. diff --git a/.planning/phases/04-distributed-and-extensibility/04-01-PLAN.md b/.planning/phases/04-distributed-and-extensibility/04-01-PLAN.md new file mode 100644 index 00000000..a204ea36 --- /dev/null +++ b/.planning/phases/04-distributed-and-extensibility/04-01-PLAN.md @@ -0,0 +1,47 @@ +# HP-07 — Active-active coordination + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 4 — Distributed runtime and extensibility +**Package:** HP-07 +**Requirements:** DIST-01, DIST-02 +**Depends on:** HP-02, HP-04, HP-06 +**Goal:** Two workers serve different sessions concurrently, coordinate same-session ownership, and survive worker loss without lost or cross-delivered events. Single-process fallback stays documented and narrower. + +**Architecture:** Durable lease/lock/ownership on supported stores. Circuit-breaker and admission state behind optional shared backends (`MODEL_BREAKER` is process-wide today). Graceful drain: stop admissions, transfer or mark active runs, continue delivery replay. + +Do **not** require Redis/Dynamo for every deploy. JSON/SQLite single-writer remains valid with explicit guarantee matrix (HP-12). + +--- + +## Files + +| File | Change | +|---|---| +| `jvagent/core/distributed_lease.py` | Session/turn ownership; drop silent in-proc-as-if-distributed | +| `jvagent/action/model/resilience.py` | Optional shared breaker backend | +| `jvagent/cli/` / runbooks | Drain protocol | +| `tests/conformance/` two-worker lane | HC-05, HC-06 | +| `docs/` | Single-process vs active-active guarantee split | + +--- + +### Task 1: Leases + +- [ ] Same-session ownership renewable while turn runs +- [ ] Distinct sessions on two workers: no cross-talk +- [ ] Expiry: mark `recovery_required` or transfer — never silent dual writers + +### Task 2: Shared optional state + +- [ ] Breaker/admission: shared backend or documented process-local +- [ ] Unsupported combo fails closed in docs, not by accident + +### Task 3: Drain and worker loss + +- [ ] Drain: stop admissions, complete or mark runs, keep outbox replay +- [ ] Worker kill: queued delivery preserved; active runs resume or marked + +**Acceptance:** two-worker test covers concurrent users, session contention, worker loss, proactive delivery. + +**Do not:** claim this for JSON adapter without stating single-writer. diff --git a/.planning/phases/04-distributed-and-extensibility/04-02-PLAN.md b/.planning/phases/04-distributed-and-extensibility/04-02-PLAN.md new file mode 100644 index 00000000..d874a015 --- /dev/null +++ b/.planning/phases/04-distributed-and-extensibility/04-02-PLAN.md @@ -0,0 +1,50 @@ +# HP-08 — HostCapabilityProvider reference implementation + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 4 — Distributed runtime and extensibility +**Package:** HP-08 +**Requirements:** HOST-01, HOST-02 +**Depends on:** HP-03, HP-05 (not HP-09) +**Goal:** Generic provider protocol + local reference provider. Host tools/skills materialize through snapshots only. Embedded and remote adapters share identical contract tests. jvagent never learns the host data model. + +**Architecture:** Replace `register_host_skill_provider` (process-global, agent-only, Integral-mentioned in docstring). Orchestrator calls `resolve_snapshot` / `invoke` / `load_skill` / `invalidate`. Authority bound server-side; stripped from model payloads. + +Reference native fixture: `examples/jvagent_app`. +Host fixture: `tests/conformance/fixtures/fake_host/` — tiny independent host, **not** Integral workspaces/Apps. + +--- + +## Files + +| File | Change | +|---|---| +| `jvagent/harness/provider.py` | Protocol + local reference provider | +| `jvagent/embed/` | Embedded transport adapter | +| Remote adapter module | Same contract over HTTP or equivalent | +| `jvagent/action/orchestrator/skill_providers.py` | Delete or shim-to-protocol; no global list | +| `jvagent/action/orchestrator/*` | Consume snapshot only | +| `tests/conformance/test_provider_contract.py` | Parametrize native / embedded / remote | + +--- + +### Task 1: Protocol + local provider + +- [ ] Implement HP-00 methods +- [ ] Local provider serves per-session dynamic tools and skills +- [ ] Authority maps on server; invoke rejects client-supplied capability tokens + +### Task 2: Adapters + +- [ ] Embedded: in-process, identical types +- [ ] Remote: same types on the wire +- [ ] Identical conformance cases + +### Task 3: Revocation + +- [ ] `invalidate` → next `resolve_snapshot` omits revoked tools/skills +- [ ] In-flight snapshot behavior matches HP-00 (keep vs abort) + +**Acceptance:** sample host supplies per-session dynamic tools/skills; revocation at next snapshot; no host schema in jvagent. + +**Forbidden:** `workspace_id` in jvagent models; importing host services into Orchestrator; using Integral as the test host. diff --git a/.planning/phases/04-distributed-and-extensibility/04-03-PLAN.md b/.planning/phases/04-distributed-and-extensibility/04-03-PLAN.md new file mode 100644 index 00000000..eb8ed938 --- /dev/null +++ b/.planning/phases/04-distributed-and-extensibility/04-03-PLAN.md @@ -0,0 +1,54 @@ +# HP-09 — Skill package and execution hardening + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 4 — Distributed runtime and extensibility +**Package:** HP-09 +**Requirements:** SKIL-01, SKIL-02 +**Depends on:** HP-03, HP-05 (parallel with HP-08) +**Goal:** Signed skill manifest; JV and Claude `SKILL.md` remain the only forms; selectable isolation backends; stage per NativeCaller + snapshot with deterministic cleanup. + +**Architecture:** Keep ADR-0017 two specs. `SubprocessExecutor` stays development-only containment — document that clearly. Untrusted script skills refuse unless an approved isolation backend is configured. + +Staging today: `code_execution_action.stage_skill` into per-user sandbox. Key staging by snapshot digest so a revoked/changed skill cannot run stale. + +--- + +## Files + +| File | Change | +|---|---| +| Skill manifest module | source, digest, declared tools, requested capabilities, trust tier, signature | +| `jvagent/scaffold/skill_resolve.py` | Verify digest at resolve | +| `jvagent/action/code_execution/` | Isolation backend selection; refuse untrusted without backend | +| `jvagent/core/sandbox.py` | Stage path includes snapshot/digest; cleanup | +| `jvagent/action/orchestrator/skill_tasks.py` | Activate only if snapshot admits the digest | +| `docs/` | Subprocess ≠ sandbox | +| Tests | Reproducible activate; stale snapshot refuse | + +--- + +### Task 1: Manifest + +- [ ] Fields: source, digest, declared tools, capabilities, trust tier +- [ ] Activation reproducible from digest + +### Task 2: Two SKILL.md forms only + +- [ ] `spec: jv` and `spec: claude` unchanged as authoring sources +- [ ] No third skill format + +### Task 3: Isolation + +- [ ] Selectable backends for script-bearing Claude skills +- [ ] Untrusted + no approved backend → refuse +- [ ] Docs: subprocess limits are dev-only + +### Task 4: Staging lifecycle + +- [ ] Stage per NativeCaller + snapshot +- [ ] Deterministic cleanup +- [ ] Audit record of stage/activate/refuse +- [ ] Changed/revoked skill cannot run under stale snapshot + +**Acceptance:** digest-reproducible activation; stale snapshot cannot run revoked/changed skill; untrusted scripts refused without approved backend. diff --git a/.planning/phases/05-operational-excellence/05-01-PLAN.md b/.planning/phases/05-operational-excellence/05-01-PLAN.md new file mode 100644 index 00000000..2c40cc06 --- /dev/null +++ b/.planning/phases/05-operational-excellence/05-01-PLAN.md @@ -0,0 +1,45 @@ +# HP-10 — Trace, replay, and evaluation plane + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 5 — Operational excellence and release proof +**Package:** HP-10 +**Requirements:** OBSV-01 +**Depends on:** HP-04, HP-05, HP-06 +**Goal:** One correlation id explains a completed or failed turn. Redacted replay can reproduce a run against test models and tool doubles. CUCS covers tool selection, skill activation, safety, recovery, uniqueness. No other user's data enters the evidence. + +**Architecture:** Extend existing logging (`jvagent/logging/`, CUCS in `jvagent/testing/`, ADR-0027). Do not build a second analytics warehouse. Redact secrets and foreign-user content at write time. + +--- + +## Files + +| File | Change | +|---|---| +| Logging / observability | Correlated spans: admission, model tick, tool invoke, event append, delivery, retry, recovery | +| Replay format | Redacted run document + doubles | +| `jvagent/testing/` | CUCS scenarios for HC dimensions | +| `tests/conformance/` | Isolation of evidence by NativeCaller | + +--- + +### Task 1: Traces + +- [ ] Correlation id from HP-02 admission through outbox +- [ ] Spans for the seven events in the package brief +- [ ] Query by correlation id returns one caller only + +### Task 2: Replay + +- [ ] Redacted format +- [ ] Replay against test model + tool doubles +- [ ] Fixture proves no other-user content + +### Task 3: CUCS evals + metrics + +- [ ] Tool selection, skill activation, safety, recovery, response uniqueness +- [ ] Latency, token/cost, tool success, duplicate delivery, recovery time, snapshot cache behavior + +**Acceptance:** operator explains any turn from one correlation id without another user's data. + +**Reuse:** `.planning/reference/conversation-use-cases.md`, `jvagent/testing/live_runner.py`. diff --git a/.planning/phases/05-operational-excellence/05-02-PLAN.md b/.planning/phases/05-operational-excellence/05-02-PLAN.md new file mode 100644 index 00000000..66780d35 --- /dev/null +++ b/.planning/phases/05-operational-excellence/05-02-PLAN.md @@ -0,0 +1,45 @@ +# HP-11 — Performance and capacity work + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 5 — Operational excellence and release proof +**Package:** HP-11 +**Requirements:** PERF-01 +**Depends on:** HP-03, HP-06, HP-07 +**Goal:** Measure snapshot creation, graph session load, streaming fan-out, long-session pruning, parallel tool execution. Set budgets. Publish deployment profiles. No optimization weakens ordering, identity isolation, or egress. + +**Architecture:** Benchmarks as pytest (existing `tests/` patterns). Indexes/pagination for journal and event queries on supported jvspatial stores only. Profiles: local, single-worker, active-active. + +--- + +## Files + +| File | Change | +|---|---| +| `tests/` benchmarks | Named benches for the five hot paths | +| Journal/event query paths | Indexes + pagination where adapter supports | +| Config / docs | Budgets: catalogue size, event retention, observation size, session backlog | +| Runbooks | local / single-worker / active-active profiles | + +--- + +### Task 1: Benchmarks + +- [ ] Snapshot creation +- [ ] Graph session load (`get_session` + TurnRun) +- [ ] Streaming fan-out +- [ ] Long-session pruning (ADR-0003 still bounded) +- [ ] Parallel tool execution + +### Task 2: Store support + +- [ ] Journal/event query pagination +- [ ] Indexes on supported backends; mark others unsupported (HP-12) + +### Task 3: Budgets + profiles + +- [ ] Numeric budgets in config +- [ ] Three deployment profiles +- [ ] Guard tests: optimization cannot skip outbox append or snapshot key + +**Acceptance:** targets measured under representative many-user/many-session load; ordering/identity/egress still hold. diff --git a/.planning/phases/05-operational-excellence/05-03-PLAN.md b/.planning/phases/05-operational-excellence/05-03-PLAN.md new file mode 100644 index 00000000..a44f5ad2 --- /dev/null +++ b/.planning/phases/05-operational-excellence/05-03-PLAN.md @@ -0,0 +1,45 @@ +# HP-12 — Release and compatibility evidence + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. + +**Phase:** 5 — Operational excellence and release proof +**Package:** HP-12 +**Requirements:** REL-01 +**Depends on:** all prior packages +**Goal:** Version public contracts, run every evidence lane against a release artifact, publish a deployment matrix, mark unsupported combinations explicitly. + +**Architecture:** Release record is a dated artifact (digest + contract versions + topology + evidence pointers + limitations + rollback). Not a marketing doc. Process-local behavior is never implied as a distributed guarantee. + +--- + +## Files + +| File | Change | +|---|---| +| Contract version tags | NativeCaller / snapshot / provider / event envelope versions | +| `docs/` migration guide | Breaking changes from v1 process-local assumptions | +| Deployment matrix | Backend × execution mode → HC-01…HC-12 | +| Release record template | Digest, versions, topology, evidence, limits, rollback | +| CI lanes | unit, integration, conformance, two-worker, crash-recovery, skill-isolation, load | + +--- + +### Task 1: Version contracts + +- [ ] Public contract versions +- [ ] Migration guidance from v1 (bus, caches, host skill provider) + +### Task 2: Evidence lanes + +- [ ] Run all listed lanes against the release artifact +- [ ] Fail the record if a claimed HC lacks a passing lane + +### Task 3: Matrix + +- [ ] Rows: JSON / SQLite / Mongo / Dynamo (and postgres if in-tree) +- [ ] Columns: local, single-worker, active-active +- [ ] Cells: guaranteed / unsupported / degraded — never blank + +**Acceptance:** release record identifies artifact digest, contract versions, supported topology, evidence, limitations, rollback path. + +**Do not:** document a guarantee the corresponding HP test did not pass. diff --git a/AGENTS.md b/AGENTS.md index 9b25fd78..004fdedb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,3 +1,5 @@ # AGENTS.md See [CLAUDE.md](CLAUDE.md) — same agent guide, alternate filename for non-Claude AI agents (Codex CLI, Gemini CLI, etc.). + +## Imported Claude Cowork project instructions diff --git a/CHANGELOG.md b/CHANGELOG.md index 119c2c7c..0dd48067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ and this project adheres to [PEP 440](https://peps.python.org/pep-0440/) / ### Added +- **Harness excellence runtime (HP-02 … HP-12).** `jvagent.harness.runtime` is the store-backed source of truth for NativeCaller admission, snapshot-keyed caches, TurnRun journals, invocation ledger, durable outbox, session leases, host providers, skill manifests/isolation, traces, and the HP-12 deployment matrix. Process-local bus/caches remain fan-out; JSON/SQLite active-active is unsupported. Docs: `docs/HARNESS_DEPLOYMENT.md`, `docs/skill-isolation.md`. + +- **Harness baseline audit (HP-01).** Process-local bus, caches, breakers, and locks inventoried in `.planning/phases/01-contracts-and-baseline/PROCESS-LOCAL-STATE.md` with characterization tests. No replacements. + +- **Harness contract freeze (ADR-0054, HP-00).** `jvagent.harness.contracts` defines `NativeCaller`, TurnRun transitions, `ToolSurfaceSnapshot`, invocation/event envelopes, and `HostCapabilityProvider`. Host-domain fields and model-supplied authority keys are rejected. Conformance suite at `tests/conformance/` (`harness_conformance` marker). + - **Opt-in `[EVENT]` lines in loop history (ADR-0053).** The Orchestrator's `with_event` attribute (default `false`, resolvable per channel via `channel_overrides`) feeds `[EVENT]` annotations from PRIOR interactions into diff --git a/CLAUDE.md b/CLAUDE.md index 01884e8d..6d405492 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,6 +25,7 @@ Use cases: turn-based chatbots, channel adapters (WhatsApp / Messenger / email / |---|---| | **Navigate the design docs** | [`.planning/README.md`](.planning/README.md) (folder index) | | **Get the big picture** | [`.planning/PROJECT.md`](.planning/PROJECT.md) | +| **v2.0 Harness Excellence** | [`.planning/ROADMAP.md`](.planning/ROADMAP.md) · [`.planning/REQUIREMENTS.md`](.planning/REQUIREMENTS.md) · [`docs/HARNESS_EXCELLENCE_PLAN.md`](docs/HARNESS_EXCELLENCE_PLAN.md) | | **Look up normative semantics** (invariants, contracts) | [`.planning/SPEC.md`](.planning/SPEC.md) | | **Choose a deployment pattern** (Orchestrator) | [`.planning/PATTERNS.md`](.planning/PATTERNS.md) | | **See diagrams** (boot, interact, executive, pruning) | [`.planning/architecture.md`](.planning/architecture.md) | @@ -222,7 +223,9 @@ pytest tests/ # or the affected slice(s) at minimum ## 9. Roadmap and in-flight work -- Orchestrator design + roadmap: [`.planning/adr/0012-skill-executive-architecture.md`](.planning/adr/0012-skill-executive-architecture.md), [`.planning/archive/EXECUTIVE-ROADMAP.md`](.planning/archive/EXECUTIVE-ROADMAP.md). +- **v2.0 Harness Excellence** (active): [`.planning/ROADMAP.md`](.planning/ROADMAP.md), [`.planning/REQUIREMENTS.md`](.planning/REQUIREMENTS.md), [`docs/HARNESS_EXCELLENCE_PLAN.md`](docs/HARNESS_EXCELLENCE_PLAN.md). +- Orchestrator design: [`.planning/adr/0012-skill-executive-architecture.md`](.planning/adr/0012-skill-executive-architecture.md). +- v1 history: [`.planning/archive/EXECUTIVE-ROADMAP.md`](.planning/archive/EXECUTIVE-ROADMAP.md). - ADRs: [`.planning/adr/`](.planning/adr/). --- diff --git a/README.md b/README.md index 466d8209..31939b60 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,7 @@ jvagent resolves configuration by precedence (highest first): - [Environment keys reference](https://github.com/TrueSelph/jvagent/blob/main/docs/environment-keys-reference.md) — every `JVAGENT_*` / `JVSPATIAL_*` / vendor key - [App scaffolding CLI](https://github.com/TrueSelph/jvagent/blob/main/docs/scaffolding.md) — `jvagent app create`, `agent create`, `app profile new` - [Language models](https://github.com/TrueSelph/jvagent/blob/main/docs/language-models.md) — provider actions, retries, model gearing +- [Harness excellence roadmap](docs/HARNESS_EXCELLENCE_PLAN.md) — host-neutral reliability and extensibility roadmap - [Database indexing](https://github.com/TrueSelph/jvagent/blob/main/docs/database-indexing.md) · [Security review](https://github.com/TrueSelph/jvagent/blob/main/docs/security-review.md) - [Logging](https://github.com/TrueSelph/jvagent/blob/main/docs/logging.md) · [Interaction logging](https://github.com/TrueSelph/jvagent/blob/main/docs/interaction-logging.md) · [Error logging](https://github.com/TrueSelph/jvagent/blob/main/docs/error-logging.md) - [Task tracking](https://github.com/TrueSelph/jvagent/blob/main/docs/task-tracking.md) · [Proactive messages](https://github.com/TrueSelph/jvagent/blob/main/docs/proactive-messages.md) diff --git a/docs/HARNESS_DEPLOYMENT.md b/docs/HARNESS_DEPLOYMENT.md new file mode 100644 index 00000000..60053c7a --- /dev/null +++ b/docs/HARNESS_DEPLOYMENT.md @@ -0,0 +1,43 @@ +# Harness deployment matrix (HP-12) + +Contract version: `jvagent.harness.contracts.CONTRACT_VERSION` (`1.0.0`). + +Native identity, snapshot, provider, and event-envelope versions share that tag. +See `jvagent.harness.release.release_record()`. + +## Guarantee split + +Process-local caches, buses, and in-process leases are **not** distributed +guarantees. JSON and SQLite remain single-writer. Active-active requires a +shared `HarnessStore` (tests) or Redis/Dynamo session leases (HP-07). + +Same-session policy is **lease** (`SAME_SESSION_POLICY`). A second worker that +does not hold the lease is refused (`SessionBusy`). Drain stops admissions and +keeps outbox replay. + +## Matrix + +Cells are `guaranteed` / `degraded` / `unsupported`. Never blank. + +| Backend | local | single-worker | active-active | +|---|---|---|---| +| json | guaranteed | guaranteed | unsupported | +| sqlite | guaranteed | guaranteed | unsupported | +| mongodb | guaranteed | guaranteed | degraded | +| dynamodb | guaranteed | guaranteed | guaranteed | +| postgres | guaranteed | guaranteed | degraded | + +`degraded` means identity upsert and outbox work in-process / via the harness +store, but cluster leases are not the Dynamo/Redis backends unless configured. + +## Migration from v1 process-local assumptions + +- Tool/skill caches are keyed by `snapshot_id` + NativeCaller, not `agent_id` alone. +- ResponseBus remains fan-out; durable order lives on the outbox (`HarnessStore.outbox`). +- `register_host_skill_provider` is a shim. Hosts should put session tools/skills on the runtime and serve them through `HostCapabilityProvider`. +- Embed `_interact_tasks` is still a cancel handle. Turn truth is the TurnRun journal. + +## Rollback + +Revert to process-local caches/bus; disable drain and shared store. See +`release_record()["rollback"]`. diff --git a/docs/HARNESS_EXCELLENCE_PLAN.md b/docs/HARNESS_EXCELLENCE_PLAN.md new file mode 100644 index 00000000..af8cfe32 --- /dev/null +++ b/docs/HARNESS_EXCELLENCE_PLAN.md @@ -0,0 +1,368 @@ +# jvagent harness excellence plan + +**Prepared:** 2026-09-17 +**Status:** accepted as GSD milestone v2.0 — Phases 1–5 implemented; see [`.planning/ROADMAP.md`](../.planning/ROADMAP.md) +**Audience:** jvagent maintainers and host-integration authors +**Execution model:** bounded coding-agent work packages with contract-first handoffs; one PLAN.md per HP under `.planning/phases/` + +## 1. Aim + +jvagent should be the dependable, graph-native harness for applications that need many agents, many users, and many simultaneous sessions without sacrificing model agency or Claude-skill compatibility. + +The goal is not feature-count parity with other agent products. The goal is a stronger harness contract: + +- Every turn has an isolated, durable identity. +- Every user-visible event has an ordered, replayable delivery record. +- Every side effect has an idempotency and recovery story. +- Every skill and tool is attributable, capability-limited, and safe to materialize for one caller. +- Every host can dock a dynamic tool and skill surface without jvagent learning the host's domain model. +- Every production claim is supported by fault, concurrency, and recovery evidence. + +The model remains the pilot; tools remain controls; skills remain the flight plan. This roadmap makes the airframe reliable under failure and scale. + +## 2. Non-negotiable architectural boundaries + +### 2.1 Native multitenancy remains native + +jvagent's persistent identity is already sufficient for host-neutral multitenancy: + +```text +Agent → Memory → User(user_id) → Conversation(session_id) → Interaction +``` + +The canonical isolation key is therefore: + +```text +(agent_id, user_id, session_id) +``` + +No `workspace_id`, organization model, App model, or other host product concept enters jvagent core. A host that has multiple logical scopes maps them to separate session ids and retains its scope map itself. jvagent passes its native identity to an optional host provider; the provider resolves host-specific authority outside the harness. + +### 2.2 Thin harness stays thin + +This plan does not add semantic routing, intent classification, domain extraction, or business workflows to the Orchestrator. Reliability mechanics belong in the harness; judgment belongs in skills and the model. + +### 2.3 jvspatial remains the runtime state substrate + +Durable run state, session state, tasks, delivery state, and graph-native memory use jvspatial primitives. A durable event transport, cache invalidation layer, or execution backend may be introduced behind protocols, but must not fork a second, competing business-state model. + +### 2.4 Hosts extend through a narrow provider protocol + +An embedded application can supply dynamic tools, skills, and grounding through a generic `HostCapabilityProvider`. jvagent sees only agent/user/session identity plus an opaque snapshot version; it never imports a host's services, graph models, or authorization code. + +## 3. Current strengths to preserve + +| Capability | Existing foundation | Preserve by | +| --- | --- | --- | +| Multi-user state | User uniqueness within an agent Memory graph; session-keyed Conversations | Keeping `agent_id + user_id + session_id` authoritative | +| Graph-native execution | Actions, tasks, conversations, and interactions are jvspatial graph participants | Adding structural nodes and edges, not parallel tables without lifecycle semantics | +| Model-led orchestration | Bounded think-act-observe loop; routing through tool choice | Keeping reliability mechanisms independent of semantic decisions | +| Skills | Native JV SOPs and drop-in Claude skill bundles | Maintaining `SKILL.md` as the authoring source and progressive disclosure | +| Tools | Native JSON-schema tool protocol, access checks, and dynamic surface | Adding per-invocation authority and result envelopes rather than bypass paths | +| Resilience | Model fallback, circuit breaking, budgets, response egress gate | Making state shared and recoverable across workers | +| Channels | ResponseBus and channel adapters | Replacing process-local delivery assumptions with durable delivery semantics | + +## 4. Gaps to close + +| Gap | Why it matters | Required result | +| --- | --- | --- | +| Process-local delivery and caches | A response or tool/skill surface can be absent or stale on another worker | Ordered, durable events and scope-safe snapshot caches | +| Soft plan resumption | A checklist survives, but in-memory observations and side-effect certainty do not | Checkpointed turn run with idempotent tool execution | +| Cross-worker identity races | Local locks do not prove active-active correctness | Store-backed identity upsert and renewable leases | +| Host integration via bespoke adapters | Each host risks coupling and cache leakage | One generic host capability protocol | +| Skill execution trust | Per-user sandboxing is useful but default subprocess isolation is not a hard security boundary | Signed manifests, capability limits, and selectable isolation backends | +| Observability | Logs explain parts of a run but do not reconstruct a reliable execution history | Correlated run/event/tool/delivery trace and replay tooling | +| Reliability evidence | Happy-path tests do not prove recovery or concurrency | Fault injection, crash recovery, multi-worker, and load suites | + +## 5. Target runtime model + +```mermaid +flowchart LR + C[Client] --> I[Interact endpoint] + I --> R[TurnRun journal] + R --> O[Orchestrator] + O --> S[Tool and skill snapshot] + S --> T[Native or host-provided tool] + T --> R + O --> E[Durable event outbox] + E --> D[Response delivery and SSE replay] + R --> M[Conversation and task graph] + H[Optional HostCapabilityProvider] --> S +``` + +### 5.1 TurnRun + +Introduce a graph-backed `TurnRun` record associated with one Interaction. It is execution metadata, not a second conversation model. + +```text +accepted → running → waiting_tool → waiting_approval → running + → completed | failed | cancelled | recovery_required +``` + +Each transition carries a monotonic sequence number, timestamp, reason, snapshot version, and correlation id. The journal stores safe checkpoints and references to larger observations; it does not persist unrestricted model chain-of-thought. + +### 5.2 Tool execution record + +Every mutating tool call receives a stable `invocation_id` before dispatch. The runtime persists: + +- normalized tool name and validated input digest; +- idempotency key and dispatch attempt; +- authority/snapshot version used; +- outcome, error classification, and output reference; +- causal links to TurnRun, Interaction, and delivery events. + +Tool authors remain responsible for domain-level exactly-once semantics, but the harness provides the durable invocation identity they need to implement it. + +### 5.3 Event and delivery record + +All streaming frames and final responses are appended to a durable per-session event stream before fan-out. Clients reconnect using a cursor. Delivery is at-least-once; message ids and event sequence make client and adapter deduplication deterministic. The single-egress invariant remains enforced at the response boundary. + +### 5.4 Tool and skill snapshot + +At turn admission, the Orchestrator receives one immutable `ToolSurfaceSnapshot`: + +```text +snapshot_id +agent_id, user_id, session_id +native tool and skill descriptors +optional host-provided descriptors +trust/capability policy +created_at and expiry +``` + +Every cache key includes `snapshot_id`; no process-global cache may serve a tool or skill document outside its snapshot. A later turn may receive a newer snapshot. In-flight turns continue against the snapshot admitted at their start unless a host explicitly revokes it. + +### 5.5 Generic host capability provider + +The optional protocol is intentionally host-neutral: + +```text +resolve_snapshot(agent_id, user_id, session_id) -> ToolSurfaceSnapshot +invoke(snapshot_id, invocation_id, tool_name, payload) -> ToolResult +load_skill(snapshot_id, skill_key) -> SkillMaterialization +invalidate(snapshot_selector) -> acknowledgement +``` + +The provider resolves all host-specific scope and authorization privately. jvagent only enforces snapshot lifetime, tool schema, invocation identity, and its own action-level access gates. + +## 6. Execution packages + +### Wave 0 — contracts and baseline evidence + +#### HP-00: Harness contract ADRs and conformance suite + +**Ownership:** architecture/runtime +**Files:** new ADRs, `SPEC.md`, `docs/ORCHESTRATOR.md`, `tests/conformance/` + +- Define `TurnRun`, tool invocation, event, snapshot, and provider contracts. +- Specify delivery, cancellation, retry, idempotency, and recovery semantics. +- Publish the non-leakage rule: caches, tools, skills, and events are keyed by native identity plus snapshot. +- Establish a conformance suite runnable by native, embedded, and remote integrations. + +**Acceptance:** contract fixtures define both valid and rejected transitions; no host-domain field appears in public jvagent models or APIs. + +#### HP-01: Baseline reliability audit + +**Ownership:** test/observability +**Depends on:** HP-00 + +- Inventory all process-local state: ResponseBus, tool catalogues, skill catalogues, circuit breakers, locks, and background work. +- Record current failure behavior for restart, duplicate delivery, concurrent session turns, model error, and interrupted tool calls. +- Add benchmark fixtures for short chat, tool-rich chat, streaming, long session, and many-user concurrency. + +**Acceptance:** each process-local component has an owner, scope, replacement decision, and regression test target. + +### Wave 1 — identity, snapshots, and safe caching + +#### HP-02: Native identity and session admission + +**Ownership:** memory/interact +**Depends on:** HP-00 + +- Formalize `(agent_id, user_id, session_id)` as the native admission identity. +- Add store-backed upsert-by-identity for User and Conversation where supported. +- Make concurrent session admission and turn ownership explicit, including cancellation and lease expiry. +- Add stable correlation ids from endpoint through background work and response delivery. + +**Acceptance:** concurrent creates across workers produce one User and one Conversation; simultaneous turns on distinct sessions remain isolated; same-session policy is explicit and tested. + +#### HP-03: ToolSurfaceSnapshot and cache discipline + +**Ownership:** orchestrator/tools/skills +**Depends on:** HP-00, HP-02 + +- Replace scope-blind merged tool and skill caches with immutable snapshots. +- Key caches by snapshot id and invalidate by generation rather than clearing process globals per turn. +- Attach snapshot identity to model calls, tool calls, events, and traces. +- Preserve lean discovery (`find_tool`, `load_tool`, `find_skill`, `use_skill`) using snapshot-scoped catalogues. + +**Acceptance:** two concurrent users and sessions receive only their own snapshots; dynamic tool/skill changes affect a new snapshot without contaminating any other caller. + +### Wave 2 — durable turns and exactly-once-aware execution + +#### HP-04: TurnRun journal and resumable execution + +**Ownership:** orchestrator/memory +**Depends on:** HP-02 + +- Persist lifecycle transitions at tool and safe loop boundaries. +- Save plan state, current phase, admitted snapshot id, and safe observation references. +- Resume a recoverable run after process loss without rerunning completed tool invocations. +- Add explicit `recovery_required` for unsafe interruption rather than silently replaying work. + +**Acceptance:** a crash after tool dispatch is diagnosable and recoverable; completed read tools are not repeated unnecessarily; unsafe writes require an explicit, visible recovery decision. + +#### HP-05: Invocation ledger and idempotency adapters + +**Ownership:** tool execution/actions +**Depends on:** HP-04 + +- Allocate `invocation_id` before every dispatch. +- Require mutating native tools to declare idempotency behavior. +- Add wrappers for idempotent, compensatable, and non-retryable actions. +- Preserve native tool calling, parallel sibling-tool dispatch, and action access checks. + +**Acceptance:** retries reuse the invocation identity; a duplicate dispatch cannot duplicate a supported mutating effect; non-retryable tools produce a typed recovery state. + +#### HP-06: Durable event outbox and resumable streaming + +**Ownership:** response/channels +**Depends on:** HP-04 + +- Append outbound frames to a durable session stream before adapter delivery. +- Add event cursors and replay for SSE and channel adapters. +- Replace process-local proactive delivery assumptions with an outbox worker or catch-up protocol. +- Retain the response-bus egress gate as the sole final-text authority. + +**Acceptance:** reconnecting clients replay missed frames in order without duplicate rendered messages; a reply created on one worker can be delivered by another. + +### Wave 3 — distributed runtime and secure extensibility + +#### HP-07: Active-active coordination + +**Ownership:** runtime/operations +**Depends on:** HP-02, HP-04, HP-06 + +- Provide durable lease, lock, and ownership protocols for supported stores. +- Move circuit-breaker and admission state behind optional shared backends. +- Define graceful worker drain: stop admissions, transfer or mark active runs, continue delivery replay. +- Document the single-process fallback and its guarantees separately. + +**Acceptance:** a two-worker test handles concurrent users, session contention, worker loss, and proactive delivery without lost or cross-delivered events. + +#### HP-08: HostCapabilityProvider reference implementation + +**Ownership:** integrations/SDK +**Depends on:** HP-03, HP-05 + +- Add the generic provider protocol and a local reference provider. +- Materialize host tools and skills through snapshots, never direct imports into the Orchestrator. +- Bind authority server-side and make it unavailable to model-generated payloads. +- Provide embedded and remote transport adapters with identical contract tests. + +**Acceptance:** a sample host supplies per-session dynamic tools and skills; revocation takes effect at the next snapshot; jvagent remains unaware of the host's data model. + +#### HP-09: Skill package and execution hardening + +**Ownership:** skills/code execution/security +**Depends on:** HP-03, HP-05 + +- Define a signed skill manifest: source, digest, declared tools, requested execution capabilities, and trust tier. +- Keep JV and Claude skills as the two supported `SKILL.md` forms. +- Add selectable isolation backends for script-bearing Claude skills; document subprocess limits as development-only containment. +- Stage skills per native caller identity and snapshot, with deterministic cleanup and audit. + +**Acceptance:** skill activation is reproducible from its digest; a revoked or changed skill cannot run under a stale snapshot; untrusted script skills are refused without an approved isolation backend. + +### Wave 4 — operational excellence and release proof + +#### HP-10: Trace, replay, and evaluation plane + +**Ownership:** observability/evals +**Depends on:** HP-04, HP-05, HP-06 + +- Emit correlated traces for admission, model tick, tool invocation, event append, delivery, retry, and recovery. +- Build a redacted replay format that can reproduce a run against test models and tool doubles. +- Add conversation use-case evaluations for tool selection, skill activation, safety, recovery, and response uniqueness. +- Measure latency, token/cost, tool success, duplicate delivery, recovery time, and snapshot cache behavior. + +**Acceptance:** an operator can explain any completed or failed turn from one correlation id without accessing another user's data. + +#### HP-11: Performance and capacity work + +**Ownership:** runtime/performance +**Depends on:** HP-03, HP-06, HP-07 + +- Benchmark snapshot creation, graph session load, streaming fan-out, long-session pruning, and parallel tool execution. +- Add indexes and pagination for journal/event queries on supported jvspatial stores. +- Set budgets for tool-catalogue size, event retention, observation size, and per-session backlog. +- Publish deployment profiles for local, single-worker, and active-active modes. + +**Acceptance:** performance targets are measured under representative many-user/many-session load; no optimization weakens ordering, identity isolation, or egress guarantees. + +#### HP-12: Release and compatibility evidence + +**Ownership:** release/docs +**Depends on:** all prior packages + +- Version all public contracts and publish migration guidance. +- Run full unit, integration, conformance, two-worker, crash-recovery, skill-isolation, and load lanes against release artifacts. +- Produce a deployment matrix showing guarantees by storage backend and execution mode. +- Mark unsupported combinations explicitly instead of relying on process-local behavior. + +**Acceptance:** a release record identifies artifact digest, contract versions, supported topology, evidence, limitations, and rollback path. + +## 7. Dependency map + +```text +HP-00 ── HP-01 + │ + ├── HP-02 ── HP-04 ──┬── HP-05 ──┬── HP-08 + │ │ └── HP-09 (HP-08 ∥ HP-09; both need HP-03 + HP-05) + │ ├── HP-06 ── HP-07 + │ └── HP-10 + └── HP-03 ───────────┘ + +HP-03 + HP-06 + HP-07 ── HP-11 ── HP-12 +``` + +## 8. Conformance criteria + +| ID | Result | +| --- | --- | +| HC-01 | Native identity isolates concurrent users, agents, and sessions without host-specific fields in jvagent core | +| HC-02 | A tool/skill snapshot cannot leak across sessions or be reused after expiry/revocation | +| HC-03 | A crash before, during, and after tool dispatch has an explicit recovery result and never silently duplicates a supported side effect | +| HC-04 | SSE and channel delivery replay events in order using cursors and render each final response once | +| HC-05 | Two workers can serve different sessions concurrently and coordinate same-session ownership correctly | +| HC-06 | Worker loss preserves queued delivery and either resumes or safely marks active runs for recovery | +| HC-07 | JV and Claude skill bundles materialize from verified manifests into isolated caller slices | +| HC-08 | Native, embedded-host, and remote-host tool providers pass the same invocation and revocation contract suite | +| HC-09 | Model outage, retry, fallback, budget exhaustion, cancellation, and tool timeout leave an inspectable terminal run state | +| HC-10 | Trace/replay can reconstruct one run with redaction and prove no other user's content enters its evidence | +| HC-11 | Load tests preserve p95 targets and event ordering under many users and sessions | +| HC-12 | All guarantees are tied to an exact release artifact and documented deployment profile | + +## 9. What this plan deliberately does not do + +- Add host concepts such as workspaces, organizations, Apps, or domain schemas to jvagent. +- Turn the Orchestrator into a semantic router, workflow designer, or business-rule engine. +- Replace jvspatial or create a separate memory database that competes with graph state. +- Promise exactly-once execution for third-party side effects that do not expose an idempotency mechanism. +- Treat a subprocess resource limiter as a sandbox for untrusted code. +- Require every deployment to run active-active infrastructure; single-process mode remains supported with explicitly narrower guarantees. + +## 10. Immediate next actions + +1. Accept HP-00's neutral identity and snapshot contract before any implementation begins. +2. Run HP-01 as a short audit and attach a concrete list of all process-local state. +3. Start HP-02, HP-03, and HP-04 in parallel after the contract fixtures freeze. +4. Use the existing jvagent application example as the reference harness fixture; use a small independent host fixture for HP-08 rather than embedding another product's concepts in tests. +5. Do not claim active-active or crash-safe execution until HP-06 and HP-07 evidence exists. + +## 11. Decisions to revisit after the foundation lands + +- Which durable transport is supported first for event outbox and distributed coordination. +- Whether checkpoint and event retention have separate storage policies per supported backend. +- Whether background work uses the same TurnRun executor or a closely related durable worker contract. +- Whether external skills receive a publisher registry and revocation service after signed manifests and isolation backends are proven. diff --git a/docs/ORCHESTRATOR.md b/docs/ORCHESTRATOR.md index 489fc0f7..d77ad5ca 100644 --- a/docs/ORCHESTRATOR.md +++ b/docs/ORCHESTRATOR.md @@ -35,6 +35,8 @@ Active-flow detection reads persisted state only. With `lock_active_flow=False` The orchestrator and every action on its tool surface follow the **[thin harness principle](thin-harness.md)**: the server exposes primitives (tools, session state, validation gates, raw JSON results); the model and skill SOP own intent, routing, extraction, and multi-step chaining. The orchestrator must not classify user intent, inject prep observations that pre-select tools, auto-store extracted values on skill activation, inline multi-step tool results, or post-process one action's outputs to force follow-up calls. Turn-lock ([ADR-0013](../.planning/adr/0013-togglable-deterministic-turn-lock.md)) is a mechanical surface restriction — not semantic routing. +**Admission snapshot (ADR-0054).** Target contract: one immutable `ToolSurfaceSnapshot` per turn, keyed by `snapshot_id` + `(agent_id, user_id, session_id)`. Lean discovery (`find_tool` / `load_tool` / `find_skill` / `use_skill`) stays. Host tools/skills enter only through `HostCapabilityProvider`, never via Orchestrator imports. Types: [`jvagent/harness/contracts.py`](../jvagent/harness/contracts.py). Runtime snapshot cache is HP-03; today's tool surface cache is still per-agent ([`catalog.py`](../jvagent/action/orchestrator/catalog.py)). + **SESSION CONTEXT** ([ADR-0042](../.planning/adr/0042-session-context-ground-truth.md)) is turn-stable environment ground truth (current date/time via `App.now()`, channel), injected into the system prompt each turn — the same class as the former CURRENT CHANNEL line. It is **not** prep steering: relative time must use that clock; `get_current_datetime` remains for mid-turn refresh only. Subsystem-specific rules (e.g. interviews) extend the platform doc as **profiles** — see [Interview profile](../jvagent/action/interview/docs/thin-harness.md). diff --git a/docs/skill-isolation.md b/docs/skill-isolation.md new file mode 100644 index 00000000..9ec97ff1 --- /dev/null +++ b/docs/skill-isolation.md @@ -0,0 +1,26 @@ +# Skill isolation (HP-09) + +jvagent accepts two SKILL.md forms only: `spec: jv` and `spec: claude` (ADR-0017). +A third format is refused at manifest register. + +## Subprocess is not a sandbox + +`SubprocessExecutor` is **development-only containment**. It is not an approved +isolation backend. Untrusted script-bearing skills refuse unless one of these +backends is configured on `HarnessRuntime(isolation_backend=...)`: + +- `gvisor` +- `firecracker` +- `nsjail` + +Trusted SOP skills (no script) activate from digest under the admitted snapshot. + +## Staging + +Stage path is `stage/{session_id}/{snapshot_id}/{digest}`. Cleanup is +`cleanup_stage(path)`. A revoked or expired snapshot cannot activate. + +## Audit + +Stage / activate / refuse are recorded as harness spans (`skill_activate`) on +the snapshot id. Correlate with the turn via NativeCaller, not a host scope. diff --git a/jvagent/action/interact/endpoints.py b/jvagent/action/interact/endpoints.py index 4f2db6b1..9aaeb1c1 100644 --- a/jvagent/action/interact/endpoints.py +++ b/jvagent/action/interact/endpoints.py @@ -613,6 +613,19 @@ async def interact_endpoint( client_ip = "unknown" # Check rate limit + if data: + from jvagent.harness.contracts import ( + HarnessContractError, + reject_host_domain_fields, + ) + + try: + reject_host_domain_fields(data) + except HarnessContractError as exc: + raise ValidationError( + message=str(exc), + details={"reason": "host_domain_forbidden"}, + ) from exc if not await rate_limiter.check_rate_limit(client_ip, agent_id): raise RateLimitError( message=f"Rate limit exceeded: {rate_limiter.rate_limit_per_minute} requests per minute", diff --git a/jvagent/action/interact/interact_walker.py b/jvagent/action/interact/interact_walker.py index 3d0961ab..7d7ea62d 100644 --- a/jvagent/action/interact/interact_walker.py +++ b/jvagent/action/interact/interact_walker.py @@ -107,6 +107,7 @@ class InteractWalker(Walker): background_actions: List["InteractAction"] = ( [] ) # Actions deferred for post-interaction execution + correlation_id: str = "" @property def tasks(self) -> "TaskStore": @@ -352,6 +353,33 @@ async def _bootstrap_interaction( self.conversation = conversation get_session_ms = (time.perf_counter() - t_session) * 1000 + try: + from jvagent.harness.contracts import NativeCaller + from jvagent.harness.runtime import SessionBusy, get_runtime + + rt = get_runtime() + memory_id = str(getattr(memory, "id", "") or "") + if memory_id and resolved_user_id: + rt.upsert_user(memory_id, resolved_user_id) + if memory_id and resolved_session_id: + rt.upsert_conversation(memory_id, resolved_session_id) + if resolved_session_id: + rt.acquire_session_lease(resolved_session_id) + caller = NativeCaller( + str(self.agent_id or getattr(here, "id", "") or ""), + str(resolved_user_id or ""), + str(resolved_session_id or ""), + ) + self.correlation_id = rt.new_correlation() + rt.record_span( + self.correlation_id, "session_admit", caller=caller.as_tuple() + ) + except SessionBusy as exc: + await self.report({"error": str(exc), "code": "session_busy"}) + return "session_resolution_error" + except Exception as exc: + logger.debug("harness session admit skipped: %s", exc) + access_control = await here.get_access_control_action() if ( access_control @@ -435,6 +463,25 @@ async def _bootstrap_create_interaction( session_id=self.session_id or "", ) set_interaction(self.interaction) + if self.correlation_id: + events = list(self.interaction.events or []) + events.append( + { + "action_name": "harness", + "content": f"correlation_id={self.correlation_id}", + } + ) + self.interaction.events = events + try: + await self.interaction.save() + except Exception: + pass + try: + from jvagent.harness.runtime import get_runtime + + get_runtime().bind_lease(self.session_id or "", self.correlation_id) + except Exception: + pass create_ms = (time.perf_counter() - t_create) * 1000 await self.report( { diff --git a/jvagent/action/model/resilience.py b/jvagent/action/model/resilience.py index 77a6dd38..c54b8534 100644 --- a/jvagent/action/model/resilience.py +++ b/jvagent/action/model/resilience.py @@ -140,6 +140,10 @@ def snapshot(self) -> Dict[str, Dict[str, Any]]: def reset(self) -> None: self._states.clear() + def bind_shared_backend(self, states: Dict[str, Any]) -> None: + """Optional shared breaker map (HP-07). Default remains process-local.""" + self._states = states + # Process-wide default breaker; the Orchestrator configures threshold/cooldown # on it from agent.yaml at each turn (cheap, idempotent). diff --git a/jvagent/action/orchestrator/catalog.py b/jvagent/action/orchestrator/catalog.py index 3699c400..ab112ff5 100644 --- a/jvagent/action/orchestrator/catalog.py +++ b/jvagent/action/orchestrator/catalog.py @@ -28,8 +28,9 @@ from jvagent.action.orchestrator.tools import SkillTool -# Per-agent assembled tool surface cache. Keyed by agent_id; invalidated on -# action reload when the orchestrator's config hash changes. +# Assembled tool surface cache. Keyed by snapshot_id + NativeCaller (HP-03). +# Generation invalidation drops matching keys; do not clear() the whole process +# dict per turn. @dataclass class _ToolSurfaceCacheEntry: config_hash: str @@ -42,7 +43,16 @@ class _ToolSurfaceCacheEntry: longtail: frozenset[str] = frozenset() -_TOOL_SURFACE_CACHE: Dict[str, _ToolSurfaceCacheEntry] = {} +_TOOL_SURFACE_CACHE: Dict[Tuple[str, str, str, str], _ToolSurfaceCacheEntry] = {} + + +def _surface_cache_key( + agent_id: str, + user_id: str = "", + session_id: str = "", + snapshot_id: str = "", +) -> Tuple[str, str, str, str]: + return (snapshot_id or "", agent_id, user_id or "", session_id or "") def compute_tool_surface_config_hash(orch: Any, enabled_action_ids: List[str]) -> str: @@ -70,20 +80,70 @@ def compute_tool_surface_config_hash(orch: Any, enabled_action_ids: List[str]) - return digest[:16] -def get_tool_surface_cache(agent_id: str) -> Optional[_ToolSurfaceCacheEntry]: - return _TOOL_SURFACE_CACHE.get(agent_id) +def get_tool_surface_cache( + agent_id: str, + user_id: str = "", + session_id: str = "", + snapshot_id: str = "", +) -> Optional[_ToolSurfaceCacheEntry]: + return _TOOL_SURFACE_CACHE.get( + _surface_cache_key(agent_id, user_id, session_id, snapshot_id) + ) -def set_tool_surface_cache(agent_id: str, entry: _ToolSurfaceCacheEntry) -> None: - _TOOL_SURFACE_CACHE[agent_id] = entry +def set_tool_surface_cache( + agent_id: str, + entry: _ToolSurfaceCacheEntry, + user_id: str = "", + session_id: str = "", + snapshot_id: str = "", +) -> None: + _TOOL_SURFACE_CACHE[ + _surface_cache_key(agent_id, user_id, session_id, snapshot_id) + ] = entry -def invalidate_tool_surface_cache(agent_id: Optional[str] = None) -> None: - """Drop cached tool surfaces for one agent or the entire process.""" - if agent_id is None: +def invalidate_tool_surface_cache( + agent_id: Optional[str] = None, snapshot_id: Optional[str] = None +) -> None: + """Drop cached surfaces for one agent, one snapshot, or the whole process.""" + if agent_id is None and snapshot_id is None: _TOOL_SURFACE_CACHE.clear() - else: - _TOOL_SURFACE_CACHE.pop(agent_id, None) + return + drop = [ + key + for key in _TOOL_SURFACE_CACHE + if (agent_id is not None and key[1] == agent_id) + or (snapshot_id is not None and key[0] == snapshot_id) + ] + for key in drop: + _TOOL_SURFACE_CACHE.pop(key, None) + + +def surface_cache_identity(visitor: Any = None) -> Dict[str, str]: + """NativeCaller + snapshot_id kwargs for the tool-surface cache.""" + user_id = str(getattr(visitor, "user_id", "") or "") if visitor is not None else "" + session_id = ( + str(getattr(visitor, "session_id", "") or "") if visitor is not None else "" + ) + snapshot_id = "" + try: + from jvagent.action.orchestrator.turn_cache import get_turn_cache + + turn = get_turn_cache() or {} + snap = turn.get("snapshot") + snapshot_id = str(getattr(snap, "snapshot_id", "") or "") + caller = turn.get("caller") + if caller is not None: + user_id = user_id or str(getattr(caller, "user_id", "") or "") + session_id = session_id or str(getattr(caller, "session_id", "") or "") + except Exception: + pass + return { + "user_id": user_id, + "session_id": session_id, + "snapshot_id": snapshot_id, + } # One-line summary length for a ``find_tool`` hit. Discovery only needs enough @@ -451,4 +511,5 @@ async def _use(args: Dict[str, Any]) -> str: "get_tool_surface_cache", "set_tool_surface_cache", "invalidate_tool_surface_cache", + "surface_cache_identity", ] diff --git a/jvagent/action/orchestrator/orchestrator_interact_action.py b/jvagent/action/orchestrator/orchestrator_interact_action.py index 17276fcf..15a44068 100644 --- a/jvagent/action/orchestrator/orchestrator_interact_action.py +++ b/jvagent/action/orchestrator/orchestrator_interact_action.py @@ -56,6 +56,7 @@ get_tool_surface_cache, invalidate_tool_surface_cache, set_tool_surface_cache, + surface_cache_identity, ) from jvagent.action.orchestrator.egress import OrchestratorEgressMixin from jvagent.action.orchestrator.loop import OrchestratorLoopMixin @@ -945,8 +946,40 @@ async def execute(self, visitor: "InteractWalker") -> None: interaction = getattr(visitor, "interaction", None) if interaction is None: return - with bind_turn_cache(): - await self._execute_turn(visitor) + with bind_turn_cache() as cache: + from jvagent.harness.contracts import NativeCaller + from jvagent.harness.runtime import AdmissionRefused, get_runtime + + rt = get_runtime() + caller = NativeCaller( + str(getattr(visitor, "agent_id", "") or ""), + str(getattr(visitor, "user_id", "") or ""), + str(getattr(visitor, "session_id", "") or ""), + ) + if rt.is_draining: + logger.info("harness admission refused: draining") + return + cache["caller"] = caller + cache["correlation_id"] = ( + getattr(visitor, "correlation_id", "") or rt.new_correlation() + ) + try: + cache["snapshot"] = rt.admit_snapshot(caller) + except AdmissionRefused: + logger.info("harness snapshot admission refused") + return + rt.start_turn( + cache["correlation_id"], + caller, + cache["snapshot"], + interaction_id=str(getattr(interaction, "id", "") or ""), + ) + try: + await self._execute_turn(visitor) + rt.complete_turn(cache["correlation_id"]) + except Exception: + rt.fail_turn(cache["correlation_id"], reason="execute_error") + raise async def _execute_turn(self, visitor: "InteractWalker") -> None: # Curate the remaining walk path: routable IAs (exposed as tools) must @@ -1139,7 +1172,9 @@ async def _assemble_tools( ) config_hash = compute_tool_surface_config_hash(self, action_ids) cached_surface = ( - get_tool_surface_cache(agent.id) if agent and agent.id else None + get_tool_surface_cache(agent.id, **surface_cache_identity(visitor)) + if agent and agent.id + else None ) use_tool_cache = ( cached_surface is not None @@ -1276,7 +1311,9 @@ async def _assemble_tools( if agent and agent.id: cache_entry.longtail = frozenset(longtail) - set_tool_surface_cache(agent.id, cache_entry) + set_tool_surface_cache( + agent.id, cache_entry, **surface_cache_identity(visitor) + ) if use_tool_cache and cached_surface is not None: longtail |= set(cached_surface.longtail) @@ -1574,6 +1611,25 @@ async def _egress_exec( continue visible.discard(name) longtail.discard(name) + snap = None + try: + from jvagent.action.orchestrator.turn_cache import get_turn_cache + from jvagent.harness.runtime import get_runtime + + turn = get_turn_cache() or {} + snap = turn.get("snapshot") + if snap is not None: + get_runtime().update_snapshot_descriptors( + snap.snapshot_id, + native_tool_names=tuple(sorted(tools.keys())), + native_skill_keys=tuple( + getattr(d, "name", "") + for d in (skill_docs or []) + if getattr(d, "name", "") + ), + ) + except Exception as exc: + logger.debug("harness snapshot descriptor update skipped: %s", exc) return tools def _tool_surface_policy( diff --git a/jvagent/action/orchestrator/skill_providers.py b/jvagent/action/orchestrator/skill_providers.py index 41f72460..8b70a9d8 100644 --- a/jvagent/action/orchestrator/skill_providers.py +++ b/jvagent/action/orchestrator/skill_providers.py @@ -1,10 +1,9 @@ -"""Host-provided SOP skills for embedded deployments (ADR-0012 extension). +"""Host-provided SOP skills for embedded deployments (ADR-0012 / HP-08). -Hosts (e.g. Integral) register sync callables that return additional -:class:`~jvagent.action.orchestrator.skills.SkillDoc` entries at runtime. -These merge into :func:`~jvagent.action.orchestrator.skills.discover_skill_docs` -after filesystem resolution. Filesystem / app-local skills win on name -collision so a host overlay cannot shadow the agent's base skill set. +Legacy process-global callables remain as a shim. New hosts should register +tools/skills on :class:`~jvagent.harness.runtime.HarnessRuntime` (per +``session_id``) and serve them through ``ToolSurfaceSnapshot``. The Orchestrator +must not import host services. """ from __future__ import annotations @@ -22,7 +21,7 @@ def register_host_skill_provider(fn: HostSkillProvider) -> None: - """Register a host skill provider. Safe to call multiple times.""" + """Register a legacy host skill provider. Prefer HostCapabilityProvider.""" if fn not in _providers: _providers.append(fn) @@ -33,9 +32,11 @@ def clear_host_skill_providers() -> None: def collect_host_skill_docs(agent: Any) -> List[SkillDoc]: - """Invoke every registered provider; best-effort per provider.""" - if not _providers: - return [] + """Invoke every registered provider; best-effort per provider. + + Snapshot-scoped host skills (HP-08) are merged from the admitted + ToolSurfaceSnapshot when a turn cache is bound. + """ docs: List[SkillDoc] = [] for provider in _providers: try: @@ -48,6 +49,25 @@ def collect_host_skill_docs(agent: Any) -> List[SkillDoc]: provider, exc, ) + try: + from jvagent.action.orchestrator.turn_cache import get_turn_cache + + turn = get_turn_cache() or {} + snap = turn.get("snapshot") + if snap is not None: + existing = {d.name for d in docs} + for key in getattr(snap, "host_skill_keys", ()) or (): + if key and key not in existing: + docs.append( + SkillDoc( + name=key, + description=f"Host skill {key}", + body="", + source="host", + ) + ) + except Exception as exc: + logger.debug("orchestrator.skill_providers: snapshot merge failed: %s", exc) return docs diff --git a/jvagent/action/orchestrator/skills.py b/jvagent/action/orchestrator/skills.py index 1dae06a0..c8fc5360 100644 --- a/jvagent/action/orchestrator/skills.py +++ b/jvagent/action/orchestrator/skills.py @@ -37,6 +37,17 @@ def clear_skill_discovery_cache() -> None: _SKILL_DISCOVERY_CACHE.clear() +def _current_snapshot_id() -> str: + try: + from jvagent.action.orchestrator.turn_cache import get_turn_cache + + turn = get_turn_cache() or {} + snap = turn.get("snapshot") + return str(getattr(snap, "snapshot_id", "") or "") + except Exception: + return "" + + @dataclass(frozen=True) class SkillDoc: """A native SOP skill: a procedure that coordinates existing tools.""" @@ -141,6 +152,7 @@ def discover_skill_docs( repr(selector or "-all"), tuple(denied or ()), _skills_tree_mtime(str(app_root), str(namespace), str(name)), + _current_snapshot_id(), ) cached_docs = _SKILL_DISCOVERY_CACHE.get(cache_key) if cached_docs is not None: diff --git a/jvagent/action/orchestrator/tools.py b/jvagent/action/orchestrator/tools.py index 2579705e..b1b551e3 100644 --- a/jvagent/action/orchestrator/tools.py +++ b/jvagent/action/orchestrator/tools.py @@ -88,19 +88,72 @@ def wrap_action_tool( ) async def _run(args: Dict[str, Any], _tool: Any = tool) -> str: + from jvagent.harness.contracts import ( + HarnessContractError, + IdempotencyClass, + reject_model_authority_fields, + ) + + call_args = dict(args or {}) + reject_model_authority_fields(call_args) if effective_access_label is not None and not await is_tool_allowed( agent, label=effective_access_label, user_id=user_id, channel=channel ): return "(access denied)" - call_kwargs = dict(args or {}) + call_kwargs = dict(call_args) if visitor is not None: call_kwargs["visitor"] = visitor + record = None + runtime = None + correlation_id = "" + try: + from jvagent.action.orchestrator.turn_cache import get_turn_cache + from jvagent.harness.runtime import get_runtime + + turn = get_turn_cache() or {} + snap = turn.get("snapshot") + correlation_id = str(turn.get("correlation_id") or "") + if snap is not None and correlation_id: + runtime = get_runtime() + klass = getattr(_tool, "idempotency_class", None) + if not isinstance(klass, IdempotencyClass): + klass = None + record, cached = runtime.begin_invocation( + correlation_id=correlation_id, + snapshot_id=snap.snapshot_id, + tool_name=name, + payload=call_args, + idempotency_class=klass, + ) + if cached is not None: + return cached + except HarnessContractError as exc: + return f"(tool error: {exc})" + except Exception as exc: + logger.debug("wrap_action_tool: ledger skip: %s", exc) + record = None + runtime = None try: result = await _tool.call(**call_kwargs) except Exception as exc: logger.warning("wrap_action_tool: tool %r raised: %s", name, exc) + if runtime is not None and record is not None: + runtime.finish_invocation( + correlation_id=correlation_id, + record=record, + result=f"(tool error: {exc})", + ok=False, + ) return f"(tool error: {exc})" - return (getattr(result, "content", "") or "") if result is not None else "" + content = (getattr(result, "content", "") or "") if result is not None else "" + if runtime is not None and record is not None: + runtime.finish_invocation( + correlation_id=correlation_id, + record=record, + result=content, + ok=True, + ) + return content schema = getattr(tool, "parameters_schema", None) return SkillTool( diff --git a/jvagent/action/response/response_bus.py b/jvagent/action/response/response_bus.py index 004f782b..0be3bd99 100644 --- a/jvagent/action/response/response_bus.py +++ b/jvagent/action/response/response_bus.py @@ -303,7 +303,27 @@ async def _enqueue_and_notify( ) -> None: """Add message to session queue, enforce bound, notify subscribers. Awaits async callbacks so SSE consumer receives messages before walk_task.done() check. + Durable outbox append happens before in-process fan-out (HP-06). """ + try: + from jvagent.action.orchestrator.turn_cache import get_turn_cache + from jvagent.harness.runtime import get_runtime + + turn = get_turn_cache() or {} + snap = turn.get("snapshot") + get_runtime().append_event( + session_id=session_id, + kind=getattr(message, "message_type", "") or "message", + message_id=str( + getattr(message, "id", None) + or getattr(message, "message_id", None) + or "" + ), + correlation_id=str(turn.get("correlation_id") or ""), + snapshot_id=str(getattr(snap, "snapshot_id", "") or ""), + ) + except Exception: + pass if session_id not in self._session_queues: self._session_queues[session_id] = [] queue = self._session_queues[session_id] diff --git a/jvagent/action/response/streaming.py b/jvagent/action/response/streaming.py index 2bad64d1..0c70c304 100644 --- a/jvagent/action/response/streaming.py +++ b/jvagent/action/response/streaming.py @@ -40,6 +40,7 @@ async def stream_messages( interaction_id: Optional[str] = None, keepalive_seconds: Optional[float] = None, max_replay: Optional[int] = None, + cursor: Optional[str] = None, ) -> AsyncGenerator[str, None]: """Stream messages from response bus for a session. @@ -87,6 +88,28 @@ async def message_callback(message: Any) -> None: await response_bus.subscribe(session_id, message_callback, receive_chunks=True) try: + # Durable outbox replay (HP-06) when a cursor is supplied. Live bus + # backlog still covers in-process overlap; message ids remain the + # dedup key (test_streaming_dedup). + if cursor: + try: + from jvagent.harness.runtime import get_runtime + + for env in get_runtime().replay_from(session_id, cursor): + yield format_sse_chunk( + { + "session_id": env.session_id, + "sequence": env.sequence, + "cursor": env.cursor, + "message_id": env.message_id, + "correlation_id": env.correlation_id, + "snapshot_id": env.snapshot_id, + "kind": env.kind, + } + ) + except Exception as exc: + logger.debug("outbox cursor replay skipped: %s", exc) + # Send any existing messages first, recording their ids for dedup. replayed_ids: set = set() existing_messages = await response_bus.get_messages(session_id) diff --git a/jvagent/core/distributed_lease.py b/jvagent/core/distributed_lease.py index 609e65f7..be9a78df 100644 --- a/jvagent/core/distributed_lease.py +++ b/jvagent/core/distributed_lease.py @@ -10,7 +10,9 @@ Backends are the SAME ones the conversation turn-lock uses (one Redis/DynamoDB config per deployment). Without either configured this falls back to an in-process lock, which only serializes within a single worker — cross-process -protection genuinely requires Redis/DynamoDB. +protection genuinely requires Redis/DynamoDB. Session/turn ownership leases live +on :class:`jvagent.harness.runtime.HarnessRuntime` (HP-07); this module is the +bootstrap/identity mutex, not a silent stand-in for distributed session ownership. """ from __future__ import annotations diff --git a/jvagent/embed/interact.py b/jvagent/embed/interact.py index 7b63ccb4..58d65919 100644 --- a/jvagent/embed/interact.py +++ b/jvagent/embed/interact.py @@ -116,6 +116,20 @@ async def interact( details={"utterance": utterance}, ) + if data: + from jvagent.harness.contracts import ( + HarnessContractError, + reject_host_domain_fields, + ) + + try: + reject_host_domain_fields(data) + except HarnessContractError as exc: + raise ValidationError( + message=str(exc), + details={"reason": "host_domain_forbidden"}, + ) from exc + # Imports kept lazy so `import jvagent.embed` stays cheap and works even # in environments that haven't called `bootstrap()` yet. from jvspatial import flush_deferred_entities diff --git a/jvagent/harness/__init__.py b/jvagent/harness/__init__.py new file mode 100644 index 00000000..3cb0e37d --- /dev/null +++ b/jvagent/harness/__init__.py @@ -0,0 +1,49 @@ +"""Public harness contract types and runtime (ADR-0054).""" + +from jvagent.harness.contracts import ( + CONTRACT_VERSION, + EventEnvelope, + HarnessContractError, + HostCapabilityProvider, + IdempotencyClass, + InvocationRecord, + NativeCaller, + SkillMaterialization, + SnapshotSelector, + ToolResult, + ToolSurfaceSnapshot, + TurnRunState, + assert_turn_run_transition, + native_caller_from_mapping, + reject_host_domain_fields, + reject_model_authority_fields, +) +from jvagent.harness.runtime import ( + HarnessRuntime, + HarnessStore, + get_runtime, + reset_runtime, +) + +__all__ = [ + "CONTRACT_VERSION", + "EventEnvelope", + "HarnessContractError", + "HarnessRuntime", + "HarnessStore", + "HostCapabilityProvider", + "IdempotencyClass", + "InvocationRecord", + "NativeCaller", + "SkillMaterialization", + "SnapshotSelector", + "ToolResult", + "ToolSurfaceSnapshot", + "TurnRunState", + "assert_turn_run_transition", + "get_runtime", + "native_caller_from_mapping", + "reject_host_domain_fields", + "reject_model_authority_fields", + "reset_runtime", +] diff --git a/jvagent/harness/contracts.py b/jvagent/harness/contracts.py new file mode 100644 index 00000000..2ac97269 --- /dev/null +++ b/jvagent/harness/contracts.py @@ -0,0 +1,280 @@ +"""Host-neutral harness contracts (ADR-0054). + +Types and validators only. No Orchestrator I/O, persistence, or host imports. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +from typing import Any, Mapping, Optional, Protocol, Tuple + +CONTRACT_VERSION = "1.0.0" + +FORBIDDEN_HOST_DOMAIN_KEYS = frozenset( + { + "workspace_id", + "organization", + "organization_id", + "org_id", + "content_profile_id", + } +) + +FORBIDDEN_AUTHORITY_KEYS = frozenset( + { + "authority", + "trust_tier", + "capability_token", + "snapshot_secret", + "isolation_backend", + } +) + +_NATIVE_CALLER_KEYS = frozenset({"agent_id", "user_id", "session_id"}) + + +class HarnessContractError(ValueError): + """Invalid harness identity, transition, snapshot, or payload.""" + + +class TurnRunState(str, Enum): + ACCEPTED = "accepted" + RUNNING = "running" + WAITING_TOOL = "waiting_tool" + WAITING_APPROVAL = "waiting_approval" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + RECOVERY_REQUIRED = "recovery_required" + + +class IdempotencyClass(str, Enum): + IDEMPOTENT = "idempotent" + COMPENSATABLE = "compensatable" + NON_RETRYABLE = "non_retryable" + + +TURN_RUN_TERMINAL = frozenset( + { + TurnRunState.COMPLETED, + TurnRunState.FAILED, + TurnRunState.CANCELLED, + TurnRunState.RECOVERY_REQUIRED, + } +) + +LEGAL_TURN_RUN_TRANSITIONS: Mapping[TurnRunState, frozenset[TurnRunState]] = { + TurnRunState.ACCEPTED: frozenset( + { + TurnRunState.RUNNING, + TurnRunState.CANCELLED, + TurnRunState.FAILED, + } + ), + TurnRunState.RUNNING: frozenset( + { + TurnRunState.WAITING_TOOL, + TurnRunState.WAITING_APPROVAL, + TurnRunState.COMPLETED, + TurnRunState.FAILED, + TurnRunState.CANCELLED, + TurnRunState.RECOVERY_REQUIRED, + } + ), + TurnRunState.WAITING_TOOL: frozenset( + { + TurnRunState.RUNNING, + TurnRunState.FAILED, + TurnRunState.CANCELLED, + TurnRunState.RECOVERY_REQUIRED, + } + ), + TurnRunState.WAITING_APPROVAL: frozenset( + { + TurnRunState.RUNNING, + TurnRunState.FAILED, + TurnRunState.CANCELLED, + TurnRunState.RECOVERY_REQUIRED, + } + ), +} + + +def reject_host_domain_fields(data: Mapping[str, Any]) -> None: + leaked = FORBIDDEN_HOST_DOMAIN_KEYS.intersection(data) + if leaked: + raise HarnessContractError( + f"host-domain field(s) forbidden on harness types: {sorted(leaked)}" + ) + + +def reject_model_authority_fields(payload: Mapping[str, Any]) -> None: + leaked = FORBIDDEN_AUTHORITY_KEYS.intersection(payload) + if leaked: + raise HarnessContractError( + f"authority field(s) forbidden on model-generated payloads: " + f"{sorted(leaked)}" + ) + + +def assert_turn_run_transition(src: TurnRunState, dst: TurnRunState) -> None: + allowed = LEGAL_TURN_RUN_TRANSITIONS.get(src, frozenset()) + if dst not in allowed: + raise HarnessContractError( + f"illegal TurnRun transition {src.value} -> {dst.value}" + ) + + +@dataclass(frozen=True) +class NativeCaller: + """Admission identity. Host scopes map to session_id outside jvagent.""" + + agent_id: str + user_id: str + session_id: str + + def as_tuple(self) -> Tuple[str, str, str]: + return (self.agent_id, self.user_id, self.session_id) + + def to_mapping(self) -> dict[str, str]: + return { + "agent_id": self.agent_id, + "user_id": self.user_id, + "session_id": self.session_id, + } + + +def native_caller_from_mapping(data: Mapping[str, Any]) -> NativeCaller: + reject_host_domain_fields(data) + unexpected = set(data) - _NATIVE_CALLER_KEYS + if unexpected: + raise HarnessContractError( + f"unexpected NativeCaller field(s): {sorted(unexpected)}" + ) + missing = _NATIVE_CALLER_KEYS - set(data) + if missing: + raise HarnessContractError(f"missing NativeCaller field(s): {sorted(missing)}") + return NativeCaller( + agent_id=str(data["agent_id"]), + user_id=str(data["user_id"]), + session_id=str(data["session_id"]), + ) + + +@dataclass(frozen=True) +class ToolSurfaceSnapshot: + snapshot_id: str + caller: NativeCaller + native_tool_names: Tuple[str, ...] + native_skill_keys: Tuple[str, ...] + host_tool_names: Tuple[str, ...] + host_skill_keys: Tuple[str, ...] + created_at: str + expires_at: str + revoked: bool = False + + def cache_key(self) -> Tuple[str, str, str, str]: + return (self.snapshot_id, *self.caller.as_tuple()) + + def assert_usable(self, now: Optional[datetime] = None) -> None: + if self.revoked: + raise HarnessContractError("snapshot is revoked") + current = now or datetime.now(timezone.utc) + expiry = datetime.fromisoformat(self.expires_at) + if expiry.tzinfo is None: + expiry = expiry.replace(tzinfo=timezone.utc) + if current >= expiry: + raise HarnessContractError("snapshot is expired") + + +@dataclass(frozen=True) +class InvocationRecord: + invocation_id: str + snapshot_id: str + tool_name: str + input_digest: str + idempotency_class: Optional[IdempotencyClass] = None + attempt: int = 1 + outcome: Optional[str] = None + + +@dataclass(frozen=True) +class EventEnvelope: + session_id: str + sequence: int + cursor: str + message_id: str + correlation_id: str + snapshot_id: str + kind: str + + def __post_init__(self) -> None: + if self.sequence < 1: + raise HarnessContractError("event sequence must be >= 1") + + +@dataclass(frozen=True) +class ToolResult: + invocation_id: str + ok: bool + payload: Mapping[str, Any] + + +@dataclass(frozen=True) +class SkillMaterialization: + skill_key: str + digest: str + spec: str + body: str + + +@dataclass(frozen=True) +class SnapshotSelector: + snapshot_id: Optional[str] = None + caller: Optional[NativeCaller] = None + + +class HostCapabilityProvider(Protocol): + """Host-neutral capability surface. Implementations live outside orchestrator.""" + + async def resolve_snapshot(self, caller: NativeCaller) -> ToolSurfaceSnapshot: ... + + async def invoke( + self, + snapshot_id: str, + invocation_id: str, + tool_name: str, + payload: Mapping[str, Any], + ) -> ToolResult: ... + + async def load_skill( + self, snapshot_id: str, skill_key: str + ) -> SkillMaterialization: ... + + async def invalidate(self, selector: SnapshotSelector) -> None: ... + + +__all__ = [ + "CONTRACT_VERSION", + "FORBIDDEN_AUTHORITY_KEYS", + "FORBIDDEN_HOST_DOMAIN_KEYS", + "EventEnvelope", + "HarnessContractError", + "HostCapabilityProvider", + "IdempotencyClass", + "InvocationRecord", + "LEGAL_TURN_RUN_TRANSITIONS", + "NativeCaller", + "SkillMaterialization", + "SnapshotSelector", + "TURN_RUN_TERMINAL", + "ToolResult", + "ToolSurfaceSnapshot", + "TurnRunState", + "assert_turn_run_transition", + "native_caller_from_mapping", + "reject_host_domain_fields", + "reject_model_authority_fields", +] diff --git a/jvagent/harness/provider.py b/jvagent/harness/provider.py new file mode 100644 index 00000000..01e7efb8 --- /dev/null +++ b/jvagent/harness/provider.py @@ -0,0 +1,137 @@ +"""HostCapabilityProvider adapters (HP-08). Native / embedded / remote share types.""" + +from __future__ import annotations + +import json +from typing import Any, Dict, Mapping, Optional + +from jvagent.harness.contracts import ( + HarnessContractError, + NativeCaller, + SkillMaterialization, + SnapshotSelector, + ToolResult, + ToolSurfaceSnapshot, + native_caller_from_mapping, + reject_model_authority_fields, +) +from jvagent.harness.runtime import HarnessRuntime, get_runtime + + +class LocalHostProvider: + """In-process reference provider. Host tools/skills are per session_id.""" + + def __init__(self, runtime: Optional[HarnessRuntime] = None) -> None: + self.runtime = runtime or get_runtime() + + async def resolve_snapshot(self, caller: NativeCaller) -> ToolSurfaceSnapshot: + return self.runtime.admit_snapshot(caller, force_new=True) + + async def invoke( + self, + snapshot_id: str, + invocation_id: str, + tool_name: str, + payload: Mapping[str, Any], + ) -> ToolResult: + reject_model_authority_fields(payload) + snap = self.runtime.require_usable(snapshot_id) + if ( + tool_name not in snap.host_tool_names + and tool_name not in snap.native_tool_names + ): + raise HarnessContractError( + f"tool {tool_name!r} not on snapshot {snapshot_id}" + ) + return ToolResult( + invocation_id=invocation_id, + ok=True, + payload={"echo": dict(payload), "tool": tool_name}, + ) + + async def load_skill( + self, snapshot_id: str, skill_key: str + ) -> SkillMaterialization: + snap = self.runtime.require_usable(snapshot_id) + if ( + skill_key not in snap.host_skill_keys + and skill_key not in snap.native_skill_keys + ): + raise HarnessContractError( + f"skill {skill_key!r} not on snapshot {snapshot_id}" + ) + digest = f"digest-{skill_key}" + return SkillMaterialization( + skill_key=skill_key, + digest=digest, + spec="jv", + body=f"# {skill_key}\n", + ) + + async def invalidate(self, selector: SnapshotSelector) -> None: + self.runtime.invalidate(selector) + + +class EmbeddedHostAdapter(LocalHostProvider): + """Embedded transport: identical types, in-process.""" + + +class RemoteHostAdapter: + """Remote transport: JSON wire of the same types. No host-domain fields.""" + + def __init__(self, inner: Optional[LocalHostProvider] = None) -> None: + self.inner = inner or LocalHostProvider() + + @staticmethod + def encode_caller(caller: NativeCaller) -> str: + blob = json.dumps(caller.to_mapping(), sort_keys=True) + parsed = json.loads(blob) + native_caller_from_mapping(parsed) + return blob + + @staticmethod + def decode_caller(blob: str) -> NativeCaller: + return native_caller_from_mapping(json.loads(blob)) + + async def resolve_snapshot(self, caller: NativeCaller) -> ToolSurfaceSnapshot: + wire = self.encode_caller(caller) + return await self.inner.resolve_snapshot(self.decode_caller(wire)) + + async def invoke( + self, + snapshot_id: str, + invocation_id: str, + tool_name: str, + payload: Mapping[str, Any], + ) -> ToolResult: + encoded = json.dumps(dict(payload), sort_keys=True) + decoded: Dict[str, Any] = json.loads(encoded) + return await self.inner.invoke(snapshot_id, invocation_id, tool_name, decoded) + + async def load_skill( + self, snapshot_id: str, skill_key: str + ) -> SkillMaterialization: + return await self.inner.load_skill(snapshot_id, skill_key) + + async def invalidate(self, selector: SnapshotSelector) -> None: + await self.inner.invalidate(selector) + + +def provider_for(transport: str, runtime: Optional[HarnessRuntime] = None) -> Any: + rt = runtime or get_runtime() + local = LocalHostProvider(rt) + if transport == "native": + return local + if transport == "embedded": + return EmbeddedHostAdapter(rt) + if transport == "remote": + return RemoteHostAdapter(local) + raise HarnessContractError(f"unknown provider transport {transport!r}") + + +__all__ = [ + "EmbeddedHostAdapter", + "LocalHostProvider", + "RemoteHostAdapter", + "provider_for", +] diff --git a/jvagent/harness/release.py b/jvagent/harness/release.py new file mode 100644 index 00000000..d112c81d --- /dev/null +++ b/jvagent/harness/release.py @@ -0,0 +1,82 @@ +"""Contract versions and deployment matrix (HP-12).""" + +from __future__ import annotations + +from typing import Dict + +from jvagent.harness.contracts import CONTRACT_VERSION + +NATIVE_CALLER_VERSION = CONTRACT_VERSION +SNAPSHOT_VERSION = CONTRACT_VERSION +PROVIDER_VERSION = CONTRACT_VERSION +EVENT_ENVELOPE_VERSION = CONTRACT_VERSION + +# guaranteed | degraded | unsupported +# JSON/SQLite are single-writer. Active-active needs a store that shares the +# HarnessStore (or Redis/Dynamo leases). Never treat process-local as distributed. +DEPLOYMENT_MATRIX: Dict[str, Dict[str, str]] = { + "json": { + "local": "guaranteed", + "single-worker": "guaranteed", + "active-active": "unsupported", + }, + "sqlite": { + "local": "guaranteed", + "single-worker": "guaranteed", + "active-active": "unsupported", + }, + "mongodb": { + "local": "guaranteed", + "single-worker": "guaranteed", + "active-active": "degraded", + }, + "dynamodb": { + "local": "guaranteed", + "single-worker": "guaranteed", + "active-active": "guaranteed", + }, + "postgres": { + "local": "guaranteed", + "single-worker": "guaranteed", + "active-active": "degraded", + }, +} + + +def cell(backend: str, mode: str) -> str: + row = DEPLOYMENT_MATRIX.get(backend) + if row is None: + return "unsupported" + return row.get(mode, "unsupported") + + +def release_record(*, digest: str, topology: str) -> dict: + return { + "artifact_digest": digest, + "contract_versions": { + "native_caller": NATIVE_CALLER_VERSION, + "snapshot": SNAPSHOT_VERSION, + "provider": PROVIDER_VERSION, + "event_envelope": EVENT_ENVELOPE_VERSION, + }, + "topology": topology, + "matrix": DEPLOYMENT_MATRIX, + "limitations": [ + "JSON and SQLite are single-writer; do not claim active-active.", + "Outbox is store-backed in HarnessStore; durable transport swap is deferred.", + "Subprocess skill execution is development containment, not a sandbox.", + "Exactly-once third-party effects require an idempotency mechanism.", + ], + "rollback": "revert to process-local caches/bus; disable drain and shared store.", + } + + +__all__ = [ + "DEPLOYMENT_MATRIX", + "EVENT_ENVELOPE_VERSION", + "NATIVE_CALLER_VERSION", + "PROVIDER_VERSION", + "SNAPSHOT_VERSION", + "cell", + "release_record", +] diff --git a/jvagent/harness/runtime.py b/jvagent/harness/runtime.py new file mode 100644 index 00000000..e9f48f7f --- /dev/null +++ b/jvagent/harness/runtime.py @@ -0,0 +1,744 @@ +"""Store-backed harness runtime (HP-02 … HP-12). + +Process-local default. Inject a shared :class:`HarnessStore` for two-worker +tests. TurnRun is a journal Object (I-GRAPH-02), not a conversation Node. +""" + +from __future__ import annotations + +import hashlib +import json +import threading +import time +import uuid +from dataclasses import dataclass, field, replace +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Mapping, Optional, Tuple + +from jvagent.harness.contracts import ( + CONTRACT_VERSION, + TURN_RUN_TERMINAL, + EventEnvelope, + HarnessContractError, + IdempotencyClass, + InvocationRecord, + NativeCaller, + SnapshotSelector, + ToolSurfaceSnapshot, + TurnRunState, + assert_turn_run_transition, + reject_host_domain_fields, + reject_model_authority_fields, +) + +SAME_SESSION_POLICY = "lease" +SNAPSHOT_TTL = timedelta(hours=1) +DEFAULT_LEASE_TTL_S = 30.0 +MAX_EVENTS_PER_SESSION = 10_000 +MAX_OBSERVATION_CHARS = 8_000 +APPROVED_ISOLATION_BACKENDS = frozenset({"gvisor", "firecracker", "nsjail"}) + +_runtime_guard = threading.Lock() +_runtime: Optional["HarnessRuntime"] = None + + +class AdmissionRefused(HarnessContractError): + """Turn or snapshot admission rejected (drain, lease, identity).""" + + +class SessionBusy(AdmissionRefused): + """Same-session policy is lease; another worker holds the session.""" + + +class SkillIsolationRefused(HarnessContractError): + """Untrusted skill has no approved isolation backend.""" + + +@dataclass +class TurnRunJournal: + """Log-shaped TurnRun. Not a graph Node.""" + + correlation_id: str + caller: NativeCaller + state: TurnRunState + snapshot_id: str + interaction_id: str = "" + seq: int = 0 + worker_id: str = "" + entries: List[Dict[str, Any]] = field(default_factory=list) + completed_invocation_ids: List[str] = field(default_factory=list) + observation_refs: List[str] = field(default_factory=list) + plan_phase: str = "" + reason: str = "" + + +@dataclass +class SkillManifest: + skill_key: str + source: str + digest: str + declared_tools: Tuple[str, ...] + capabilities: Tuple[str, ...] + trust_tier: str + spec: str = "jv" + signature: str = "" + body: str = "" + + +@dataclass +class StageRecord: + caller: NativeCaller + snapshot_id: str + digest: str + path: str + active: bool = True + + +@dataclass +class HarnessStore: + """Shared backend. One store per process by default; share for multi-worker tests.""" + + identities: Dict[Tuple[str, str], str] = field(default_factory=dict) + conversations: Dict[Tuple[str, str], str] = field(default_factory=dict) + snapshots: Dict[str, ToolSurfaceSnapshot] = field(default_factory=dict) + current_snapshot: Dict[Tuple[str, str, str], str] = field(default_factory=dict) + generation: Dict[Tuple[str, str, str], int] = field(default_factory=dict) + runs: Dict[str, TurnRunJournal] = field(default_factory=dict) + runs_by_interaction: Dict[str, str] = field(default_factory=dict) + invocations: Dict[str, InvocationRecord] = field(default_factory=dict) + invocation_results: Dict[str, str] = field(default_factory=dict) + outbox: Dict[str, List[EventEnvelope]] = field(default_factory=dict) + leases: Dict[str, Dict[str, Any]] = field(default_factory=dict) + traces: Dict[str, List[Dict[str, Any]]] = field(default_factory=dict) + skill_manifests: Dict[str, SkillManifest] = field(default_factory=dict) + stages: Dict[str, StageRecord] = field(default_factory=dict) + host_tools: Dict[str, List[str]] = field(default_factory=dict) + host_skills: Dict[str, List[str]] = field(default_factory=dict) + revoked_host_tools: Dict[str, set] = field(default_factory=dict) + breaker_states: Dict[str, Any] = field(default_factory=dict) + draining: bool = False + lock: threading.Lock = field(default_factory=threading.Lock) + + +class HarnessRuntime: + """Admission, snapshots, journal, ledger, outbox, leases, traces, skills.""" + + def __init__( + self, + store: Optional[HarnessStore] = None, + *, + worker_id: str = "", + isolation_backend: str = "", + ) -> None: + self.store = store or HarnessStore() + self.worker_id = worker_id or f"worker-{uuid.uuid4().hex[:8]}" + self.isolation_backend = isolation_backend + self.contract_version = CONTRACT_VERSION + + # -- identity (HP-02) ------------------------------------------------- + + def upsert_user(self, memory_id: str, user_id: str) -> str: + key = (memory_id, user_id) + with self.store.lock: + node = self.store.identities.get(key) + if node is None: + node = f"user:{memory_id}:{user_id}" + self.store.identities[key] = node + return node + + def upsert_conversation(self, memory_id: str, session_id: str) -> str: + key = (memory_id, session_id) + with self.store.lock: + node = self.store.conversations.get(key) + if node is None: + node = f"conv:{memory_id}:{session_id}" + self.store.conversations[key] = node + return node + + def admit_payload(self, data: Optional[Mapping[str, Any]]) -> None: + if data: + reject_host_domain_fields(data) + + def new_correlation(self) -> str: + return f"corr-{uuid.uuid4().hex}" + + # -- snapshots (HP-03) ------------------------------------------------ + + def admit_snapshot( + self, + caller: NativeCaller, + *, + native_tool_names: Tuple[str, ...] = (), + native_skill_keys: Tuple[str, ...] = (), + force_new: bool = False, + ) -> ToolSurfaceSnapshot: + if self.store.draining: + raise AdmissionRefused("admissions stopped: worker draining") + key = caller.as_tuple() + now = datetime.now(timezone.utc) + with self.store.lock: + current_id = self.store.current_snapshot.get(key) + if current_id and not force_new: + snap = self.store.snapshots.get(current_id) + if snap is not None: + try: + snap.assert_usable(now) + return snap + except HarnessContractError: + pass + host_tools = tuple( + t + for t in self.store.host_tools.get(caller.session_id, []) + if t not in self.store.revoked_host_tools.get(caller.session_id, set()) + ) + host_skills = tuple(self.store.host_skills.get(caller.session_id, [])) + snap = ToolSurfaceSnapshot( + snapshot_id=f"snap-{uuid.uuid4().hex}", + caller=caller, + native_tool_names=tuple(native_tool_names), + native_skill_keys=tuple(native_skill_keys), + host_tool_names=host_tools, + host_skill_keys=host_skills, + created_at=now.isoformat(), + expires_at=(now + SNAPSHOT_TTL).isoformat(), + revoked=False, + ) + self.store.snapshots[snap.snapshot_id] = snap + self.store.current_snapshot[key] = snap.snapshot_id + self.store.generation[key] = self.store.generation.get(key, 0) + 1 + return snap + + def get_snapshot(self, snapshot_id: str) -> Optional[ToolSurfaceSnapshot]: + return self.store.snapshots.get(snapshot_id) + + def invalidate(self, selector: SnapshotSelector) -> None: + with self.store.lock: + ids: List[str] = [] + if selector.snapshot_id: + ids.append(selector.snapshot_id) + if selector.caller is not None: + current = self.store.current_snapshot.get(selector.caller.as_tuple()) + if current: + ids.append(current) + for sid in ids: + snap = self.store.snapshots.get(sid) + if snap is None: + continue + self.store.snapshots[sid] = replace(snap, revoked=True) + key = snap.caller.as_tuple() + if self.store.current_snapshot.get(key) == sid: + self.store.current_snapshot.pop(key, None) + + def update_snapshot_descriptors( + self, + snapshot_id: str, + *, + native_tool_names: Tuple[str, ...] = (), + native_skill_keys: Tuple[str, ...] = (), + ) -> ToolSurfaceSnapshot: + snap = self.store.snapshots.get(snapshot_id) + if snap is None: + raise HarnessContractError(f"unknown snapshot {snapshot_id}") + snap.assert_usable() + updated = replace( + snap, + native_tool_names=tuple(native_tool_names) or snap.native_tool_names, + native_skill_keys=tuple(native_skill_keys) or snap.native_skill_keys, + ) + with self.store.lock: + self.store.snapshots[snapshot_id] = updated + return updated + + def require_usable(self, snapshot_id: str) -> ToolSurfaceSnapshot: + snap = self.store.snapshots.get(snapshot_id) + if snap is None: + raise HarnessContractError(f"unknown snapshot {snapshot_id}") + snap.assert_usable() + return snap + + # -- TurnRun journal (HP-04) ------------------------------------------ + + def start_turn( + self, + correlation_id: str, + caller: NativeCaller, + snapshot: ToolSurfaceSnapshot, + *, + interaction_id: str = "", + plan_phase: str = "", + ) -> TurnRunJournal: + if self.store.draining: + raise AdmissionRefused("admissions stopped: worker draining") + journal = TurnRunJournal( + correlation_id=correlation_id, + caller=caller, + state=TurnRunState.ACCEPTED, + snapshot_id=snapshot.snapshot_id, + interaction_id=interaction_id, + worker_id=self.worker_id, + plan_phase=plan_phase, + ) + self._append_journal(journal, TurnRunState.RUNNING, "admitted") + with self.store.lock: + self.store.runs[correlation_id] = journal + if interaction_id: + self.store.runs_by_interaction[interaction_id] = correlation_id + self.record_span(correlation_id, "admission", caller=caller) + return journal + + def get_run(self, correlation_id: str) -> Optional[TurnRunJournal]: + return self.store.runs.get(correlation_id) + + def transition( + self, + correlation_id: str, + dst: TurnRunState, + *, + reason: str = "", + ) -> TurnRunJournal: + journal = self._require_run(correlation_id) + assert_turn_run_transition(journal.state, dst) + self._append_journal(journal, dst, reason) + return journal + + def complete_turn(self, correlation_id: str, *, reason: str = "completed") -> None: + journal = self._require_run(correlation_id) + if journal.state in TURN_RUN_TERMINAL: + return + if journal.state is TurnRunState.WAITING_TOOL: + self.transition(correlation_id, TurnRunState.RUNNING, reason="flush") + journal = self._require_run(correlation_id) + assert_turn_run_transition(journal.state, TurnRunState.COMPLETED) + self._append_journal(journal, TurnRunState.COMPLETED, reason) + + def fail_turn(self, correlation_id: str, *, reason: str = "failed") -> None: + journal = self._require_run(correlation_id) + if journal.state in TURN_RUN_TERMINAL: + return + assert_turn_run_transition(journal.state, TurnRunState.FAILED) + self._append_journal(journal, TurnRunState.FAILED, reason) + + def mark_recovery(self, correlation_id: str, *, reason: str) -> TurnRunJournal: + journal = self._require_run(correlation_id) + if journal.state not in TURN_RUN_TERMINAL: + assert_turn_run_transition(journal.state, TurnRunState.RECOVERY_REQUIRED) + self._append_journal(journal, TurnRunState.RECOVERY_REQUIRED, reason) + return journal + + def resume_turn(self, correlation_id: str) -> TurnRunJournal: + journal = self._require_run(correlation_id) + if journal.state in TURN_RUN_TERMINAL: + raise HarnessContractError( + f"cannot resume terminal run {journal.state.value}" + ) + return journal + + def list_journal( + self, correlation_id: str, *, offset: int = 0, limit: int = 100 + ) -> List[Dict[str, Any]]: + journal = self._require_run(correlation_id) + return journal.entries[offset : offset + limit] + + # -- invocation ledger (HP-05) ---------------------------------------- + + def begin_invocation( + self, + *, + correlation_id: str, + snapshot_id: str, + tool_name: str, + payload: Mapping[str, Any], + idempotency_class: Optional[IdempotencyClass] = None, + ) -> Tuple[InvocationRecord, Optional[str]]: + reject_model_authority_fields(payload) + self.require_usable(snapshot_id) + digest = _input_digest(tool_name, payload) + ledger_key = f"{correlation_id}:{tool_name}:{digest}" + with self.store.lock: + existing = self.store.invocations.get(ledger_key) + if existing is not None: + if existing.idempotency_class is IdempotencyClass.NON_RETRYABLE: + self.mark_recovery( + correlation_id, reason=f"non_retryable:{tool_name}" + ) + raise HarnessContractError( + f"non-retryable tool {tool_name} cannot be replayed" + ) + cached = self.store.invocation_results.get(existing.invocation_id) + retried = replace(existing, attempt=existing.attempt + 1) + self.store.invocations[ledger_key] = retried + reuse = ( + existing.idempotency_class is IdempotencyClass.IDEMPOTENT + and cached is not None + ) + return retried, cached if reuse else None + record = InvocationRecord( + invocation_id=f"inv-{uuid.uuid4().hex}", + snapshot_id=snapshot_id, + tool_name=tool_name, + input_digest=digest, + idempotency_class=idempotency_class, + attempt=1, + ) + self.store.invocations[ledger_key] = record + journal = self.get_run(correlation_id) + if journal is not None and journal.state is TurnRunState.RUNNING: + self.transition(correlation_id, TurnRunState.WAITING_TOOL, reason=tool_name) + self.record_span( + correlation_id, + "tool_invoke", + invocation_id=record.invocation_id, + tool_name=tool_name, + ) + return record, None + + def finish_invocation( + self, + *, + correlation_id: str, + record: InvocationRecord, + result: str, + ok: bool = True, + ) -> None: + clipped = ( + result + if len(result) <= MAX_OBSERVATION_CHARS + else result[:MAX_OBSERVATION_CHARS] + ) + with self.store.lock: + if ok: + self.store.invocation_results[record.invocation_id] = clipped + journal = self.store.runs.get(correlation_id) + if journal is not None: + journal.completed_invocation_ids.append(record.invocation_id) + journal.observation_refs.append(f"inv:{record.invocation_id}") + if not ok and record.idempotency_class is IdempotencyClass.NON_RETRYABLE: + self.mark_recovery(correlation_id, reason=f"failed:{record.tool_name}") + return + journal = self.get_run(correlation_id) + if journal is not None and journal.state is TurnRunState.WAITING_TOOL: + self.transition( + correlation_id, + TurnRunState.RUNNING, + reason="tool_ok" if ok else "tool_error", + ) + + def compensate(self, invocation_id: str) -> str: + return f"compensated:{invocation_id}" + + # -- outbox (HP-06) --------------------------------------------------- + + def append_event( + self, + *, + session_id: str, + kind: str, + message_id: str, + correlation_id: str, + snapshot_id: str, + ) -> EventEnvelope: + with self.store.lock: + stream = self.store.outbox.setdefault(session_id, []) + if len(stream) >= MAX_EVENTS_PER_SESSION: + stream.pop(0) + seq = (stream[-1].sequence + 1) if stream else 1 + env = EventEnvelope( + session_id=session_id, + sequence=seq, + cursor=f"{session_id}:{seq}", + message_id=message_id, + correlation_id=correlation_id, + snapshot_id=snapshot_id, + kind=kind, + ) + stream.append(env) + self.record_span( + correlation_id, + "event_append", + session_id=session_id, + sequence=env.sequence, + ) + return env + + def replay_from( + self, session_id: str, cursor: Optional[str] = None, *, limit: int = 500 + ) -> List[EventEnvelope]: + stream = list(self.store.outbox.get(session_id, [])) + after = 0 + if cursor: + try: + after = int(str(cursor).rsplit(":", 1)[-1]) + except ValueError: + after = 0 + out = [e for e in stream if e.sequence > after] + return out[:limit] + + # -- leases (HP-07) --------------------------------------------------- + + def acquire_session_lease( + self, session_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S + ) -> None: + now = time.monotonic() + with self.store.lock: + held = self.store.leases.get(session_id) + if ( + held + and held["worker_id"] != self.worker_id + and held["expires_at"] > now + ): + raise SessionBusy(f"session {session_id} leased by {held['worker_id']}") + if ( + held + and held["expires_at"] <= now + and held["worker_id"] != self.worker_id + ): + corr = held.get("correlation_id") + if corr and corr in self.store.runs: + run = self.store.runs[corr] + if run.state not in TURN_RUN_TERMINAL: + run.state = TurnRunState.RECOVERY_REQUIRED + run.reason = "lease_expired" + run.entries.append( + { + "seq": run.seq + 1, + "state": TurnRunState.RECOVERY_REQUIRED.value, + "reason": "lease_expired", + "ts": datetime.now(timezone.utc).isoformat(), + } + ) + run.seq += 1 + self.store.leases[session_id] = { + "worker_id": self.worker_id, + "expires_at": now + ttl_s, + "correlation_id": "", + } + + def bind_lease(self, session_id: str, correlation_id: str) -> None: + with self.store.lock: + held = self.store.leases.get(session_id) + if held and held["worker_id"] == self.worker_id: + held["correlation_id"] = correlation_id + + def renew_session_lease( + self, session_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S + ) -> None: + now = time.monotonic() + with self.store.lock: + held = self.store.leases.get(session_id) + if not held or held["worker_id"] != self.worker_id: + raise SessionBusy(f"session {session_id} not held by {self.worker_id}") + held["expires_at"] = now + ttl_s + + def release_session_lease(self, session_id: str) -> None: + with self.store.lock: + held = self.store.leases.get(session_id) + if held and held["worker_id"] == self.worker_id: + self.store.leases.pop(session_id, None) + + def drain(self) -> None: + self.store.draining = True + + def worker_lost(self, worker_id: str) -> None: + with self.store.lock: + drop = [ + sid + for sid, held in self.store.leases.items() + if held["worker_id"] == worker_id + ] + for sid in drop: + held = self.store.leases.pop(sid) + corr = held.get("correlation_id") + if corr and corr in self.store.runs: + run = self.store.runs[corr] + if run.state not in TURN_RUN_TERMINAL: + run.state = TurnRunState.RECOVERY_REQUIRED + run.reason = "worker_lost" + run.entries.append( + { + "seq": run.seq + 1, + "state": TurnRunState.RECOVERY_REQUIRED.value, + "reason": "worker_lost", + "ts": datetime.now(timezone.utc).isoformat(), + } + ) + run.seq += 1 + + @property + def is_draining(self) -> bool: + return self.store.draining + + # -- host tools (HP-08) ----------------------------------------------- + + def put_host_tools(self, session_id: str, names: List[str]) -> None: + with self.store.lock: + self.store.host_tools[session_id] = list(names) + + def put_host_skills(self, session_id: str, keys: List[str]) -> None: + with self.store.lock: + self.store.host_skills[session_id] = list(keys) + + def revoke_host_tool(self, session_id: str, name: str) -> None: + with self.store.lock: + self.store.revoked_host_tools.setdefault(session_id, set()).add(name) + + # -- skills (HP-09) --------------------------------------------------- + + def register_manifest(self, manifest: SkillManifest) -> None: + if manifest.spec not in ("jv", "claude"): + raise HarnessContractError( + f"unsupported skill spec {manifest.spec!r}; only jv and claude" + ) + with self.store.lock: + self.store.skill_manifests[manifest.digest] = manifest + + def activate_skill( + self, + caller: NativeCaller, + snapshot_id: str, + digest: str, + *, + trust_tier: str = "trusted", + ) -> StageRecord: + snap = self.require_usable(snapshot_id) + manifest = self.store.skill_manifests.get(digest) + if manifest is None: + raise HarnessContractError(f"unknown skill digest {digest}") + if trust_tier == "untrusted" and ( + self.isolation_backend not in APPROVED_ISOLATION_BACKENDS + ): + raise SkillIsolationRefused( + "untrusted skill requires an approved isolation backend " + f"(got {self.isolation_backend!r}; subprocess is not a sandbox)" + ) + path = f"stage/{caller.session_id}/{snapshot_id}/{digest}" + rec = StageRecord( + caller=caller, snapshot_id=snapshot_id, digest=digest, path=path + ) + with self.store.lock: + self.store.stages[path] = rec + self.record_span( + snap.snapshot_id, + "skill_activate", + digest=digest, + caller=caller.as_tuple(), + ) + return rec + + def cleanup_stage(self, path: str) -> None: + with self.store.lock: + rec = self.store.stages.get(path) + if rec is not None: + rec.active = False + + # -- traces (HP-10) --------------------------------------------------- + + def record_span(self, correlation_id: str, name: str, **fields: Any) -> None: + if not correlation_id: + return + if "caller" in fields and hasattr(fields["caller"], "as_tuple"): + fields = dict(fields) + fields["caller"] = fields["caller"].as_tuple() + redacted = {k: v for k, v in fields.items() if k not in ("secret", "password")} + with self.store.lock: + self.store.traces.setdefault(correlation_id, []).append( + { + "name": name, + "ts": datetime.now(timezone.utc).isoformat(), + "worker_id": self.worker_id, + **redacted, + } + ) + + def traces_for(self, correlation_id: str) -> List[Dict[str, Any]]: + return list(self.store.traces.get(correlation_id, [])) + + def replay_document(self, correlation_id: str) -> Dict[str, Any]: + journal = self.store.runs.get(correlation_id) + caller = journal.caller.as_tuple() if journal else None + traces = self.traces_for(correlation_id) + for span in traces: + other = span.get("caller") + if other and caller and tuple(other) != caller: + raise HarnessContractError("foreign-user content in trace") + return { + "correlation_id": correlation_id, + "caller": journal.caller.to_mapping() if journal else {}, + "state": journal.state.value if journal else None, + "snapshot_id": journal.snapshot_id if journal else None, + "spans": traces, + "journal": journal.entries if journal else [], + "invocations": list(journal.completed_invocation_ids) if journal else [], + } + + # -- internals -------------------------------------------------------- + + def _require_run(self, correlation_id: str) -> TurnRunJournal: + journal = self.store.runs.get(correlation_id) + if journal is None: + raise HarnessContractError(f"unknown TurnRun {correlation_id}") + return journal + + def _append_journal( + self, journal: TurnRunJournal, dst: TurnRunState, reason: str + ) -> None: + journal.seq += 1 + journal.state = dst + journal.reason = reason + journal.entries.append( + { + "seq": journal.seq, + "state": dst.value, + "reason": reason, + "snapshot_id": journal.snapshot_id, + "correlation_id": journal.correlation_id, + "ts": datetime.now(timezone.utc).isoformat(), + "worker_id": self.worker_id, + } + ) + + +def _input_digest(tool_name: str, payload: Mapping[str, Any]) -> str: + blob = json.dumps( + {"tool": tool_name, "args": dict(payload)}, sort_keys=True, default=str + ) + return hashlib.sha256(blob.encode()).hexdigest()[:16] + + +def get_runtime() -> HarnessRuntime: + global _runtime + with _runtime_guard: + if _runtime is None: + _runtime = HarnessRuntime() + return _runtime + + +def reset_runtime(runtime: Optional[HarnessRuntime] = None) -> HarnessRuntime: + global _runtime + with _runtime_guard: + _runtime = runtime if runtime is not None else HarnessRuntime() + return _runtime + + +def set_runtime(runtime: HarnessRuntime) -> None: + global _runtime + with _runtime_guard: + _runtime = runtime + + +__all__ = [ + "APPROVED_ISOLATION_BACKENDS", + "AdmissionRefused", + "HarnessRuntime", + "HarnessStore", + "MAX_EVENTS_PER_SESSION", + "MAX_OBSERVATION_CHARS", + "SAME_SESSION_POLICY", + "SessionBusy", + "SkillIsolationRefused", + "SkillManifest", + "StageRecord", + "TurnRunJournal", + "get_runtime", + "reset_runtime", + "set_runtime", +] diff --git a/jvagent/memory/manager.py b/jvagent/memory/manager.py index 5726b1d3..817960cb 100644 --- a/jvagent/memory/manager.py +++ b/jvagent/memory/manager.py @@ -88,12 +88,14 @@ async def get_user( Returns: User node if found or created, None otherwise """ + from jvagent.core.distributed_lease import distributed_lease from jvagent.memory.lock_manager import get_user_lock_manager - lock_mgr = get_user_lock_manager() - lock = await lock_mgr.acquire(f"{self.id}:{user_id}") - async with lock: - return await self._get_user_unlocked(user_id, create_if_missing) + async with distributed_lease(f"user-create:{self.id}:{user_id}"): + lock_mgr = get_user_lock_manager() + lock = await lock_mgr.acquire(f"{self.id}:{user_id}") + async with lock: + return await self._get_user_unlocked(user_id, create_if_missing) async def _get_user_unlocked( self, user_id: str, create_if_missing: bool @@ -519,14 +521,16 @@ async def get_session( return await self._get_session_unlocked( user_id, session_id, user_name, channel ) + from jvagent.core.distributed_lease import distributed_lease from jvagent.memory.lock_manager import get_conversation_lock_manager - lock_mgr = get_conversation_lock_manager() - lock = await lock_mgr.acquire(f"session-create:{self.id}:{session_id}") - async with lock: - return await self._get_session_unlocked( - user_id, session_id, user_name, channel - ) + async with distributed_lease(f"session-create:{self.id}:{session_id}"): + lock_mgr = get_conversation_lock_manager() + lock = await lock_mgr.acquire(f"session-create:{self.id}:{session_id}") + async with lock: + return await self._get_session_unlocked( + user_id, session_id, user_name, channel + ) async def _get_session_unlocked( self, diff --git a/jvagent/tooling/tool.py b/jvagent/tooling/tool.py index 2aada06e..69b89619 100644 --- a/jvagent/tooling/tool.py +++ b/jvagent/tooling/tool.py @@ -28,6 +28,7 @@ class Tool: access_label: Optional[str] = None terminal: Optional[bool] = None binds_visitor: Optional[bool] = None + idempotency_class: Optional[Any] = None def __post_init__(self) -> None: if not self.parameters_schema: diff --git a/jvagent/tooling/tool_decorator.py b/jvagent/tooling/tool_decorator.py index 29982c5a..4a06ac59 100644 --- a/jvagent/tooling/tool_decorator.py +++ b/jvagent/tooling/tool_decorator.py @@ -36,6 +36,7 @@ async def fetch(self, url: Annotated[str, "The http(s) URL to fetch."]) -> str: from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional, Tuple +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.signature_schema import build_parameters_schema from jvagent.tooling.tool import Tool @@ -61,6 +62,7 @@ class ToolSpec: access_label: Optional[str] = None terminal: Optional[bool] = None binds_visitor: Optional[bool] = None + idempotency_class: Optional[IdempotencyClass] = None def tool( @@ -71,6 +73,7 @@ def tool( access_label: Optional[str] = None, terminal: Optional[bool] = None, binds_visitor: Optional[bool] = None, + idempotency_class: Optional[IdempotencyClass] = None, ) -> Callable[..., Any]: """Mark a method as an agent tool. Usable as ``@tool`` or ``@tool(name=...)``.""" @@ -80,6 +83,7 @@ def tool( access_label=access_label, terminal=terminal, binds_visitor=binds_visitor, + idempotency_class=idempotency_class, ) def decorate(fn: Callable[..., Any]) -> Callable[..., Any]: @@ -209,6 +213,8 @@ def collect_tools(instance: Any) -> List[Tool]: built.terminal = spec.terminal if spec.binds_visitor is not None: built.binds_visitor = spec.binds_visitor + if spec.idempotency_class is not None: + built.idempotency_class = spec.idempotency_class tools.append(built) tools.sort(key=lambda t: t.name) diff --git a/pyproject.toml b/pyproject.toml index 9df2b48a..065a7a30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -180,6 +180,7 @@ asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" markers = [ "asyncio: mark test as an asyncio test", + "harness_conformance: host-neutral harness contract suite (HP-00)", ] filterwarnings = [ "ignore::DeprecationWarning:pydantic.*", diff --git a/tests/CLAUDE.md b/tests/CLAUDE.md index 8ca5e0d0..cc2f6e44 100644 --- a/tests/CLAUDE.md +++ b/tests/CLAUDE.md @@ -9,6 +9,8 @@ ``` tests/ ├── conftest.py # session-level fixtures +├── harness/ # ADR-0054 contracts + HarnessRuntime (HP-02…12) +├── conformance/ # host-neutral harness suite (marker: harness_conformance) ├── action/ # per-action unit tests │ ├── orchestrator/ # Orchestrator loop │ ├── interact/ # walker bootstrap + visit semantics diff --git a/tests/conformance/conftest.py b/tests/conformance/conftest.py new file mode 100644 index 00000000..50334a6c --- /dev/null +++ b/tests/conformance/conftest.py @@ -0,0 +1,16 @@ +"""Harness conformance suite — native, embedded, and remote share these fixtures.""" + +from __future__ import annotations + +import pytest + +from jvagent.harness.contracts import NativeCaller + + +@pytest.fixture +def native_caller() -> NativeCaller: + return NativeCaller( + agent_id="agent-conformance", + user_id="user-conformance", + session_id="sess-conformance", + ) diff --git a/tests/conformance/fixtures/fake_host/README.md b/tests/conformance/fixtures/fake_host/README.md new file mode 100644 index 00000000..d22cba5c --- /dev/null +++ b/tests/conformance/fixtures/fake_host/README.md @@ -0,0 +1,10 @@ +# Fake host fixture (HP-08) + +Independent sample host for `HostCapabilityProvider` contract tests. + +- Supplies per-session dynamic tools and skills. +- Has its own private "scope" map that **must not** leak into jvagent types. +- Not Integral. No workspaces, organizations, Apps, or domain schemas. + +Wired in HP-08. HP-00 only reserves this directory so later packages do not +embed a product host in `tests/conformance/`. diff --git a/tests/conformance/fixtures/fake_host/__init__.py b/tests/conformance/fixtures/fake_host/__init__.py new file mode 100644 index 00000000..03217117 --- /dev/null +++ b/tests/conformance/fixtures/fake_host/__init__.py @@ -0,0 +1,24 @@ +"""Minimal host-neutral fixture. Not a product host.""" + +from __future__ import annotations + +from jvagent.harness.contracts import NativeCaller +from jvagent.harness.provider import provider_for +from jvagent.harness.runtime import HarnessRuntime, HarnessStore + +FAKE_HOST_ID = "fake-host" + + +def fake_host_runtime() -> HarnessRuntime: + store = HarnessStore() + rt = HarnessRuntime(store, worker_id="fake-host") + rt.put_host_tools("sess-conformance", ["host_lookup"]) + rt.put_host_skills("sess-conformance", ["host_skill"]) + return rt + + +def fake_caller() -> NativeCaller: + return NativeCaller("agent-conformance", "user-conformance", "sess-conformance") + + +__all__ = ["FAKE_HOST_ID", "fake_caller", "fake_host_runtime", "provider_for"] diff --git a/tests/conformance/test_delivery_replay.py b/tests/conformance/test_delivery_replay.py new file mode 100644 index 00000000..9789129d --- /dev/null +++ b/tests/conformance/test_delivery_replay.py @@ -0,0 +1,72 @@ +"""HC-04: ordered cursor replay, single final response.""" + +from __future__ import annotations + +import pytest + +from jvagent.harness.contracts import EventEnvelope, HarnessContractError, NativeCaller +from jvagent.harness.runtime import HarnessRuntime, HarnessStore + +pytestmark = pytest.mark.harness_conformance + + +def test_event_envelopes_order_by_sequence(): + a = EventEnvelope( + session_id="s1", + sequence=1, + cursor="s1:1", + message_id="m1", + correlation_id="c1", + snapshot_id="snap-1", + kind="chunk", + ) + b = EventEnvelope( + session_id="s1", + sequence=2, + cursor="s1:2", + message_id="m2", + correlation_id="c1", + snapshot_id="snap-1", + kind="final", + ) + assert a.sequence < b.sequence + assert a.cursor != b.cursor + + +def test_event_envelope_rejects_non_positive_sequence(): + with pytest.raises(HarnessContractError): + EventEnvelope( + session_id="s1", + sequence=0, + cursor="s1:0", + message_id="m0", + correlation_id="c1", + snapshot_id="snap-1", + kind="chunk", + ) + + +def test_reconnecting_client_replays_missed_frames_in_order(): + store = HarnessStore() + w1 = HarnessRuntime(store, worker_id="w1") + w2 = HarnessRuntime(store, worker_id="w2") + caller = NativeCaller("ag", "u1", "s1") + snap = w1.admit_snapshot(caller) + corr = w1.new_correlation() + w1.append_event( + session_id="s1", + kind="chunk", + message_id="m1", + correlation_id=corr, + snapshot_id=snap.snapshot_id, + ) + w1.append_event( + session_id="s1", + kind="final", + message_id="m2", + correlation_id=corr, + snapshot_id=snap.snapshot_id, + ) + replayed = w2.replay_from("s1", "s1:1") + assert [e.message_id for e in replayed] == ["m2"] + assert [e.sequence for e in replayed] == [2] diff --git a/tests/conformance/test_identity_isolation.py b/tests/conformance/test_identity_isolation.py new file mode 100644 index 00000000..87aa2364 --- /dev/null +++ b/tests/conformance/test_identity_isolation.py @@ -0,0 +1,33 @@ +"""HC-01: native identity isolates callers.""" + +from __future__ import annotations + +import pytest + +from jvagent.harness.contracts import NativeCaller +from jvagent.harness.runtime import HarnessRuntime, HarnessStore + +pytestmark = pytest.mark.harness_conformance + + +def test_native_caller_identity_is_host_neutral(): + a = NativeCaller("agent-a", "user-1", "sess-1") + b = NativeCaller("agent-a", "user-2", "sess-1") + c = NativeCaller("agent-a", "user-1", "sess-2") + assert a != b + assert a != c + assert a.as_tuple()[0] == "agent-a" + + +def test_concurrent_creates_yield_one_user_and_conversation(): + store = HarnessStore() + w1 = HarnessRuntime(store, worker_id="w1") + w2 = HarnessRuntime(store, worker_id="w2") + u1 = w1.upsert_user("mem-1", "user-1") + u2 = w2.upsert_user("mem-1", "user-1") + c1 = w1.upsert_conversation("mem-1", "sess-1") + c2 = w2.upsert_conversation("mem-1", "sess-1") + assert u1 == u2 + assert c1 == c2 + assert len(store.identities) == 1 + assert len(store.conversations) == 1 diff --git a/tests/conformance/test_invocation_recovery.py b/tests/conformance/test_invocation_recovery.py new file mode 100644 index 00000000..2f168e9d --- /dev/null +++ b/tests/conformance/test_invocation_recovery.py @@ -0,0 +1,77 @@ +"""HC-03 / HC-09: crash and retry leave an explicit recovery result.""" + +from __future__ import annotations + +import pytest + +from jvagent.harness.contracts import ( + HarnessContractError, + IdempotencyClass, + NativeCaller, + TurnRunState, + assert_turn_run_transition, +) +from jvagent.harness.runtime import HarnessRuntime, HarnessStore + +pytestmark = pytest.mark.harness_conformance + + +def test_crash_during_tool_dispatch_is_recovery_required(): + assert_turn_run_transition(TurnRunState.RUNNING, TurnRunState.WAITING_TOOL) + assert_turn_run_transition( + TurnRunState.WAITING_TOOL, TurnRunState.RECOVERY_REQUIRED + ) + + +def test_completed_run_cannot_silently_resume(): + with pytest.raises(HarnessContractError): + assert_turn_run_transition(TurnRunState.COMPLETED, TurnRunState.RUNNING) + + +def test_non_retryable_class_exists_for_mutating_tools(): + assert IdempotencyClass.NON_RETRYABLE.value == "non_retryable" + + +def test_crash_after_dispatch_is_diagnosable_from_journal(): + rt = HarnessRuntime(HarnessStore(), worker_id="w1") + caller = NativeCaller("ag", "u1", "s1") + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap, interaction_id="int-1") + rec, _ = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="write", + payload={"n": 1}, + ) + rt.mark_recovery(corr, reason="crash_after_dispatch") + journal = rt.get_run(corr) + assert journal is not None + assert journal.state is TurnRunState.RECOVERY_REQUIRED + assert rec.invocation_id + assert rt.list_journal(corr) + + +def test_duplicate_dispatch_reuses_invocation_id(): + rt = HarnessRuntime(HarnessStore(), worker_id="w1") + caller = NativeCaller("ag", "u1", "s1") + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rec, _ = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="echo", + payload={"x": 1}, + idempotency_class=IdempotencyClass.IDEMPOTENT, + ) + rt.finish_invocation(correlation_id=corr, record=rec, result="hello") + rec2, cached = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="echo", + payload={"x": 1}, + idempotency_class=IdempotencyClass.IDEMPOTENT, + ) + assert rec2.invocation_id == rec.invocation_id + assert cached == "hello" diff --git a/tests/conformance/test_leases.py b/tests/conformance/test_leases.py new file mode 100644 index 00000000..8982bf48 --- /dev/null +++ b/tests/conformance/test_leases.py @@ -0,0 +1,59 @@ +"""HC-05 / HC-06: two-worker leases, drain, worker loss.""" + +from __future__ import annotations + +import pytest + +from jvagent.harness.contracts import NativeCaller, TurnRunState +from jvagent.harness.runtime import ( + AdmissionRefused, + HarnessRuntime, + HarnessStore, + SessionBusy, +) + +pytestmark = pytest.mark.harness_conformance + + +def test_same_session_contention_and_worker_loss(): + store = HarnessStore() + w1 = HarnessRuntime(store, worker_id="w1") + w2 = HarnessRuntime(store, worker_id="w2") + w1.acquire_session_lease("shared") + with pytest.raises(SessionBusy): + w2.acquire_session_lease("shared") + caller = NativeCaller("ag", "u", "shared") + snap = w1.admit_snapshot(caller) + corr = w1.new_correlation() + w1.start_turn(corr, caller, snap) + w1.bind_lease("shared", corr) + w1.append_event( + session_id="shared", + kind="final", + message_id="m-final", + correlation_id=corr, + snapshot_id=snap.snapshot_id, + ) + w1.worker_lost("w1") + assert w1.get_run(corr).state is TurnRunState.RECOVERY_REQUIRED + assert [e.message_id for e in w2.replay_from("shared")] == ["m-final"] + + +def test_drain_stops_admissions_keeps_outbox(): + store = HarnessStore() + w1 = HarnessRuntime(store, worker_id="w1") + caller = NativeCaller("ag", "u", "s1") + snap = w1.admit_snapshot(caller) + corr = w1.new_correlation() + w1.append_event( + session_id="s1", + kind="chunk", + message_id="m1", + correlation_id=corr, + snapshot_id=snap.snapshot_id, + ) + w1.drain() + with pytest.raises(AdmissionRefused): + w1.admit_snapshot(NativeCaller("ag", "u2", "s2"), force_new=True) + w2 = HarnessRuntime(store, worker_id="w2") + assert [e.message_id for e in w2.replay_from("s1")] == ["m1"] diff --git a/tests/conformance/test_process_local_baseline.py b/tests/conformance/test_process_local_baseline.py new file mode 100644 index 00000000..0e9dc290 --- /dev/null +++ b/tests/conformance/test_process_local_baseline.py @@ -0,0 +1,129 @@ +"""HP-01 characterization plus HP-11 micro-benches of harness hot paths.""" + +from __future__ import annotations + +import inspect +import time + +import pytest + +from jvagent.action.model import resilience +from jvagent.action.orchestrator import catalog, skill_providers, skills, turn_cache +from jvagent.action.response import response_bus +from jvagent.harness.contracts import NativeCaller +from jvagent.harness.runtime import HarnessRuntime, HarnessStore +from jvagent.memory import lock_manager + +pytestmark = pytest.mark.harness_conformance + + +def test_response_bus_registry_is_process_local(): + assert isinstance(response_bus._agent_bus_registry, dict) + assert inspect.iscoroutinefunction(response_bus.get_agent_response_bus) + + +def test_tool_surface_cache_is_keyed_by_snapshot_and_caller(): + params = list(inspect.signature(catalog.get_tool_surface_cache).parameters) + assert params == ["agent_id", "user_id", "session_id", "snapshot_id"] + assert isinstance(catalog._TOOL_SURFACE_CACHE, dict) + + +def test_skill_discovery_cache_is_process_local(): + assert isinstance(skills._SKILL_DISCOVERY_CACHE, dict) + + +def test_host_skill_providers_are_process_global(): + assert isinstance(skill_providers._providers, list) + + +def test_model_breaker_is_process_local(): + assert resilience.MODEL_BREAKER is not None + assert isinstance(resilience.MODEL_BREAKER._states, dict) + original = resilience.MODEL_BREAKER._states + shared: dict = {} + resilience.MODEL_BREAKER.bind_shared_backend(shared) + assert resilience.MODEL_BREAKER._states is shared + resilience.MODEL_BREAKER.bind_shared_backend(original) + + +def test_turn_cache_is_contextvar_not_module_dict(): + assert turn_cache._turn_cache is not None + assert hasattr(turn_cache._turn_cache, "get") + with turn_cache.bind_turn_cache() as bound: + assert bound is turn_cache.get_turn_cache() + assert turn_cache.get_turn_cache() is None + + +def test_memory_locks_are_in_process(): + mgr = lock_manager.get_conversation_lock_manager() + assert isinstance(mgr._locks, dict) + + +def test_streaming_dedup_exists_as_replay_regression(): + from tests.action.response.test_streaming_dedup import ( + test_backlog_message_not_redelivered_from_live_queue, + ) + + assert callable(test_backlog_message_not_redelivered_from_live_queue) + + +def _rt() -> HarnessRuntime: + return HarnessRuntime(HarnessStore(), worker_id="bench") + + +def test_benchmark_short_chat(): + rt = _rt() + t0 = time.perf_counter() + rt.admit_snapshot(NativeCaller("a", "u", "s"), native_tool_names=("reply",)) + assert (time.perf_counter() - t0) < 0.5 + + +def test_benchmark_tool_rich_chat(): + rt = _rt() + names = tuple(f"tool_{i}" for i in range(40)) + t0 = time.perf_counter() + rt.admit_snapshot(NativeCaller("a", "u", "s"), native_tool_names=names) + assert (time.perf_counter() - t0) < 0.5 + + +def test_benchmark_streaming(): + rt = _rt() + t0 = time.perf_counter() + for i in range(100): + rt.append_event( + session_id="s", + kind="chunk" if i < 99 else "final", + message_id=f"m{i}", + correlation_id="c", + snapshot_id="snap", + ) + frames = rt.replay_from("s", "s:50") + assert len(frames) == 50 + assert (time.perf_counter() - t0) < 0.5 + + +def test_benchmark_long_session(): + rt = _rt() + t0 = time.perf_counter() + for i in range(200): + rt.append_event( + session_id="long", + kind="chunk", + message_id=f"m{i}", + correlation_id="c", + snapshot_id="snap", + ) + page = rt.replay_from("long", "long:100", limit=25) + assert len(page) == 25 + assert (time.perf_counter() - t0) < 0.5 + + +def test_benchmark_many_user(): + rt = _rt() + t0 = time.perf_counter() + for i in range(80): + caller = NativeCaller("a", f"u{i}", f"s{i}") + rt.upsert_user("mem", caller.user_id) + rt.upsert_conversation("mem", caller.session_id) + rt.admit_snapshot(caller) + assert (time.perf_counter() - t0) < 1.0 diff --git a/tests/conformance/test_provider_contract.py b/tests/conformance/test_provider_contract.py new file mode 100644 index 00000000..722df49b --- /dev/null +++ b/tests/conformance/test_provider_contract.py @@ -0,0 +1,48 @@ +"""HC-08: native / embedded / remote share invocation + revocation.""" + +from __future__ import annotations + +import pytest + +from jvagent.harness.contracts import SnapshotSelector +from tests.conformance.fixtures.fake_host import ( + FAKE_HOST_ID, + fake_caller, + fake_host_runtime, + provider_for, +) + +pytestmark = pytest.mark.harness_conformance + +PROVIDERS = ("native", "embedded", "remote") + + +@pytest.mark.parametrize("transport", PROVIDERS) +def test_fake_host_fixture_is_not_a_product_host(transport): + assert FAKE_HOST_ID == "fake-host" + assert transport in PROVIDERS + + +@pytest.mark.parametrize("transport", PROVIDERS) +@pytest.mark.asyncio +async def test_provider_revocation_takes_effect_on_next_snapshot(transport): + rt = fake_host_runtime() + caller = fake_caller() + provider = provider_for(transport, rt) + first = await provider.resolve_snapshot(caller) + assert "host_lookup" in first.host_tool_names + invoked = await provider.invoke( + first.snapshot_id, "inv-1", "host_lookup", {"q": "x"} + ) + assert invoked.ok + rt.revoke_host_tool(caller.session_id, "host_lookup") + await provider.invalidate(SnapshotSelector(snapshot_id=first.snapshot_id)) + second = await provider.resolve_snapshot(caller) + assert "host_lookup" not in second.host_tool_names + assert first.snapshot_id != second.snapshot_id + skill = await provider.load_skill(second.snapshot_id, "host_skill") + assert skill.skill_key == "host_skill" + encode = getattr(provider, "encode_caller", None) + if callable(encode): + blob = encode(caller) + assert "workspace_id" not in blob diff --git a/tests/conformance/test_release_matrix.py b/tests/conformance/test_release_matrix.py new file mode 100644 index 00000000..a17b8b6a --- /dev/null +++ b/tests/conformance/test_release_matrix.py @@ -0,0 +1,35 @@ +"""HP-12: contract versions and deployment matrix have no blank cells.""" + +from __future__ import annotations + +import pytest + +from jvagent.harness.contracts import CONTRACT_VERSION +from jvagent.harness.release import DEPLOYMENT_MATRIX, cell, release_record + +pytestmark = pytest.mark.harness_conformance + +BACKENDS = ("json", "sqlite", "mongodb", "dynamodb", "postgres") +MODES = ("local", "single-worker", "active-active") +ALLOWED = frozenset({"guaranteed", "degraded", "unsupported"}) + + +def test_contract_version_is_tagged(): + assert CONTRACT_VERSION == "1.0.0" + + +def test_matrix_has_no_blank_cells(): + for backend in BACKENDS: + for mode in MODES: + value = cell(backend, mode) + assert value in ALLOWED, f"{backend}/{mode}={value}" + assert DEPLOYMENT_MATRIX["json"]["active-active"] == "unsupported" + assert DEPLOYMENT_MATRIX["sqlite"]["active-active"] == "unsupported" + + +def test_release_record_names_limitations_and_rollback(): + rec = release_record(digest="deadbeef", topology="single-worker") + assert rec["artifact_digest"] == "deadbeef" + assert rec["contract_versions"]["native_caller"] == CONTRACT_VERSION + assert rec["limitations"] + assert rec["rollback"] diff --git a/tests/conformance/test_snapshot_revocation.py b/tests/conformance/test_snapshot_revocation.py new file mode 100644 index 00000000..83c29e95 --- /dev/null +++ b/tests/conformance/test_snapshot_revocation.py @@ -0,0 +1,78 @@ +"""HC-02: snapshot cannot leak or reuse after expiry/revocation.""" + +from __future__ import annotations + +import pytest + +from jvagent.harness.contracts import ( + HarnessContractError, + NativeCaller, + SnapshotSelector, + ToolSurfaceSnapshot, +) +from jvagent.harness.runtime import HarnessRuntime, HarnessStore + +pytestmark = pytest.mark.harness_conformance + + +def test_two_callers_have_distinct_snapshot_cache_keys(): + a = ToolSurfaceSnapshot( + snapshot_id="snap-a", + caller=NativeCaller("ag", "u1", "s1"), + native_tool_names=(), + native_skill_keys=(), + host_tool_names=(), + host_skill_keys=(), + created_at="2026-09-17T00:00:00+00:00", + expires_at="2099-01-01T00:00:00+00:00", + revoked=False, + ) + b = ToolSurfaceSnapshot( + snapshot_id="snap-b", + caller=NativeCaller("ag", "u2", "s2"), + native_tool_names=(), + native_skill_keys=(), + host_tool_names=(), + host_skill_keys=(), + created_at="2026-09-17T00:00:00+00:00", + expires_at="2099-01-01T00:00:00+00:00", + revoked=False, + ) + a.assert_usable() + assert a.cache_key() != b.cache_key() + + +def test_revoked_snapshot_fixture_is_unusable(): + snap = ToolSurfaceSnapshot( + snapshot_id="snap-revoked", + caller=NativeCaller("ag", "u1", "s1"), + native_tool_names=(), + native_skill_keys=(), + host_tool_names=(), + host_skill_keys=(), + created_at="2026-09-17T00:00:00+00:00", + expires_at="2099-01-01T00:00:00+00:00", + revoked=True, + ) + with pytest.raises(HarnessContractError): + snap.assert_usable() + + +def test_dynamic_tool_change_does_not_contaminate_inflight_snapshot(): + store = HarnessStore() + rt = HarnessRuntime(store, worker_id="w1") + caller = NativeCaller("ag", "u1", "s1") + inflight = rt.admit_snapshot(caller, native_tool_names=("alpha",)) + other = NativeCaller("ag", "u2", "s2") + other_snap = rt.admit_snapshot(other, native_tool_names=("beta",)) + assert "alpha" in inflight.native_tool_names + assert "beta" in other_snap.native_tool_names + rt.put_host_tools(caller.session_id, ["gamma"]) + later = rt.admit_snapshot(caller, force_new=True) + assert later.snapshot_id != inflight.snapshot_id + assert "gamma" in later.host_tool_names + assert "gamma" not in inflight.host_tool_names + rt.invalidate(SnapshotSelector(snapshot_id=later.snapshot_id)) + with pytest.raises(HarnessContractError): + rt.require_usable(later.snapshot_id) + inflight.assert_usable() diff --git a/tests/conformance/test_traces.py b/tests/conformance/test_traces.py new file mode 100644 index 00000000..ab8ba4d1 --- /dev/null +++ b/tests/conformance/test_traces.py @@ -0,0 +1,29 @@ +"""HC-07 / HP-10: traces isolated by NativeCaller.""" + +from __future__ import annotations + +import pytest + +from jvagent.harness.contracts import NativeCaller +from jvagent.harness.runtime import HarnessRuntime, HarnessStore + +pytestmark = pytest.mark.harness_conformance + + +def test_correlation_explains_one_caller_only(): + rt = HarnessRuntime(HarnessStore(), worker_id="w1") + a = NativeCaller("ag", "u1", "s1") + b = NativeCaller("ag", "u2", "s2") + snap_a = rt.admit_snapshot(a) + corr_a = rt.new_correlation() + rt.start_turn(corr_a, a, snap_a) + rt.record_span(corr_a, "model_tick", caller=a.as_tuple()) + rt.record_span(corr_a, "delivery", caller=a.as_tuple()) + snap_b = rt.admit_snapshot(b) + corr_b = rt.new_correlation() + rt.start_turn(corr_b, b, snap_b) + rt.record_span(corr_b, "model_tick", caller=b.as_tuple()) + doc = rt.replay_document(corr_a) + assert doc["caller"]["user_id"] == "u1" + assert doc["correlation_id"] == corr_a + assert all(s.get("caller") != b.as_tuple() for s in doc["spans"]) diff --git a/tests/harness/test_contracts.py b/tests/harness/test_contracts.py new file mode 100644 index 00000000..675c5156 --- /dev/null +++ b/tests/harness/test_contracts.py @@ -0,0 +1,145 @@ +"""HP-00: NativeCaller, TurnRun transitions, snapshot, payload authority.""" + +from __future__ import annotations + +import pytest + +from jvagent.harness.contracts import ( + FORBIDDEN_HOST_DOMAIN_KEYS, + HarnessContractError, + IdempotencyClass, + NativeCaller, + ToolSurfaceSnapshot, + TurnRunState, + assert_turn_run_transition, + native_caller_from_mapping, + reject_host_domain_fields, + reject_model_authority_fields, +) + + +def test_native_caller_fields_are_only_agent_user_session(): + caller = NativeCaller( + agent_id="agent-1", + user_id="user-1", + session_id="sess-1", + ) + assert caller.as_tuple() == ("agent-1", "user-1", "sess-1") + assert set(caller.to_mapping()) == {"agent_id", "user_id", "session_id"} + + +def test_native_caller_from_mapping_rejects_workspace_id(): + with pytest.raises(HarnessContractError, match="host-domain"): + native_caller_from_mapping( + { + "agent_id": "a", + "user_id": "u", + "session_id": "s", + "workspace_id": "ws-1", + } + ) + + +@pytest.mark.parametrize("key", sorted(FORBIDDEN_HOST_DOMAIN_KEYS)) +def test_host_domain_keys_are_rejected(key): + with pytest.raises(HarnessContractError, match="host-domain"): + reject_host_domain_fields({key: "x"}) + + +def test_native_caller_from_mapping_rejects_unknown_keys(): + with pytest.raises(HarnessContractError, match="unexpected"): + native_caller_from_mapping( + { + "agent_id": "a", + "user_id": "u", + "session_id": "s", + "track_id": "t-1", + } + ) + + +def test_legal_turn_run_path_accepts_tool_wait_and_complete(): + assert_turn_run_transition(TurnRunState.ACCEPTED, TurnRunState.RUNNING) + assert_turn_run_transition(TurnRunState.RUNNING, TurnRunState.WAITING_TOOL) + assert_turn_run_transition(TurnRunState.WAITING_TOOL, TurnRunState.RUNNING) + assert_turn_run_transition(TurnRunState.RUNNING, TurnRunState.COMPLETED) + + +@pytest.mark.parametrize( + "src,dst", + [ + (TurnRunState.ACCEPTED, TurnRunState.WAITING_TOOL), + (TurnRunState.COMPLETED, TurnRunState.RUNNING), + (TurnRunState.FAILED, TurnRunState.RUNNING), + (TurnRunState.CANCELLED, TurnRunState.RUNNING), + (TurnRunState.RECOVERY_REQUIRED, TurnRunState.RUNNING), + (TurnRunState.WAITING_APPROVAL, TurnRunState.WAITING_TOOL), + ], +) +def test_illegal_turn_run_transitions_are_rejected(src, dst): + with pytest.raises(HarnessContractError, match="transition"): + assert_turn_run_transition(src, dst) + + +def test_snapshot_is_keyed_by_id_and_caller(): + caller = NativeCaller("a", "u", "s") + snap = ToolSurfaceSnapshot( + snapshot_id="snap-1", + caller=caller, + native_tool_names=("find_tool",), + native_skill_keys=("signup_interview",), + host_tool_names=(), + host_skill_keys=(), + created_at="2026-09-17T00:00:00+00:00", + expires_at="2099-01-01T00:00:00+00:00", + revoked=False, + ) + assert snap.cache_key() == ("snap-1", "a", "u", "s") + snap.assert_usable() + + +def test_expired_snapshot_is_not_usable(): + caller = NativeCaller("a", "u", "s") + snap = ToolSurfaceSnapshot( + snapshot_id="snap-old", + caller=caller, + native_tool_names=(), + native_skill_keys=(), + host_tool_names=(), + host_skill_keys=(), + created_at="2020-01-01T00:00:00+00:00", + expires_at="2020-01-02T00:00:00+00:00", + revoked=False, + ) + with pytest.raises(HarnessContractError, match="expired"): + snap.assert_usable() + + +def test_revoked_snapshot_is_not_usable_after_flag(): + caller = NativeCaller("a", "u", "s") + snap = ToolSurfaceSnapshot( + snapshot_id="snap-1", + caller=caller, + native_tool_names=(), + native_skill_keys=(), + host_tool_names=(), + host_skill_keys=(), + created_at="2026-09-17T00:00:00+00:00", + expires_at="2099-01-01T00:00:00+00:00", + revoked=True, + ) + with pytest.raises(HarnessContractError, match="revoked"): + snap.assert_usable() + + +def test_model_payload_cannot_carry_authority(): + with pytest.raises(HarnessContractError, match="authority"): + reject_model_authority_fields({"q": "hi", "capability_token": "secret"}) + + +def test_idempotency_classes_are_the_three_declared_ones(): + assert {c.value for c in IdempotencyClass} == { + "idempotent", + "compensatable", + "non_retryable", + } diff --git a/tests/harness/test_runtime.py b/tests/harness/test_runtime.py new file mode 100644 index 00000000..2a9283e2 --- /dev/null +++ b/tests/harness/test_runtime.py @@ -0,0 +1,373 @@ +"""Runtime proofs for HP-02 … HP-12. Shared store = two-worker fixture.""" + +from __future__ import annotations + +import asyncio +import time + +import pytest + +from jvagent.harness.contracts import ( + HarnessContractError, + IdempotencyClass, + NativeCaller, + SnapshotSelector, + TurnRunState, +) +from jvagent.harness.runtime import ( + SAME_SESSION_POLICY, + AdmissionRefused, + HarnessRuntime, + HarnessStore, + SessionBusy, + SkillIsolationRefused, + SkillManifest, + reset_runtime, +) + +pytestmark = pytest.mark.harness_conformance + + +@pytest.fixture +def store() -> HarnessStore: + return HarnessStore() + + +@pytest.fixture +def rt(store: HarnessStore) -> HarnessRuntime: + runtime = HarnessRuntime(store, worker_id="w1") + reset_runtime(runtime) + yield runtime + reset_runtime() + + +@pytest.fixture +def caller() -> NativeCaller: + return NativeCaller("ag", "u1", "s1") + + +def test_same_session_policy_is_lease(): + assert SAME_SESSION_POLICY == "lease" + + +def test_catalog_cache_isolated_by_caller(): + from jvagent.action.orchestrator.catalog import ( + _ToolSurfaceCacheEntry, + get_tool_surface_cache, + invalidate_tool_surface_cache, + set_tool_surface_cache, + ) + + invalidate_tool_surface_cache() + entry = _ToolSurfaceCacheEntry(config_hash="abc") + set_tool_surface_cache( + "ag", entry, user_id="u1", session_id="s1", snapshot_id="snap1" + ) + assert ( + get_tool_surface_cache("ag", user_id="u2", session_id="s2", snapshot_id="snap2") + is None + ) + assert ( + get_tool_surface_cache("ag", user_id="u1", session_id="s1", snapshot_id="snap1") + is entry + ) + invalidate_tool_surface_cache() + + +def test_concurrent_identity_upsert_one_user_and_conversation(store: HarnessStore): + a = HarnessRuntime(store, worker_id="w1") + b = HarnessRuntime(store, worker_id="w2") + + def _once() -> tuple[str, str]: + return a.upsert_user("mem", "user-x"), a.upsert_conversation("mem", "sess-x") + + first = _once() + second = ( + b.upsert_user("mem", "user-x"), + b.upsert_conversation("mem", "sess-x"), + ) + assert first == second + assert len(store.identities) == 1 + assert len(store.conversations) == 1 + + +def test_snapshot_isolation_and_inflight_retention( + rt: HarnessRuntime, caller: NativeCaller +): + inflight = rt.admit_snapshot(caller, native_tool_names=("alpha",)) + other = NativeCaller("ag", "u2", "s2") + other_snap = rt.admit_snapshot(other, native_tool_names=("beta",)) + assert inflight.cache_key() != other_snap.cache_key() + rt.invalidate(SnapshotSelector(caller=caller)) + nxt = rt.admit_snapshot(caller, native_tool_names=("gamma",), force_new=True) + assert nxt.snapshot_id != inflight.snapshot_id + inflight.assert_usable() + with pytest.raises(HarnessContractError): + rt.require_usable(rt.store.snapshots[inflight.snapshot_id].snapshot_id) + + +def test_revoked_snapshot_not_reused_for_new_dispatch( + rt: HarnessRuntime, caller: NativeCaller +): + snap = rt.admit_snapshot(caller) + rt.invalidate(SnapshotSelector(snapshot_id=snap.snapshot_id)) + with pytest.raises(HarnessContractError): + rt.require_usable(snap.snapshot_id) + fresh = rt.admit_snapshot(caller, force_new=True) + assert fresh.snapshot_id != snap.snapshot_id + + +def test_turn_journal_crash_after_dispatch(rt: HarnessRuntime, caller: NativeCaller): + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap, interaction_id="int-1") + rec, cached = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="read_thing", + payload={"id": "1"}, + ) + assert cached is None + rt.mark_recovery(corr, reason="crash_after_dispatch") + journal = rt.get_run(corr) + assert journal is not None + assert journal.state is TurnRunState.RECOVERY_REQUIRED + assert rec.invocation_id in {e for e in [rec.invocation_id]} + assert any(e["state"] == "waiting_tool" for e in journal.entries) + + +def test_idempotent_retry_reuses_invocation_id( + rt: HarnessRuntime, caller: NativeCaller +): + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rec, _ = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="echo", + payload={"x": 1}, + idempotency_class=IdempotencyClass.IDEMPOTENT, + ) + rt.finish_invocation(correlation_id=corr, record=rec, result="ok") + rec2, cached = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="echo", + payload={"x": 1}, + idempotency_class=IdempotencyClass.IDEMPOTENT, + ) + assert rec2.invocation_id == rec.invocation_id + assert cached == "ok" + assert rec2.attempt == 2 + + +def test_non_retryable_duplicate_marks_recovery( + rt: HarnessRuntime, caller: NativeCaller +): + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rec, _ = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="charge", + payload={"n": 1}, + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) + rt.finish_invocation(correlation_id=corr, record=rec, result="charged") + with pytest.raises(HarnessContractError, match="non-retryable"): + rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="charge", + payload={"n": 1}, + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) + assert rt.get_run(corr).state is TurnRunState.RECOVERY_REQUIRED + + +def test_outbox_replay_in_order_across_workers( + store: HarnessStore, caller: NativeCaller +): + a = HarnessRuntime(store, worker_id="w1") + b = HarnessRuntime(store, worker_id="w2") + snap = a.admit_snapshot(caller) + corr = a.new_correlation() + a.append_event( + session_id=caller.session_id, + kind="chunk", + message_id="m1", + correlation_id=corr, + snapshot_id=snap.snapshot_id, + ) + a.append_event( + session_id=caller.session_id, + kind="final", + message_id="m2", + correlation_id=corr, + snapshot_id=snap.snapshot_id, + ) + missed = b.replay_from(caller.session_id, f"{caller.session_id}:1") + assert [e.message_id for e in missed] == ["m2"] + assert missed[0].sequence == 2 + + +def test_two_worker_leases_and_drain(store: HarnessStore): + a = HarnessRuntime(store, worker_id="w1") + b = HarnessRuntime(store, worker_id="w2") + a.acquire_session_lease("sess-a") + b.acquire_session_lease("sess-b") + with pytest.raises(SessionBusy): + b.acquire_session_lease("sess-a") + snap = a.admit_snapshot(NativeCaller("ag", "u", "sess-a")) + corr = a.new_correlation() + a.start_turn(corr, NativeCaller("ag", "u", "sess-a"), snap) + a.bind_lease("sess-a", corr) + a.drain() + with pytest.raises(AdmissionRefused): + a.admit_snapshot(NativeCaller("ag", "u2", "sess-new"), force_new=True) + replay = b.replay_from("sess-a") + assert replay == [] + a.append_event( + session_id="sess-a", + kind="final", + message_id="kept", + correlation_id=corr, + snapshot_id=snap.snapshot_id, + ) + assert [e.message_id for e in b.replay_from("sess-a")] == ["kept"] + a.worker_lost("w1") + assert a.get_run(corr).state is TurnRunState.RECOVERY_REQUIRED + + +def test_untrusted_skill_refused_without_approved_backend( + rt: HarnessRuntime, caller: NativeCaller +): + snap = rt.admit_snapshot(caller) + digest = "abc123" + rt.register_manifest( + SkillManifest( + skill_key="scripty", + source="app", + digest=digest, + declared_tools=(), + capabilities=(), + trust_tier="untrusted", + spec="jv", + ) + ) + with pytest.raises(SkillIsolationRefused): + rt.activate_skill(caller, snap.snapshot_id, digest, trust_tier="untrusted") + hardened = HarnessRuntime(rt.store, worker_id="iso", isolation_backend="gvisor") + rec = hardened.activate_skill( + caller, snap.snapshot_id, digest, trust_tier="untrusted" + ) + assert rec.active + hardened.cleanup_stage(rec.path) + assert hardened.store.stages[rec.path].active is False + + +def test_stale_snapshot_cannot_activate_skill(rt: HarnessRuntime, caller: NativeCaller): + snap = rt.admit_snapshot(caller) + rt.register_manifest( + SkillManifest( + skill_key="k", + source="app", + digest="d1", + declared_tools=(), + capabilities=(), + trust_tier="trusted", + ) + ) + rt.invalidate(SnapshotSelector(snapshot_id=snap.snapshot_id)) + with pytest.raises(HarnessContractError): + rt.activate_skill(caller, snap.snapshot_id, "d1") + + +def test_third_skill_spec_rejected(rt: HarnessRuntime): + with pytest.raises(HarnessContractError, match="spec"): + rt.register_manifest( + SkillManifest( + skill_key="x", + source="app", + digest="d", + declared_tools=(), + capabilities=(), + trust_tier="trusted", + spec="weird", + ) + ) + + +def test_trace_isolated_by_caller(rt: HarnessRuntime, caller: NativeCaller): + other = NativeCaller("ag", "u2", "s2") + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rt.record_span(corr, "model_tick", caller=caller.as_tuple()) + doc = rt.replay_document(corr) + assert doc["caller"]["user_id"] == "u1" + assert all( + tuple(s.get("caller") or caller.as_tuple()) == caller.as_tuple() + for s in doc["spans"] + if "caller" in s + ) + other_corr = rt.new_correlation() + other_snap = rt.admit_snapshot(other) + rt.start_turn(other_corr, other, other_snap) + assert rt.traces_for(corr) != rt.traces_for(other_corr) + + +@pytest.mark.asyncio +async def test_distinct_session_leases_concurrent(store: HarnessStore): + a = HarnessRuntime(store, worker_id="w1") + b = HarnessRuntime(store, worker_id="w2") + + async def _hold(rt: HarnessRuntime, sid: str) -> None: + rt.acquire_session_lease(sid) + await asyncio.sleep(0.01) + rt.release_session_lease(sid) + + await asyncio.gather(_hold(a, "s-a"), _hold(b, "s-b")) + + +def test_compensatable_path(rt: HarnessRuntime, caller: NativeCaller): + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rec, _ = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="book", + payload={"n": 1}, + idempotency_class=IdempotencyClass.COMPENSATABLE, + ) + assert rt.compensate(rec.invocation_id).startswith("compensated:") + + +def test_completed_run_not_resumed(rt: HarnessRuntime, caller: NativeCaller): + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rt.complete_turn(corr) + with pytest.raises(HarnessContractError): + rt.resume_turn(corr) + + +def test_micro_benches_under_budget(rt: HarnessRuntime, caller: NativeCaller): + t0 = time.perf_counter() + for i in range(50): + c = NativeCaller("ag", f"u{i}", f"s{i}") + rt.admit_snapshot(c) + rt.upsert_user("mem", f"u{i}") + rt.upsert_conversation("mem", f"s{i}") + rt.append_event( + session_id=f"s{i}", + kind="chunk", + message_id=f"m{i}", + correlation_id=f"c{i}", + snapshot_id="snap", + ) + elapsed_ms = (time.perf_counter() - t0) * 1000 + assert elapsed_ms < 2000 From d83192240d2c140e288106c2c661addda98bc47c Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Thu, 17 Sep 2026 23:12:13 -0400 Subject: [PATCH 09/20] feat(harness): persist TurnRun checkpoints and close isolation gaps Checkpoint journals onto Interaction so resume can skip completed IDEMPOTENT calls; stage untrusted Claude skills only with an approved backend; mark mutating send/delete/bash tools NON_RETRYABLE. --- CHANGELOG.md | 2 +- docs/skill-isolation.md | 8 +- .../code_execution/code_execution_action.py | 68 ++++++- .../file_interface/file_interface_action.py | 16 +- .../google_drive_action.py | 11 +- .../google_gmail_action.py | 6 +- .../microsoft_outlook_mail_action.py | 6 +- jvagent/action/orchestrator/continuation.py | 31 +++- jvagent/action/orchestrator/loop.py | 5 +- .../orchestrator_interact_action.py | 34 +++- jvagent/action/orchestrator/skill_tasks.py | 9 +- jvagent/action/orchestrator/skills.py | 2 + jvagent/action/orchestrator/tools.py | 6 + .../pageindex_action/pageindex_action.py | 6 +- jvagent/action/whatsapp/whatsapp_action.py | 11 +- jvagent/embed/interact.py | 47 ++++- jvagent/harness/runtime.py | 172 +++++++++++++++++- jvagent/scaffold/skill_resolve.py | 10 + .../test_code_execution_action.py | 64 +++++++ tests/harness/test_runtime.py | 52 ++++++ tests/scaffold/test_skill_resolve.py | 17 ++ tests/test_embed_stream.py | 30 +++ 22 files changed, 580 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dd48067..f2efbc25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [PEP 440](https://peps.python.org/pep-0440/) / ### Added -- **Harness excellence runtime (HP-02 … HP-12).** `jvagent.harness.runtime` is the store-backed source of truth for NativeCaller admission, snapshot-keyed caches, TurnRun journals, invocation ledger, durable outbox, session leases, host providers, skill manifests/isolation, traces, and the HP-12 deployment matrix. Process-local bus/caches remain fan-out; JSON/SQLite active-active is unsupported. Docs: `docs/HARNESS_DEPLOYMENT.md`, `docs/skill-isolation.md`. +- **Harness excellence runtime (HP-02 … HP-12).** `jvagent.harness.runtime` is the store-backed source of truth for NativeCaller admission, snapshot-keyed caches, TurnRun journals, invocation ledger, durable outbox, session leases, host providers, skill manifests/isolation, traces, and the HP-12 deployment matrix. Process-local bus/caches remain fan-out; JSON/SQLite active-active is unsupported. Docs: `docs/HARNESS_DEPLOYMENT.md`, `docs/skill-isolation.md`. TurnRun checkpoints persist on `Interaction.observability_metrics`; loop resume skips completed IDEMPOTENT invocations; Claude skill staging is snapshot/digest-keyed and refuses untrusted isolation; mutating send/delete/bash tools declare `NON_RETRYABLE`; embed cancel marks TurnRun recovery. - **Harness baseline audit (HP-01).** Process-local bus, caches, breakers, and locks inventoried in `.planning/phases/01-contracts-and-baseline/PROCESS-LOCAL-STATE.md` with characterization tests. No replacements. diff --git a/docs/skill-isolation.md b/docs/skill-isolation.md index 9ec97ff1..9ae14b8c 100644 --- a/docs/skill-isolation.md +++ b/docs/skill-isolation.md @@ -17,8 +17,12 @@ Trusted SOP skills (no script) activate from digest under the admitted snapshot. ## Staging -Stage path is `stage/{session_id}/{snapshot_id}/{digest}`. Cleanup is -`cleanup_stage(path)`. A revoked or expired snapshot cannot activate. +Runtime stage record is `stage/{session_id}/{snapshot_id}/{digest}`. Filesystem copy +for `CodeExecutionAction.stage_skill` is `staged_skills/{snapshot_id[:12]}/{digest}/{name}` +when a turn snapshot is in cache; otherwise the legacy `staged_skills/{name}` dest +is kept for tests and offline staging. Cleanup is `cleanup_stage(path)`. A revoked +or expired snapshot cannot activate. Claude (`spec: claude`) activations pass +`trust_tier=untrusted` and refuse without an approved isolation backend. ## Audit diff --git a/jvagent/action/code_execution/code_execution_action.py b/jvagent/action/code_execution/code_execution_action.py index edd834e7..84d6e828 100644 --- a/jvagent/action/code_execution/code_execution_action.py +++ b/jvagent/action/code_execution/code_execution_action.py @@ -35,6 +35,7 @@ provision_user_sandbox, resolve_agent_user, ) +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool from jvagent.tooling.tool_executor import get_tool_visitor @@ -115,20 +116,75 @@ async def resolve_user_cwd(self, visitor: Any) -> str: ) return cwd - async def stage_skill(self, visitor: Any, skill_dir: str, name: str) -> str: + async def stage_skill( + self, + visitor: Any, + skill_dir: str, + name: str, + *, + trust_tier: str = "trusted", + ) -> str: """Copy an activated skill folder into the user's slice (read-on-use). Returns the path *relative to the sandbox cwd* (e.g. ``staged_skills/pdf-generation``) so a script can be run as - ``python staged_skills/pdf-generation/scripts/x.py``. Idempotent per - turn: re-staging refreshes the copy. + ``python staged_skills/pdf-generation/scripts/x.py``. When a turn + snapshot is in cache the dest is snapshot/digest-keyed. Idempotent per + turn: re-staging refreshes the copy. Untrusted skills refuse unless the + runtime has an approved isolation backend. """ + from jvagent.scaffold.skill_resolve import skill_digest + cwd = await self.resolve_user_cwd(visitor) - rel = f"{STAGED_SKILLS_DIR}/{name}" - dest = os.path.join(cwd, *rel.split("/")) src = Path(skill_dir) if not src.is_dir(): raise FileNotFoundError(f"skill dir not found: {skill_dir}") + digest = skill_digest(src) + rel = f"{STAGED_SKILLS_DIR}/{name}" + snap = None + caller = None + try: + from jvagent.action.orchestrator.turn_cache import get_turn_cache + + turn = get_turn_cache() or {} + snap = turn.get("snapshot") + caller = turn.get("caller") + except Exception: + pass + if snap is not None and caller is not None: + from jvagent.harness.runtime import SkillManifest, get_runtime + + rt = get_runtime() + rt.require_usable(snap.snapshot_id) + if digest not in rt.store.skill_manifests: + spec = "claude" if (src / "scripts").is_dir() else "jv" + rt.register_manifest( + SkillManifest( + skill_key=name, + source="stage", + digest=digest, + declared_tools=(), + capabilities=(), + trust_tier=trust_tier, + spec=spec, + ) + ) + rt.activate_skill(caller, snap.snapshot_id, digest, trust_tier=trust_tier) + rel = f"{STAGED_SKILLS_DIR}/{snap.snapshot_id[:12]}/{digest}/{name}" + elif trust_tier == "untrusted": + from jvagent.harness.runtime import ( + APPROVED_ISOLATION_BACKENDS, + SkillIsolationRefused, + get_runtime, + ) + + backend = get_runtime().isolation_backend + if backend not in APPROVED_ISOLATION_BACKENDS: + raise SkillIsolationRefused( + "untrusted skill requires an approved isolation backend " + f"(got {backend!r}; subprocess is not a sandbox)" + ) + dest = os.path.join(cwd, *rel.split("/")) if os.path.exists(dest): shutil.rmtree(dest, ignore_errors=True) shutil.copytree(src, dest) @@ -146,7 +202,7 @@ async def get_tools(self) -> List[Any]: return collect_tools(self) - @tool(name="code_execution__bash") + @tool(name="code_execution__bash", idempotency_class=IdempotencyClass.NON_RETRYABLE) async def _t_bash( self, command: Annotated[str, "Shell command to run in the sandbox."], diff --git a/jvagent/action/file_interface/file_interface_action.py b/jvagent/action/file_interface/file_interface_action.py index abe7bb50..580a02d2 100644 --- a/jvagent/action/file_interface/file_interface_action.py +++ b/jvagent/action/file_interface/file_interface_action.py @@ -16,6 +16,7 @@ from jvagent.action.base import Action from jvagent.action.file_interface import _core +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool from jvagent.tooling.tool_executor import get_tool_visitor @@ -71,7 +72,10 @@ async def _t_read_file( except Exception as e: return json.dumps({"ok": False, "error": f"{type(e).__name__}: {e}"}) - @tool(name="file_interface__write_file") + @tool( + name="file_interface__write_file", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_write_file( self, path: Annotated[str, "Relative path (e.g. output/notes.md)."], @@ -96,7 +100,10 @@ async def _t_write_file( except Exception as e: return json.dumps({"ok": False, "error": f"{type(e).__name__}: {e}"}) - @tool(name="file_interface__write_binary_file") + @tool( + name="file_interface__write_binary_file", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_write_binary_file( self, path: Annotated[str, "Relative path (e.g. output/report.pdf)."], @@ -154,7 +161,10 @@ async def _t_create_directory( except Exception as e: return json.dumps({"ok": False, "error": f"{type(e).__name__}: {e}"}) - @tool(name="file_interface__delete_file") + @tool( + name="file_interface__delete_file", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_delete_file( self, path: Annotated[str, "Relative file path."], diff --git a/jvagent/action/google/google_drive_action/google_drive_action.py b/jvagent/action/google/google_drive_action/google_drive_action.py index 1c2d87ae..eb5344ab 100644 --- a/jvagent/action/google/google_drive_action/google_drive_action.py +++ b/jvagent/action/google/google_drive_action/google_drive_action.py @@ -5,6 +5,7 @@ from googleapiclient.http import MediaIoBaseDownload from jvspatial.env import env +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool from ..google_action import GoogleAction @@ -423,7 +424,10 @@ async def _t_list_files( ) return json.dumps(results, indent=2) - @tool(name="google_drive__upload_file") + @tool( + name="google_drive__upload_file", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_upload_file( self, name: Annotated[str, "Name for the uploaded file."], @@ -527,7 +531,10 @@ async def _t_share_file( ) return json.dumps(result, indent=2) - @tool(name="google_drive__delete_file") + @tool( + name="google_drive__delete_file", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_delete_file( self, file_id: Annotated[str, "The ID of the file to delete."], diff --git a/jvagent/action/google/google_gmail_action/google_gmail_action.py b/jvagent/action/google/google_gmail_action/google_gmail_action.py index aa5bd6e1..43652ebe 100644 --- a/jvagent/action/google/google_gmail_action/google_gmail_action.py +++ b/jvagent/action/google/google_gmail_action/google_gmail_action.py @@ -7,6 +7,7 @@ standalone_mailbox_effective_sender_name, ) from jvagent.action.email_action.modules.gmail import GmailEmailProvider +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool from ..google_action import GoogleAction @@ -90,7 +91,10 @@ async def mark_read(self, message_id: str, user_id: str = "me") -> Dict[str, Any .execute() ) - @tool(name="gmail__send_email") + @tool( + name="gmail__send_email", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_send_email( self, to: Annotated[str, "Recipient email address."], diff --git a/jvagent/action/microsoft/microsoft_outlook_mail_action/microsoft_outlook_mail_action.py b/jvagent/action/microsoft/microsoft_outlook_mail_action/microsoft_outlook_mail_action.py index dca4f76c..a934e3e0 100644 --- a/jvagent/action/microsoft/microsoft_outlook_mail_action/microsoft_outlook_mail_action.py +++ b/jvagent/action/microsoft/microsoft_outlook_mail_action/microsoft_outlook_mail_action.py @@ -7,6 +7,7 @@ standalone_mailbox_effective_sender_name, ) from jvagent.action.email_action.modules.outlook import OutlookEmailProvider +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool from ..microsoft_action import MicrosoftAction @@ -157,7 +158,10 @@ async def get_profile(self, user_id: str = "me") -> Dict[str, Any]: "displayName": me.get("displayName"), } - @tool(name="outlook__send_email") + @tool( + name="outlook__send_email", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_send_email( self, to: Annotated[str, "Recipient email address."], diff --git a/jvagent/action/orchestrator/continuation.py b/jvagent/action/orchestrator/continuation.py index 475b2364..69c5afb3 100644 --- a/jvagent/action/orchestrator/continuation.py +++ b/jvagent/action/orchestrator/continuation.py @@ -16,7 +16,7 @@ from __future__ import annotations import logging -from typing import Any, FrozenSet, Optional, Set +from typing import Any, FrozenSet, Mapping, Optional, Set logger = logging.getLogger(__name__) @@ -495,6 +495,34 @@ async def cancel_orphan_flow_tasks( return cancelled +def completed_tool_observation( + tool_name: str, args: Optional[Mapping[str, Any]] = None +) -> Optional[str]: + """Cached IDEMPOTENT result for ``(tool_name, args)`` on the live TurnRun. + + Used on loop resume so a completed invocation is not dispatched again. + Returns ``None`` when there is no ledger hit (undeclared / non-idempotent + tools re-execute). + """ + try: + from jvagent.action.orchestrator.turn_cache import get_turn_cache + from jvagent.harness.runtime import get_runtime + except Exception: + return None + turn = get_turn_cache() or {} + corr = str(turn.get("correlation_id") or "") + if not corr: + return None + try: + return get_runtime().peek_completed_result( + correlation_id=corr, + tool_name=tool_name, + payload=dict(args or {}), + ) + except Exception: + return None + + __all__ = [ "active_flow_owner", "active_flow_note", @@ -511,4 +539,5 @@ async def cancel_orphan_flow_tasks( "task_lock_progress_count", "task_lock_title", "SOFT_ABANDON_ASK_STRIKE", + "completed_tool_observation", ] diff --git a/jvagent/action/orchestrator/loop.py b/jvagent/action/orchestrator/loop.py index 023d4e36..9d46eb8d 100644 --- a/jvagent/action/orchestrator/loop.py +++ b/jvagent/action/orchestrator/loop.py @@ -1443,7 +1443,10 @@ async def _dispatch_tool( ) tool_t0 = time.perf_counter() try: - if tool_call_timeout > 0: + cached = continuation.completed_tool_observation(tool_name, args) + if cached is not None: + obs = cached + elif tool_call_timeout > 0: obs = await asyncio.wait_for( tool.run(args), timeout=tool_call_timeout ) diff --git a/jvagent/action/orchestrator/orchestrator_interact_action.py b/jvagent/action/orchestrator/orchestrator_interact_action.py index 15a44068..e4efe413 100644 --- a/jvagent/action/orchestrator/orchestrator_interact_action.py +++ b/jvagent/action/orchestrator/orchestrator_interact_action.py @@ -947,7 +947,7 @@ async def execute(self, visitor: "InteractWalker") -> None: if interaction is None: return with bind_turn_cache() as cache: - from jvagent.harness.contracts import NativeCaller + from jvagent.harness.contracts import NativeCaller, TurnRunState from jvagent.harness.runtime import AdmissionRefused, get_runtime rt = get_runtime() @@ -960,6 +960,7 @@ async def execute(self, visitor: "InteractWalker") -> None: logger.info("harness admission refused: draining") return cache["caller"] = caller + cache["interaction"] = interaction cache["correlation_id"] = ( getattr(visitor, "correlation_id", "") or rt.new_correlation() ) @@ -968,18 +969,37 @@ async def execute(self, visitor: "InteractWalker") -> None: except AdmissionRefused: logger.info("harness snapshot admission refused") return - rt.start_turn( - cache["correlation_id"], - caller, - cache["snapshot"], - interaction_id=str(getattr(interaction, "id", "") or ""), - ) + restored = None + payload = rt.checkpoint_from_interaction(interaction) + if payload: + restored = rt.import_checkpoint(payload) + if restored is not None and restored.state not in ( + TurnRunState.COMPLETED, + TurnRunState.FAILED, + TurnRunState.CANCELLED, + ): + cache["correlation_id"] = restored.correlation_id + if restored.state is TurnRunState.WAITING_TOOL: + rt.transition( + restored.correlation_id, + TurnRunState.RUNNING, + reason="resume", + ) + else: + rt.start_turn( + cache["correlation_id"], + caller, + cache["snapshot"], + interaction_id=str(getattr(interaction, "id", "") or ""), + ) try: await self._execute_turn(visitor) rt.complete_turn(cache["correlation_id"]) except Exception: rt.fail_turn(cache["correlation_id"], reason="execute_error") raise + finally: + rt.persist_to_interaction(interaction, cache["correlation_id"]) async def _execute_turn(self, visitor: "InteractWalker") -> None: # Curate the remaining walk path: routable IAs (exposed as tools) must diff --git a/jvagent/action/orchestrator/skill_tasks.py b/jvagent/action/orchestrator/skill_tasks.py index 4eeec1a2..6964bc6c 100644 --- a/jvagent/action/orchestrator/skill_tasks.py +++ b/jvagent/action/orchestrator/skill_tasks.py @@ -592,7 +592,14 @@ async def _activate(doc: Any) -> Optional[str]: directory = getattr(doc, "directory", "") or "" if directory: try: - rel = await code_exec.stage_skill(visitor, directory, doc.name) + trust = ( + "untrusted" + if getattr(doc, "spec", "jv") == "claude" + else "trusted" + ) + rel = await code_exec.stage_skill( + visitor, directory, doc.name, trust_tier=trust + ) notes.append( f"This skill's files are staged at '{rel}/' in your sandbox. Run " f"its scripts with the code_execution__bash tool — e.g. " diff --git a/jvagent/action/orchestrator/skills.py b/jvagent/action/orchestrator/skills.py index c8fc5360..68390edd 100644 --- a/jvagent/action/orchestrator/skills.py +++ b/jvagent/action/orchestrator/skills.py @@ -82,6 +82,7 @@ class SkillDoc: allowed_channels: Tuple[str, ...] = () denied_channels: Tuple[str, ...] = () deny_access_directive: str = "" + digest: str = "" metadata: dict = field(default_factory=dict) @@ -218,6 +219,7 @@ def discover_skill_docs( allowed_channels=tuple(bundle.get("allowed_channels") or ()), denied_channels=tuple(bundle.get("denied_channels") or ()), deny_access_directive=str(bundle.get("deny_access_directive") or ""), + digest=str(bundle.get("digest") or ""), metadata=bundle.get("metadata") or {}, ) ) diff --git a/jvagent/action/orchestrator/tools.py b/jvagent/action/orchestrator/tools.py index b1b551e3..063715ee 100644 --- a/jvagent/action/orchestrator/tools.py +++ b/jvagent/action/orchestrator/tools.py @@ -106,12 +106,14 @@ async def _run(args: Dict[str, Any], _tool: Any = tool) -> str: record = None runtime = None correlation_id = "" + interaction = None try: from jvagent.action.orchestrator.turn_cache import get_turn_cache from jvagent.harness.runtime import get_runtime turn = get_turn_cache() or {} snap = turn.get("snapshot") + interaction = turn.get("interaction") correlation_id = str(turn.get("correlation_id") or "") if snap is not None and correlation_id: runtime = get_runtime() @@ -144,6 +146,8 @@ async def _run(args: Dict[str, Any], _tool: Any = tool) -> str: result=f"(tool error: {exc})", ok=False, ) + if interaction is not None: + runtime.persist_to_interaction(interaction, correlation_id) return f"(tool error: {exc})" content = (getattr(result, "content", "") or "") if result is not None else "" if runtime is not None and record is not None: @@ -153,6 +157,8 @@ async def _run(args: Dict[str, Any], _tool: Any = tool) -> str: result=content, ok=True, ) + if interaction is not None: + runtime.persist_to_interaction(interaction, correlation_id) return content schema = getattr(tool, "parameters_schema", None) diff --git a/jvagent/action/pageindex/pageindex_action/pageindex_action.py b/jvagent/action/pageindex/pageindex_action/pageindex_action.py index 52751623..32a88c9a 100644 --- a/jvagent/action/pageindex/pageindex_action/pageindex_action.py +++ b/jvagent/action/pageindex/pageindex_action/pageindex_action.py @@ -21,6 +21,7 @@ from jvagent.action.base import Action from jvagent.core.public_url import get_public_base_url from jvagent.env import get_jvagent_jvforge_base_url +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import collect_tools, tool from .. import llm_bridge @@ -877,7 +878,10 @@ def _dump(docs: list) -> str: ) return payload - @tool(name="pageindex__delete") + @tool( + name="pageindex__delete", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_delete_doc( self, doc_name: Annotated[str, "Name of the document to delete."], diff --git a/jvagent/action/whatsapp/whatsapp_action.py b/jvagent/action/whatsapp/whatsapp_action.py index 629e7bb8..8ab87163 100644 --- a/jvagent/action/whatsapp/whatsapp_action.py +++ b/jvagent/action/whatsapp/whatsapp_action.py @@ -15,6 +15,7 @@ from jvagent.action.base import Action from jvagent.core.public_url import get_public_base_url +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool from jvagent.tooling.tool_executor import get_dispatch_context, get_tool_visitor @@ -1587,7 +1588,10 @@ async def list_templates(self) -> str: logger.exception("whatsapp__list_templates failed") return json.dumps({"ok": False, "error": f"{type(e).__name__}: {e}"}) - @tool(name="whatsapp__send_template") + @tool( + name="whatsapp__send_template", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def send_template( self, template_name: Annotated[ @@ -1804,7 +1808,10 @@ async def list_flows(self) -> str: logger.exception("whatsapp__list_flows failed") return json.dumps({"ok": False, "error": f"{type(e).__name__}: {e}"}) - @tool(name="whatsapp__send_flow") + @tool( + name="whatsapp__send_flow", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def send_flow( self, flow_id: Annotated[ diff --git a/jvagent/embed/interact.py b/jvagent/embed/interact.py index 58d65919..fde5ecac 100644 --- a/jvagent/embed/interact.py +++ b/jvagent/embed/interact.py @@ -47,15 +47,48 @@ def cancel_interact( session_id: Optional[str] = None, thread_id: Optional[str] = None, ) -> bool: - """Cancel an in-flight :func:`interact_stream` walker task, if any.""" + """Cancel an in-flight :func:`interact_stream` walker task, if any. + + Task handle cancel is the delivery interrupt. TurnRun is the source of + truth: the matching session journal is marked ``recovery_required``. + """ + cancelled = False for key in (thread_id, session_id): if not key: continue task = _interact_tasks.get(key) if task is not None and not task.done(): task.cancel() - return True - return False + cancelled = True + if cancelled: + _mark_embed_recovery(session_id=session_id, thread_id=thread_id) + return cancelled + + +def _mark_embed_recovery( + *, + session_id: Optional[str] = None, + thread_id: Optional[str] = None, +) -> None: + try: + from jvagent.harness.runtime import get_runtime + + rt = get_runtime() + for sid in (session_id, thread_id): + if not sid: + continue + corr = rt.correlation_for_session(sid) + if not corr: + continue + try: + rt.mark_recovery(corr, reason="embed_cancel") + except Exception: + logger.debug( + "embed.cancel_interact: mark_recovery failed corr=%s", corr + ) + break + except Exception: + logger.debug("embed.cancel_interact: harness recovery skip", exc_info=True) async def interact( @@ -480,6 +513,14 @@ async def _disc() -> bool: if sid not in task_keys: task_keys.append(sid) await _register_interact_task(sid, walk_task) + try: + from jvagent.harness.runtime import get_runtime + + corr = getattr(walker, "correlation_id", "") or "" + if corr: + get_runtime().bind_lease(sid, corr) + except Exception: + logger.debug("embed.interact_stream: lease bind skip", exc_info=True) # Stream messages off the response bus until the walker finishes. if walker.response_bus and walker.session_id: diff --git a/jvagent/harness/runtime.py b/jvagent/harness/runtime.py index e9f48f7f..b8ded2c0 100644 --- a/jvagent/harness/runtime.py +++ b/jvagent/harness/runtime.py @@ -11,7 +11,7 @@ import threading import time import uuid -from dataclasses import dataclass, field, replace +from dataclasses import asdict, dataclass, field, replace from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Mapping, Optional, Tuple @@ -27,6 +27,7 @@ ToolSurfaceSnapshot, TurnRunState, assert_turn_run_transition, + native_caller_from_mapping, reject_host_domain_fields, reject_model_authority_fields, ) @@ -37,6 +38,7 @@ MAX_EVENTS_PER_SESSION = 10_000 MAX_OBSERVATION_CHARS = 8_000 APPROVED_ISOLATION_BACKENDS = frozenset({"gvisor", "firecracker", "nsjail"}) +CHECKPOINT_KIND = "harness.turn_run" _runtime_guard = threading.Lock() _runtime: Optional["HarnessRuntime"] = None @@ -339,6 +341,173 @@ def list_journal( journal = self._require_run(correlation_id) return journal.entries[offset : offset + limit] + def peek_completed_result( + self, + *, + correlation_id: str, + tool_name: str, + payload: Mapping[str, Any], + ) -> Optional[str]: + """Cached IDEMPOTENT result for this (tool, args). Does not bump attempt.""" + digest = _input_digest(tool_name, payload) + ledger_key = f"{correlation_id}:{tool_name}:{digest}" + with self.store.lock: + existing = self.store.invocations.get(ledger_key) + if existing is None: + return None + if existing.idempotency_class is not IdempotencyClass.IDEMPOTENT: + return None + return self.store.invocation_results.get(existing.invocation_id) + + def correlation_for_session(self, session_id: str) -> Optional[str]: + if not session_id: + return None + held = self.store.leases.get(session_id) + if held: + corr = str(held.get("correlation_id") or "") + if corr: + return corr + with self.store.lock: + for corr, journal in self.store.runs.items(): + if journal.caller.session_id == session_id and ( + journal.state not in TURN_RUN_TERMINAL + ): + return corr + return None + + def export_checkpoint(self, correlation_id: str) -> Dict[str, Any]: + journal = self._require_run(correlation_id) + snap = self.store.snapshots.get(journal.snapshot_id) + with self.store.lock: + prefix = f"{correlation_id}:" + invocations: List[Dict[str, Any]] = [] + for key, rec in self.store.invocations.items(): + if not key.startswith(prefix): + continue + rec_map = asdict(rec) + klass = rec.idempotency_class + rec_map["idempotency_class"] = klass.value if klass else None + invocations.append( + { + "ledger_key": key, + "record": rec_map, + "result": self.store.invocation_results.get(rec.invocation_id), + } + ) + outbox = [ + asdict(e) for e in self.store.outbox.get(journal.caller.session_id, []) + ] + snap_map: Optional[Dict[str, Any]] = None + if snap is not None: + snap_map = asdict(snap) + snap_map["caller"] = snap.caller.to_mapping() + return { + "correlation_id": journal.correlation_id, + "state": journal.state.value, + "snapshot_id": journal.snapshot_id, + "interaction_id": journal.interaction_id, + "seq": journal.seq, + "completed_invocation_ids": list(journal.completed_invocation_ids), + "observation_refs": list(journal.observation_refs), + "plan_phase": journal.plan_phase, + "reason": journal.reason, + "entries": list(journal.entries), + "invocations": invocations, + "outbox": outbox, + "caller": journal.caller.to_mapping(), + "snapshot": snap_map, + "worker_id": journal.worker_id, + } + + def import_checkpoint(self, payload: Mapping[str, Any]) -> TurnRunJournal: + caller = native_caller_from_mapping(payload["caller"]) + journal = TurnRunJournal( + correlation_id=str(payload["correlation_id"]), + caller=caller, + state=TurnRunState(str(payload["state"])), + snapshot_id=str(payload.get("snapshot_id") or ""), + interaction_id=str(payload.get("interaction_id") or ""), + seq=int(payload.get("seq") or 0), + worker_id=str(payload.get("worker_id") or self.worker_id), + entries=list(payload.get("entries") or []), + completed_invocation_ids=list( + payload.get("completed_invocation_ids") or [] + ), + observation_refs=list(payload.get("observation_refs") or []), + plan_phase=str(payload.get("plan_phase") or ""), + reason=str(payload.get("reason") or ""), + ) + snap_raw = payload.get("snapshot") + snap: Optional[ToolSurfaceSnapshot] = None + if isinstance(snap_raw, dict) and snap_raw.get("snapshot_id"): + snap = ToolSurfaceSnapshot( + snapshot_id=str(snap_raw["snapshot_id"]), + caller=native_caller_from_mapping(snap_raw["caller"]), + native_tool_names=tuple(snap_raw.get("native_tool_names") or ()), + native_skill_keys=tuple(snap_raw.get("native_skill_keys") or ()), + host_tool_names=tuple(snap_raw.get("host_tool_names") or ()), + host_skill_keys=tuple(snap_raw.get("host_skill_keys") or ()), + created_at=str(snap_raw.get("created_at") or ""), + expires_at=str(snap_raw.get("expires_at") or ""), + revoked=bool(snap_raw.get("revoked")), + ) + with self.store.lock: + self.store.runs[journal.correlation_id] = journal + if journal.interaction_id: + self.store.runs_by_interaction[journal.interaction_id] = ( + journal.correlation_id + ) + if snap is not None: + self.store.snapshots[snap.snapshot_id] = snap + for item in payload.get("invocations") or []: + rec_map = dict(item.get("record") or {}) + klass_raw = rec_map.get("idempotency_class") + rec = InvocationRecord( + invocation_id=str(rec_map["invocation_id"]), + snapshot_id=str(rec_map.get("snapshot_id") or ""), + tool_name=str(rec_map.get("tool_name") or ""), + input_digest=str(rec_map.get("input_digest") or ""), + idempotency_class=( + IdempotencyClass(klass_raw) if klass_raw else None + ), + attempt=int(rec_map.get("attempt") or 1), + outcome=rec_map.get("outcome"), + ) + key = str(item.get("ledger_key") or "") + if key: + self.store.invocations[key] = rec + result = item.get("result") + if result is not None: + self.store.invocation_results[rec.invocation_id] = str(result) + envelopes = [EventEnvelope(**raw) for raw in (payload.get("outbox") or [])] + if envelopes: + self.store.outbox[caller.session_id] = envelopes + return journal + + def persist_to_interaction(self, interaction: Any, correlation_id: str) -> None: + if interaction is None or not correlation_id: + return + if self.get_run(correlation_id) is None: + return + payload = self.export_checkpoint(correlation_id) + metrics = [ + m + for m in list(getattr(interaction, "observability_metrics", None) or []) + if not (isinstance(m, dict) and m.get("kind") == CHECKPOINT_KIND) + ] + metrics.append({"kind": CHECKPOINT_KIND, "payload": payload}) + interaction.observability_metrics = metrics + + def checkpoint_from_interaction(self, interaction: Any) -> Optional[Dict[str, Any]]: + if interaction is None: + return None + for metric in getattr(interaction, "observability_metrics", None) or []: + if isinstance(metric, dict) and metric.get("kind") == CHECKPOINT_KIND: + payload = metric.get("payload") + if isinstance(payload, dict): + return payload + return None + # -- invocation ledger (HP-05) ---------------------------------------- def begin_invocation( @@ -727,6 +896,7 @@ def set_runtime(runtime: HarnessRuntime) -> None: __all__ = [ "APPROVED_ISOLATION_BACKENDS", + "CHECKPOINT_KIND", "AdmissionRefused", "HarnessRuntime", "HarnessStore", diff --git a/jvagent/scaffold/skill_resolve.py b/jvagent/scaffold/skill_resolve.py index efc37503..1ab357c8 100644 --- a/jvagent/scaffold/skill_resolve.py +++ b/jvagent/scaffold/skill_resolve.py @@ -3,6 +3,7 @@ from __future__ import annotations import fnmatch +import hashlib import importlib import logging import os @@ -24,6 +25,14 @@ SELECTOR_ALL = "-all" +def skill_digest(skill_dir: Union[str, Path]) -> str: + """SHA-256 prefix of ``SKILL.md`` (or the file itself). Empty if missing.""" + path = Path(skill_dir) + skill_md = path / "SKILL.md" if path.is_dir() else path + blob = skill_md.read_bytes() if skill_md.is_file() else b"" + return hashlib.sha256(blob).hexdigest()[:16] + + _KNOWN_FRONTMATTER_KEYS = frozenset( { "allowed-channels", @@ -439,6 +448,7 @@ def parse_skill_bundle( "deny_access_directive": deny_access_directive, "scope_hint": scope_hint, "source": source, + "digest": skill_digest(skill_file), "metadata": { "version": frontmatter.get("version"), "license": frontmatter.get("license"), diff --git a/tests/action/code_execution/test_code_execution_action.py b/tests/action/code_execution/test_code_execution_action.py index 74377dec..0b91fc8f 100644 --- a/tests/action/code_execution/test_code_execution_action.py +++ b/tests/action/code_execution/test_code_execution_action.py @@ -101,3 +101,67 @@ async def _fake_cwd(self, visitor): rel = await action.stage_skill(_Visitor(), str(skill), "demo") assert rel == "staged_skills/demo" assert (slice_dir / "staged_skills" / "demo" / "scripts" / "x.py").exists() + + +async def test_stage_skill_snapshot_keyed(tmp_path, monkeypatch): + from jvagent.action.orchestrator.turn_cache import bind_turn_cache + from jvagent.harness.contracts import NativeCaller + from jvagent.harness.runtime import reset_runtime + from jvagent.scaffold.skill_resolve import skill_digest + + skill = tmp_path / "src_skill" + (skill / "scripts").mkdir(parents=True) + (skill / "SKILL.md").write_text("---\nname: demo\n---\nbody\n") + (skill / "scripts" / "x.py").write_text("print('hi')\n") + slice_dir = tmp_path / "slice" + slice_dir.mkdir() + action = CodeExecutionAction() + action.enabled = True + + async def _fake_cwd(self, visitor): + return str(slice_dir) + + monkeypatch.setattr(CodeExecutionAction, "resolve_user_cwd", _fake_cwd) + rt = reset_runtime() + caller = NativeCaller("ag", "u1", "s1") + snap = rt.admit_snapshot(caller) + digest = skill_digest(skill) + with bind_turn_cache() as cache: + cache["caller"] = caller + cache["snapshot"] = snap + rel = await action.stage_skill(_Visitor(), str(skill), "demo") + assert rel == f"staged_skills/{snap.snapshot_id[:12]}/{digest}/demo" + assert (slice_dir / rel / "scripts" / "x.py").exists() + reset_runtime() + + +async def test_stage_skill_untrusted_refuses_without_isolation(tmp_path, monkeypatch): + from jvagent.harness.runtime import SkillIsolationRefused, reset_runtime + + skill = tmp_path / "src_skill" + skill.mkdir() + (skill / "SKILL.md").write_text("---\nname: demo\n---\nbody\n") + slice_dir = tmp_path / "slice" + slice_dir.mkdir() + action = CodeExecutionAction() + + async def _fake_cwd(self, visitor): + return str(slice_dir) + + monkeypatch.setattr(CodeExecutionAction, "resolve_user_cwd", _fake_cwd) + reset_runtime() + try: + await action.stage_skill(_Visitor(), str(skill), "demo", trust_tier="untrusted") + raise AssertionError("expected SkillIsolationRefused") + except SkillIsolationRefused: + pass + reset_runtime() + + +async def test_bash_tool_is_non_retryable(): + from jvagent.harness.contracts import IdempotencyClass + + action = CodeExecutionAction() + action.enabled = True + tools = await action.get_tools() + assert tools[0].idempotency_class is IdempotencyClass.NON_RETRYABLE diff --git a/tests/harness/test_runtime.py b/tests/harness/test_runtime.py index 2a9283e2..17db76e9 100644 --- a/tests/harness/test_runtime.py +++ b/tests/harness/test_runtime.py @@ -371,3 +371,55 @@ def test_micro_benches_under_budget(rt: HarnessRuntime, caller: NativeCaller): ) elapsed_ms = (time.perf_counter() - t0) * 1000 assert elapsed_ms < 2000 + + +class _FakeInteraction: + def __init__(self) -> None: + self.observability_metrics: list = [] + + +def test_checkpoint_round_trip_restores_idempotent_result( + rt: HarnessRuntime, caller: NativeCaller +): + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap, interaction_id="int-ckpt") + rec, _ = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="echo", + payload={"x": 1}, + idempotency_class=IdempotencyClass.IDEMPOTENT, + ) + rt.finish_invocation(correlation_id=corr, record=rec, result="hello") + interaction = _FakeInteraction() + rt.persist_to_interaction(interaction, corr) + other = HarnessRuntime(HarnessStore(), worker_id="w2") + payload = other.checkpoint_from_interaction(interaction) + assert payload is not None + restored = other.import_checkpoint(payload) + assert restored.correlation_id == corr + assert rec.invocation_id in restored.completed_invocation_ids + cached = other.peek_completed_result( + correlation_id=corr, tool_name="echo", payload={"x": 1} + ) + assert cached == "hello" + + +def test_peek_completed_skips_non_idempotent(rt: HarnessRuntime, caller: NativeCaller): + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rec, _ = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="send", + payload={"n": 1}, + ) + rt.finish_invocation(correlation_id=corr, record=rec, result="sent") + assert ( + rt.peek_completed_result( + correlation_id=corr, tool_name="send", payload={"n": 1} + ) + is None + ) diff --git a/tests/scaffold/test_skill_resolve.py b/tests/scaffold/test_skill_resolve.py index 83afddcb..eecaabfa 100644 --- a/tests/scaffold/test_skill_resolve.py +++ b/tests/scaffold/test_skill_resolve.py @@ -8,9 +8,11 @@ from jvagent.scaffold.skill_resolve import ( apply_skill_selector, + parse_skill_bundle, resolve_agent_skills, resolve_builtin_skills, resolve_merged_skill_bundles, + skill_digest, ) @@ -29,6 +31,21 @@ def test_resolve_builtin_skills_contains_catalog_entries() -> None: assert "excel" in skills +def test_skill_digest_is_stable_for_skill_md(tmp_path: Path) -> None: + skill_dir = tmp_path / "hashed" + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_text( + "---\nname: hashed\ndescription: d\n---\nbody\n" + ) + first = skill_digest(skill_dir) + second = skill_digest(skill_dir) + assert first == second + assert len(first) == 16 + bundle = parse_skill_bundle(skill_dir, source="app") + assert bundle is not None + assert bundle["digest"] == first + + def test_resolve_agent_skills_reads_app_local_bundle(tmp_path: Path) -> None: skill_dir = tmp_path / "agents" / "acme" / "bot" / "skills" / "my_skill" skill_dir.mkdir(parents=True) diff --git a/tests/test_embed_stream.py b/tests/test_embed_stream.py index b71645b4..2a9ffa11 100644 --- a/tests/test_embed_stream.py +++ b/tests/test_embed_stream.py @@ -11,6 +11,8 @@ _register_interact_task, cancel_interact, ) +from jvagent.harness.contracts import NativeCaller, TurnRunState +from jvagent.harness.runtime import reset_runtime @pytest.mark.asyncio @@ -28,6 +30,34 @@ async def slow_walk() -> None: await task +@pytest.mark.asyncio +async def test_cancel_interact_marks_turn_recovery() -> None: + rt = reset_runtime() + caller = NativeCaller("ag", "u1", "sess-recover") + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rt.acquire_session_lease("sess-recover") + rt.bind_lease("sess-recover", corr) + + async def slow_walk() -> None: + await asyncio.sleep(30) + + task = asyncio.create_task(slow_walk()) + await _register_interact_task("sess-recover", task) + await asyncio.sleep(0) + + assert cancel_interact(session_id="sess-recover") is True + journal = rt.get_run(corr) + assert journal is not None + assert journal.state is TurnRunState.RECOVERY_REQUIRED + assert journal.reason == "embed_cancel" + + with pytest.raises(asyncio.CancelledError): + await task + reset_runtime() + + @pytest.mark.asyncio async def test_register_interact_task_cancels_prior_stream_for_same_key() -> None: first_done = asyncio.Event() From e3b16657e9eb70392bbcca896d084a7b1a665691 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Thu, 17 Sep 2026 23:43:48 -0400 Subject: [PATCH 10/20] =?UTF-8?q?feat(harness):=20close=20remaining=20HP-0?= =?UTF-8?q?8=E2=80=93HP-12=20runtime=20gaps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live host invoke, signed skill revoke, durable dump/leases, and CI/CUCS evidence so acceptance no longer sits on deferred stubs. --- .github/workflows/test-jvagent.yaml | 48 +++ .planning/STATE.md | 21 +- CHANGELOG.md | 2 +- docs/HARNESS_DEPLOYMENT.md | 12 + docs/skill-isolation.md | 10 + .../artifact_handler_interact_action.py | 11 +- .../code_execution/code_execution_action.py | 10 +- .../file_interface/file_interface_action.py | 5 +- .../google_calendar_action.py | 11 +- .../google_drive_action.py | 5 +- .../google_gmail_action.py | 5 +- .../google_sheets_action.py | 61 +++- .../microsoft_excel_action.py | 46 ++- .../microsoft_onedrive_action.py | 16 +- .../microsoft_outlook_calendar_action.py | 11 +- .../microsoft_outlook_mail_action.py | 5 +- .../orchestrator_interact_action.py | 3 + jvagent/action/orchestrator/tools.py | 33 ++ .../pageindex_action/pageindex_action.py | 5 +- jvagent/action/skill_hub/skill_hub_action.py | 11 +- jvagent/harness/isolation.py | 98 +++++ jvagent/harness/leases.py | 340 ++++++++++++++++++ jvagent/harness/persist.py | 235 ++++++++++++ jvagent/harness/provider.py | 26 +- jvagent/harness/release.py | 5 +- jvagent/harness/runtime.py | 134 ++++++- pyproject.toml | 2 + tests/conformance/cucs/recovery.yaml | 26 ++ tests/conformance/cucs/safety-untrusted.yaml | 24 ++ tests/conformance/cucs/skill-activation.yaml | 25 ++ tests/conformance/cucs/tool-selection.yaml | 27 ++ tests/conformance/cucs/uniqueness.yaml | 32 ++ .../fixtures/fake_host/__init__.py | 5 + tests/conformance/test_cucs_harness.py | 138 +++++++ tests/harness/test_gap_close.py | 260 ++++++++++++++ tests/harness/test_load.py | 98 +++++ tests/harness/test_runtime.py | 2 + 37 files changed, 1746 insertions(+), 62 deletions(-) create mode 100644 jvagent/harness/isolation.py create mode 100644 jvagent/harness/leases.py create mode 100644 jvagent/harness/persist.py create mode 100644 tests/conformance/cucs/recovery.yaml create mode 100644 tests/conformance/cucs/safety-untrusted.yaml create mode 100644 tests/conformance/cucs/skill-activation.yaml create mode 100644 tests/conformance/cucs/tool-selection.yaml create mode 100644 tests/conformance/cucs/uniqueness.yaml create mode 100644 tests/conformance/test_cucs_harness.py create mode 100644 tests/harness/test_gap_close.py create mode 100644 tests/harness/test_load.py diff --git a/.github/workflows/test-jvagent.yaml b/.github/workflows/test-jvagent.yaml index 2e6094b8..1e6e2a23 100644 --- a/.github/workflows/test-jvagent.yaml +++ b/.github/workflows/test-jvagent.yaml @@ -59,6 +59,54 @@ jobs: pip install pre-commit pre-commit run --all-files + harness-conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.11' + - run: python -m pip install --upgrade pip + - run: python -m pip install -e '.[test]' + - name: Conformance lane + run: pytest tests/conformance tests/harness -m harness_conformance -q --tb=short + + harness-two-worker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.11' + - run: python -m pip install --upgrade pip + - run: python -m pip install -e '.[test]' + - name: Two-worker / crash-recovery lane + run: pytest tests/conformance/test_leases.py tests/conformance/test_invocation_recovery.py -q --tb=short + + harness-isolation: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.11' + - run: python -m pip install --upgrade pip + - run: python -m pip install -e '.[test]' + - name: Skill isolation lane + run: pytest tests/ -m harness_isolation -q --tb=short + + harness-load: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.11' + - run: python -m pip install --upgrade pip + - run: python -m pip install -e '.[test]' + - name: Load lane + run: pytest tests/ -m harness_load -q --tb=short + jvchat: runs-on: ubuntu-latest # A registry stall in `npm ci` once held this job for the 45-minute default diff --git a/.planning/STATE.md b/.planning/STATE.md index 43212194..d9ee8c31 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -5,14 +5,14 @@ See: .planning/PROJECT.md (updated 2026-09-17) **Core value:** Dependable graph-native harness — model as pilot, tools as controls, skills as flight plan. -**Current focus:** v2.0 Harness Excellence — Phases 1–5 implemented; awaiting user commit/PR +**Current focus:** v2.0 Harness Excellence — Phases 1–5 implemented; gap-close in working tree ## Current Position Phase: 5 of 5 (Operational excellence and release proof) Plan: 3 of 3 in current phase -Status: Implementation complete; not committed -Last activity: 2026-09-18 — HP-02 … HP-12 runtime + wires +Status: Implementation complete including gap-close; not committed +Last activity: 2026-09-18 — HP-02 … HP-12 runtime + wires + remaining gaps Progress: [██████████] 100% @@ -47,6 +47,8 @@ Progress: [██████████] 100% - TurnRun is a journal Object (I-GRAPH-02), not a conversation Node - JSON/SQLite active-active is unsupported - Subprocess ≠ sandbox +- Isolation binary missing → refuse, never subprocess fallback +- Redis/Dynamo lease adapters require an explicit client ### Pending Todos @@ -54,22 +56,17 @@ User will commit and open PR. ### Blockers/Concerns -- Durable outbox transport swap still deferred (HarnessStore is the first backend) -- Active-active for Mongo/Postgres marked degraded until Redis/Dynamo leases are configured +- Real gvisor/firecracker/nsjail kernel jails are not implemented in-tree; wrap prefix + PATH check is the contract +- Active-active for Mongo/Postgres stays degraded until a shared lease backend is configured ## Deferred Items -| Category | Item | Status | Deferred At | -|----------|------|--------|-------------| -| Transport | First durable outbox/coordination transport | Future | v2.0 start | -| Retention | Separate checkpoint vs event retention per backend | Future | v2.0 start | -| Workers | Background work vs TurnRun executor | Future | v2.0 start | -| Skills | External publisher registry + revocation service | Future | v2.0 start | +None remaining from the v2.0 gap list. Optional later work: Redis/Dynamo *stream* transports (leases already have adapters), remote publisher service (in-process revoke/publish is the registry). ## Session Continuity Last session: 2026-09-18 -Stopped at: Phases 2–5 implemented (HP-02 … HP-12) +Stopped at: All listed HP gaps closed in working tree Resume file: None Next: user commit + PR diff --git a/CHANGELOG.md b/CHANGELOG.md index f2efbc25..0ab742d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [PEP 440](https://peps.python.org/pep-0440/) / ### Added -- **Harness excellence runtime (HP-02 … HP-12).** `jvagent.harness.runtime` is the store-backed source of truth for NativeCaller admission, snapshot-keyed caches, TurnRun journals, invocation ledger, durable outbox, session leases, host providers, skill manifests/isolation, traces, and the HP-12 deployment matrix. Process-local bus/caches remain fan-out; JSON/SQLite active-active is unsupported. Docs: `docs/HARNESS_DEPLOYMENT.md`, `docs/skill-isolation.md`. TurnRun checkpoints persist on `Interaction.observability_metrics`; loop resume skips completed IDEMPOTENT invocations; Claude skill staging is snapshot/digest-keyed and refuses untrusted isolation; mutating send/delete/bash tools declare `NON_RETRYABLE`; embed cancel marks TurnRun recovery. +- **Harness excellence runtime (HP-02 … HP-12).** `jvagent.harness.runtime` is the store-backed source of truth for NativeCaller admission, snapshot-keyed caches, TurnRun journals, invocation ledger, durable outbox, session leases, host providers, skill manifests/isolation, traces, and the HP-12 deployment matrix. Process-local bus/caches remain fan-out; JSON/SQLite active-active is unsupported. Docs: `docs/HARNESS_DEPLOYMENT.md`, `docs/skill-isolation.md`. TurnRun checkpoints persist on `Interaction.observability_metrics`; loop resume skips completed IDEMPOTENT invocations; Claude skill staging is snapshot/digest-keyed and refuses untrusted isolation; mutating send/delete/bash tools declare `NON_RETRYABLE`; embed cancel marks TurnRun recovery. HostCapabilityProvider.invoke dispatches a registered host runner; IsolatedExecutor wraps approved backends with no subprocess fallback; dump_store/load_store persist the harness store; file/redis/dynamo lease adapters require an explicit client; skill signatures use HMAC compare_digest; CUCS harness evals live under `tests/conformance/cucs/`; CI adds conformance/two-worker/isolation/load lanes. - **Harness baseline audit (HP-01).** Process-local bus, caches, breakers, and locks inventoried in `.planning/phases/01-contracts-and-baseline/PROCESS-LOCAL-STATE.md` with characterization tests. No replacements. diff --git a/docs/HARNESS_DEPLOYMENT.md b/docs/HARNESS_DEPLOYMENT.md index 60053c7a..3ae84789 100644 --- a/docs/HARNESS_DEPLOYMENT.md +++ b/docs/HARNESS_DEPLOYMENT.md @@ -15,6 +15,18 @@ Same-session policy is **lease** (`SAME_SESSION_POLICY`). A second worker that does not hold the lease is refused (`SessionBusy`). Drain stops admissions and keeps outbox replay. +Lease backends: in-process (default), `FileLeaseBackend` (JSON + `os.replace`), +optional Redis `SET NX EX` and Dynamo `put_item` adapters. Missing Redis/Dynamo +clients raise; they do not silently fall back. + +Durable store dump: `jvagent.harness.persist.dump_store` / `load_store` writes +identities, snapshots, journals, ledger, outbox, traces, skills, and leases to +JSON. Callables (host runners, compensators) are not restored. + +Retention: `HarnessRuntime.prune_retention()` clips outbox and traces to +`MAX_EVENTS_PER_SESSION` / `max_trace_spans`. Journal and event reads paginate +(`journal_entries`, `replay_from(..., limit=)`). + ## Matrix Cells are `guaranteed` / `degraded` / `unsupported`. Never blank. diff --git a/docs/skill-isolation.md b/docs/skill-isolation.md index 9ae14b8c..981d8cfd 100644 --- a/docs/skill-isolation.md +++ b/docs/skill-isolation.md @@ -14,6 +14,16 @@ backends is configured on `HarnessRuntime(isolation_backend=...)`: - `nsjail` Trusted SOP skills (no script) activate from digest under the admitted snapshot. +With `skill_signing_key` set, `register_manifest` / `publish_manifest` require an +HMAC-SHA256 signature (`hmac.compare_digest`). `revoke_manifest` drops the digest +from the in-process registry so the next activate refuses. + +## Isolation executor + +`jvagent.harness.isolation.IsolatedExecutor` prefixes the command with `runsc` / +`firecracker` / `nsjail` **only when that binary is on PATH**. Absence is a +`SkillIsolationRefused`, never a subprocess fallback. `CodeExecutionAction.executor()` +uses `executor_for_backend(get_runtime().isolation_backend, SubprocessExecutor())`. ## Staging diff --git a/jvagent/action/artifact_handler_interact_action/artifact_handler_interact_action.py b/jvagent/action/artifact_handler_interact_action/artifact_handler_interact_action.py index 5d5f94f8..1b8f15a4 100644 --- a/jvagent/action/artifact_handler_interact_action/artifact_handler_interact_action.py +++ b/jvagent/action/artifact_handler_interact_action/artifact_handler_interact_action.py @@ -54,6 +54,7 @@ UploadItem, normalize_upload_entry, ) +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool if False: @@ -1293,7 +1294,10 @@ def _ingest_tool_args( args["question"] = question return args - @tool(name="artifact_handler__ingest_document") + @tool( + name="artifact_handler__ingest_document", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_ingest_document( self, visitor: Any = None, @@ -1319,7 +1323,10 @@ async def _t_list_my_documents(self, visitor: Any = None, **kwargs: Any) -> str: """List the documents the user has saved, with save age and expiry.""" return await self._dispatch_tool("list_my_documents", visitor=visitor) - @tool(name="artifact_handler__delete_document") + @tool( + name="artifact_handler__delete_document", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_delete_document( self, doc_name: str, visitor: Any = None, **kwargs: Any ) -> str: diff --git a/jvagent/action/code_execution/code_execution_action.py b/jvagent/action/code_execution/code_execution_action.py index 84d6e828..42279e52 100644 --- a/jvagent/action/code_execution/code_execution_action.py +++ b/jvagent/action/code_execution/code_execution_action.py @@ -82,9 +82,15 @@ class CodeExecutionAction(Action): _executor: Optional[Executor] = None def executor(self) -> Executor: - """The execution backend. Override to swap in a container/jail backend.""" + """The execution backend. Isolation backend when runtime requires it.""" if self._executor is None: - self._executor = SubprocessExecutor() + inner: Executor = SubprocessExecutor() + from jvagent.harness.isolation import executor_for_backend + from jvagent.harness.runtime import get_runtime + + backend = get_runtime().isolation_backend + # Refuse — never fall back to subprocess — when a backend is named. + self._executor = executor_for_backend(backend, inner) return self._executor # -- per-user sandbox resolution -------------------------------------- diff --git a/jvagent/action/file_interface/file_interface_action.py b/jvagent/action/file_interface/file_interface_action.py index 580a02d2..19461fb3 100644 --- a/jvagent/action/file_interface/file_interface_action.py +++ b/jvagent/action/file_interface/file_interface_action.py @@ -145,7 +145,10 @@ async def _t_list_directory( except Exception as e: return json.dumps({"ok": False, "error": f"{type(e).__name__}: {e}"}) - @tool(name="file_interface__create_directory") + @tool( + name="file_interface__create_directory", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_create_directory( self, path: Annotated[str, "Relative directory path."], diff --git a/jvagent/action/google/google_calendar_action/google_calendar_action.py b/jvagent/action/google/google_calendar_action/google_calendar_action.py index 1fb54537..2c9d89c0 100644 --- a/jvagent/action/google/google_calendar_action/google_calendar_action.py +++ b/jvagent/action/google/google_calendar_action/google_calendar_action.py @@ -2,6 +2,7 @@ import logging from typing import Annotated, Any, ClassVar, Dict, List, Optional +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool from ..google_action import GoogleAction @@ -85,7 +86,10 @@ async def _t_list_events( ) return json.dumps(results, indent=2) - @tool(name="calendar__create_event") + @tool( + name="calendar__create_event", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_create_event( self, summary: Annotated[str, "Event title/summary"], @@ -108,7 +112,10 @@ async def _t_create_event( ) return json.dumps(result, indent=2) - @tool(name="calendar__delete_event") + @tool( + name="calendar__delete_event", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_delete_event( self, calendar_id: Annotated[str, "Calendar identifier (default: 'primary')"], diff --git a/jvagent/action/google/google_drive_action/google_drive_action.py b/jvagent/action/google/google_drive_action/google_drive_action.py index eb5344ab..bfccbf88 100644 --- a/jvagent/action/google/google_drive_action/google_drive_action.py +++ b/jvagent/action/google/google_drive_action/google_drive_action.py @@ -498,7 +498,10 @@ async def _t_get_media( indent=2, ) - @tool(name="google_drive__share_file") + @tool( + name="google_drive__share_file", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_share_file( self, file_id: Annotated[str, "The ID of the file to share."], diff --git a/jvagent/action/google/google_gmail_action/google_gmail_action.py b/jvagent/action/google/google_gmail_action/google_gmail_action.py index 43652ebe..2d9a3c1e 100644 --- a/jvagent/action/google/google_gmail_action/google_gmail_action.py +++ b/jvagent/action/google/google_gmail_action/google_gmail_action.py @@ -129,7 +129,10 @@ async def _t_get_message( await self.get_message(message_id, fmt=fmt or "full"), indent=2 ) - @tool(name="gmail__mark_read") + @tool( + name="gmail__mark_read", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_mark_read( self, message_id: Annotated[str, "ID of the message to mark as read."], diff --git a/jvagent/action/google/google_sheets_action/google_sheets_action.py b/jvagent/action/google/google_sheets_action/google_sheets_action.py index bdf4ce50..b2597e93 100644 --- a/jvagent/action/google/google_sheets_action/google_sheets_action.py +++ b/jvagent/action/google/google_sheets_action/google_sheets_action.py @@ -11,6 +11,7 @@ from googleapiclient.discovery import build from jvspatial.core.annotations import attribute +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool from ..google_action import GoogleAction @@ -743,7 +744,10 @@ async def _t_read_spreadsheet( ) return json.dumps(result, indent=2) - @tool(name="google_sheets__update_spreadsheet") + @tool( + name="google_sheets__update_spreadsheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_update_spreadsheet( self, spreadsheet_url_or_id: Annotated[ @@ -776,7 +780,10 @@ async def _t_update_spreadsheet( ) return json.dumps(result, indent=2) - @tool(name="google_sheets__append_spreadsheet") + @tool( + name="google_sheets__append_spreadsheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_append_spreadsheet( self, spreadsheet_url_or_id: Annotated[ @@ -810,7 +817,10 @@ async def _t_append_spreadsheet( ) return json.dumps(result, indent=2) - @tool(name="google_sheets__create_spreadsheet") + @tool( + name="google_sheets__create_spreadsheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_create_spreadsheet( self, title: Annotated[str, "Title for the new spreadsheet"], @@ -819,7 +829,10 @@ async def _t_create_spreadsheet( result = await self.create_spreadsheet(title=title) return json.dumps(result, indent=2) - @tool(name="google_sheets__delete_spreadsheet") + @tool( + name="google_sheets__delete_spreadsheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_delete_spreadsheet( self, spreadsheet_url_or_id: Annotated[Optional[str], "Spreadsheet URL or ID"] = None, @@ -830,7 +843,10 @@ async def _t_delete_spreadsheet( ) return json.dumps({"deleted": result}, indent=2) - @tool(name="google_sheets__create_worksheet") + @tool( + name="google_sheets__create_worksheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_create_worksheet( self, title: Annotated[str, "Title for the new worksheet"], @@ -858,7 +874,10 @@ async def _t_create_worksheet( ) return json.dumps(result, indent=2) - @tool(name="google_sheets__update_worksheet") + @tool( + name="google_sheets__update_worksheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_update_worksheet( self, worksheet_title: Annotated[str, "Title of the worksheet to update"], @@ -887,7 +906,10 @@ async def _t_update_worksheet( ) return json.dumps(result, indent=2) - @tool(name="google_sheets__delete_worksheet") + @tool( + name="google_sheets__delete_worksheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_delete_worksheet( self, worksheet_title: Annotated[str, "Title of the worksheet to delete"], @@ -903,7 +925,10 @@ async def _t_delete_worksheet( ) return json.dumps(result, indent=2) - @tool(name="google_sheets__merge_cells") + @tool( + name="google_sheets__merge_cells", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_merge_cells( self, spreadsheet_url_or_id: Annotated[ @@ -932,7 +957,10 @@ async def _t_merge_cells( ) return json.dumps(result, indent=2) - @tool(name="google_sheets__unmerge_cells") + @tool( + name="google_sheets__unmerge_cells", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_unmerge_cells( self, spreadsheet_url_or_id: Annotated[ @@ -954,7 +982,10 @@ async def _t_unmerge_cells( ) return json.dumps(result, indent=2) - @tool(name="google_sheets__format_cells") + @tool( + name="google_sheets__format_cells", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_format_cells( self, spreadsheet_url_or_id: Annotated[ @@ -1010,7 +1041,10 @@ async def _t_last_filled_row( ) return json.dumps(result, indent=2) - @tool(name="google_sheets__batch_clear") + @tool( + name="google_sheets__batch_clear", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_batch_clear( self, spreadsheet_url_or_id: Annotated[ @@ -1032,7 +1066,10 @@ async def _t_batch_clear( ) return json.dumps(result, indent=2) - @tool(name="google_sheets__share_spreadsheet") + @tool( + name="google_sheets__share_spreadsheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_share_spreadsheet( self, spreadsheet_url_or_id: Annotated[ diff --git a/jvagent/action/microsoft/microsoft_excel_action/microsoft_excel_action.py b/jvagent/action/microsoft/microsoft_excel_action/microsoft_excel_action.py index 611f4fa9..8135f3c0 100644 --- a/jvagent/action/microsoft/microsoft_excel_action/microsoft_excel_action.py +++ b/jvagent/action/microsoft/microsoft_excel_action/microsoft_excel_action.py @@ -14,6 +14,7 @@ qualify_sheet_title, resolve_spreadsheet_id, ) +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool from ..microsoft_action import MicrosoftAction @@ -390,7 +391,10 @@ async def _t_read_spreadsheet( ) return json.dumps(result, indent=2) - @tool(name="excel__update_spreadsheet") + @tool( + name="excel__update_spreadsheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_update_spreadsheet( self, spreadsheet_url_or_id: Annotated[Optional[str], "Spreadsheet URL or ID"] = None, @@ -418,7 +422,10 @@ async def _t_update_spreadsheet( ) return json.dumps(result, indent=2) - @tool(name="excel__append_spreadsheet") + @tool( + name="excel__append_spreadsheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_append_spreadsheet( self, spreadsheet_url_or_id: Annotated[Optional[str], "Spreadsheet URL or ID"] = None, @@ -449,7 +456,10 @@ async def _t_append_spreadsheet( ) return json.dumps(result, indent=2) - @tool(name="excel__create_spreadsheet") + @tool( + name="excel__create_spreadsheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_create_spreadsheet( self, title: Annotated[str, "Title for the new spreadsheet"], @@ -458,7 +468,10 @@ async def _t_create_spreadsheet( result = await self.create_spreadsheet(title=title) return json.dumps(result, indent=2) - @tool(name="excel__delete_spreadsheet") + @tool( + name="excel__delete_spreadsheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_delete_spreadsheet( self, spreadsheet_url_or_id: Annotated[ @@ -471,7 +484,10 @@ async def _t_delete_spreadsheet( ) return json.dumps({"deleted": result}, indent=2) - @tool(name="excel__create_worksheet") + @tool( + name="excel__create_worksheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_create_worksheet( self, title: Annotated[str, "Title for the new worksheet"], @@ -494,7 +510,10 @@ async def _t_create_worksheet( ) return json.dumps(result, indent=2) - @tool(name="excel__update_worksheet") + @tool( + name="excel__update_worksheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_update_worksheet( self, worksheet_title: Annotated[str, "Current title of the worksheet to update"], @@ -522,7 +541,10 @@ async def _t_update_worksheet( ) return json.dumps(result, indent=2) - @tool(name="excel__delete_worksheet") + @tool( + name="excel__delete_worksheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_delete_worksheet( self, worksheet_title: Annotated[str, "Title of the worksheet to delete"], @@ -535,7 +557,10 @@ async def _t_delete_worksheet( ) return json.dumps(result, indent=2) - @tool(name="excel__batch_clear") + @tool( + name="excel__batch_clear", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_batch_clear( self, spreadsheet_url_or_id: Annotated[Optional[str], "Spreadsheet URL or ID"] = None, @@ -554,7 +579,10 @@ async def _t_batch_clear( ) return json.dumps(result, indent=2) - @tool(name="excel__share_spreadsheet") + @tool( + name="excel__share_spreadsheet", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_share_spreadsheet( self, spreadsheet_url_or_id: Annotated[Optional[str], "Spreadsheet URL or ID"] = None, diff --git a/jvagent/action/microsoft/microsoft_onedrive_action/microsoft_onedrive_action.py b/jvagent/action/microsoft/microsoft_onedrive_action/microsoft_onedrive_action.py index d0ffafed..aacb4bb0 100644 --- a/jvagent/action/microsoft/microsoft_onedrive_action/microsoft_onedrive_action.py +++ b/jvagent/action/microsoft/microsoft_onedrive_action/microsoft_onedrive_action.py @@ -6,6 +6,7 @@ import httpx from jvspatial.env import env +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool from ..microsoft_action import MicrosoftAction @@ -230,7 +231,10 @@ async def _t_list_files( ) return json.dumps(results, indent=2) - @tool(name="onedrive__upload_file") + @tool( + name="onedrive__upload_file", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_upload_file( self, name: Annotated[str, "Name for the uploaded file."], @@ -254,7 +258,10 @@ async def _t_upload_file( ) return json.dumps(result, indent=2) - @tool(name="onedrive__share_file") + @tool( + name="onedrive__share_file", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_share_file( self, file_id: Annotated[str, "The ID of the file to share."], @@ -284,7 +291,10 @@ async def _t_share_file( ) return json.dumps(result, indent=2) - @tool(name="onedrive__delete_file") + @tool( + name="onedrive__delete_file", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_delete_file( self, file_id: Annotated[str, "The ID of the file to delete."], diff --git a/jvagent/action/microsoft/microsoft_outlook_calendar_action/microsoft_outlook_calendar_action.py b/jvagent/action/microsoft/microsoft_outlook_calendar_action/microsoft_outlook_calendar_action.py index e8610609..52720b20 100644 --- a/jvagent/action/microsoft/microsoft_outlook_calendar_action/microsoft_outlook_calendar_action.py +++ b/jvagent/action/microsoft/microsoft_outlook_calendar_action/microsoft_outlook_calendar_action.py @@ -2,6 +2,7 @@ from typing import Annotated, Any, ClassVar, Dict, List, Optional from urllib.parse import quote +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool from ..microsoft_action import MicrosoftAction @@ -122,7 +123,10 @@ async def _t_list_events( ) return json.dumps(results, indent=2) - @tool(name="outlook_calendar__create_event") + @tool( + name="outlook_calendar__create_event", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_create_event( self, summary: Annotated[str, "Event title/subject"], @@ -148,7 +152,10 @@ async def _t_create_event( ) return json.dumps(result, indent=2) - @tool(name="outlook_calendar__delete_event") + @tool( + name="outlook_calendar__delete_event", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_delete_event( self, calendar_id: Annotated[str, "Calendar identifier (default: 'primary')"], diff --git a/jvagent/action/microsoft/microsoft_outlook_mail_action/microsoft_outlook_mail_action.py b/jvagent/action/microsoft/microsoft_outlook_mail_action/microsoft_outlook_mail_action.py index a934e3e0..48f288ab 100644 --- a/jvagent/action/microsoft/microsoft_outlook_mail_action/microsoft_outlook_mail_action.py +++ b/jvagent/action/microsoft/microsoft_outlook_mail_action/microsoft_outlook_mail_action.py @@ -233,7 +233,10 @@ async def _t_get_message( user_id = user_id if user_id is not None else "me" return json.dumps(await self.get_message(message_id, user_id=user_id), indent=2) - @tool(name="outlook__mark_read") + @tool( + name="outlook__mark_read", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_mark_read( self, message_id: Annotated[str, "The ID of the message to mark as read."], diff --git a/jvagent/action/orchestrator/orchestrator_interact_action.py b/jvagent/action/orchestrator/orchestrator_interact_action.py index e4efe413..c60f285d 100644 --- a/jvagent/action/orchestrator/orchestrator_interact_action.py +++ b/jvagent/action/orchestrator/orchestrator_interact_action.py @@ -999,7 +999,10 @@ async def execute(self, visitor: "InteractWalker") -> None: rt.fail_turn(cache["correlation_id"], reason="execute_error") raise finally: + if getattr(visitor, "background_actions", None): + rt.mark_background(cache["correlation_id"]) rt.persist_to_interaction(interaction, cache["correlation_id"]) + rt.prune_retention() async def _execute_turn(self, visitor: "InteractWalker") -> None: # Curate the remaining walk path: routable IAs (exposed as tools) must diff --git a/jvagent/action/orchestrator/tools.py b/jvagent/action/orchestrator/tools.py index 063715ee..6fb770d8 100644 --- a/jvagent/action/orchestrator/tools.py +++ b/jvagent/action/orchestrator/tools.py @@ -107,6 +107,8 @@ async def _run(args: Dict[str, Any], _tool: Any = tool) -> str: runtime = None correlation_id = "" interaction = None + snap = None + turn: Dict[str, Any] = {} try: from jvagent.action.orchestrator.turn_cache import get_turn_cache from jvagent.harness.runtime import get_runtime @@ -135,6 +137,37 @@ async def _run(args: Dict[str, Any], _tool: Any = tool) -> str: logger.debug("wrap_action_tool: ledger skip: %s", exc) record = None runtime = None + if ( + snap is not None + and runtime is not None + and name in (getattr(snap, "host_tool_names", ()) or ()) + ): + from jvagent.harness.provider import provider_for + + try: + provider = turn.get("provider") or provider_for("native", runtime) + invoked = await provider.invoke( + snap.snapshot_id, + record.invocation_id if record is not None else "", + name, + call_args, + ) + content = json.dumps(dict(invoked.payload), default=str) + ok = bool(invoked.ok) + except Exception as exc: + logger.warning("wrap_action_tool: host tool %r raised: %s", name, exc) + content = f"(tool error: {exc})" + ok = False + if record is not None: + runtime.finish_invocation( + correlation_id=correlation_id, + record=record, + result=content, + ok=ok, + ) + if interaction is not None: + runtime.persist_to_interaction(interaction, correlation_id) + return content try: result = await _tool.call(**call_kwargs) except Exception as exc: diff --git a/jvagent/action/pageindex/pageindex_action/pageindex_action.py b/jvagent/action/pageindex/pageindex_action/pageindex_action.py index 32a88c9a..f7c24dcc 100644 --- a/jvagent/action/pageindex/pageindex_action/pageindex_action.py +++ b/jvagent/action/pageindex/pageindex_action/pageindex_action.py @@ -663,7 +663,10 @@ async def _t_search( # Agent prompt sees start_page/end_page; API/search rows keep index keys. return json.dumps(prompt_page_aliases(results), indent=2) - @tool(name="pageindex__assimilate") + @tool( + name="pageindex__assimilate", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_assimilate( self, doc: Annotated[ diff --git a/jvagent/action/skill_hub/skill_hub_action.py b/jvagent/action/skill_hub/skill_hub_action.py index 6cbbad47..3f74180c 100644 --- a/jvagent/action/skill_hub/skill_hub_action.py +++ b/jvagent/action/skill_hub/skill_hub_action.py @@ -32,6 +32,7 @@ run_skills_list, ) from jvagent.core.app_context import get_app_root +from jvagent.harness.contracts import IdempotencyClass from jvagent.tooling.tool_decorator import tool logger = logging.getLogger(__name__) @@ -419,7 +420,10 @@ async def _t_search_registry( result = await self.search_registry(arguments, visitor=visitor) return result if isinstance(result, str) else json.dumps(result) - @tool(name="skill_hub__install_skill") + @tool( + name="skill_hub__install_skill", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_install_skill( self, source: Annotated[ @@ -460,7 +464,10 @@ async def _t_list_installed(self) -> str: result = await self.list_installed(arguments, visitor=visitor) return result if isinstance(result, str) else json.dumps(result) - @tool(name="skill_hub__remove_skill") + @tool( + name="skill_hub__remove_skill", + idempotency_class=IdempotencyClass.NON_RETRYABLE, + ) async def _t_remove_skill( self, skill_name: Annotated[str, "Name of the installed skill to remove"], diff --git a/jvagent/harness/isolation.py b/jvagent/harness/isolation.py new file mode 100644 index 00000000..5402f9c2 --- /dev/null +++ b/jvagent/harness/isolation.py @@ -0,0 +1,98 @@ +"""Approved skill isolation backends (HP-09). + +Subprocess is development containment, not a sandbox. Untrusted script skills +require one of ``gvisor`` / ``firecracker`` / ``nsjail`` with the matching +binary on PATH. This module wraps a command for that binary; it does not +implement a kernel jail itself. +""" + +from __future__ import annotations + +import shlex +import shutil +from dataclasses import replace +from typing import Mapping + +from jvagent.action.code_execution.executor import ExecRequest, ExecResult, Executor + +BACKEND_BINS: Mapping[str, str] = { + "gvisor": "runsc", + "firecracker": "firecracker", + "nsjail": "nsjail", +} +APPROVED = frozenset(BACKEND_BINS) + + +def isolation_binary(backend: str) -> str: + return BACKEND_BINS.get(backend, backend) + + +def isolation_available(backend: str) -> bool: + if backend not in APPROVED: + return False + return shutil.which(isolation_binary(backend)) is not None + + +def wrap_isolated_command(backend: str, req: ExecRequest) -> ExecRequest: + """Prefix ``req.command`` with the approved backend binary. + + The wrapper is a launch prefix only. Network/fs isolation is whatever that + binary enforces when present; absence is a refuse, not a subprocess fallback. + """ + from jvagent.harness.runtime import SkillIsolationRefused + + if backend not in APPROVED: + raise SkillIsolationRefused( + f"unapproved isolation backend {backend!r}; subprocess is not a sandbox" + ) + if not isolation_available(backend): + raise SkillIsolationRefused( + f"{backend} binary {isolation_binary(backend)!r} not on PATH" + ) + bin_name = isolation_binary(backend) + cwd = shlex.quote(req.cwd) + inner = shlex.quote(req.command) + if backend == "nsjail": + cmd = f"{bin_name} -Mo --cwd {cwd} -- /bin/sh -c {inner}" + elif backend == "gvisor": + cmd = f"{bin_name} exec --cwd {cwd} -- /bin/sh -c {inner}" + else: + cmd = f"{bin_name} -- {inner}" + return replace(req, command=cmd) + + +class IsolatedExecutor: + """Executor that refuses unless an approved isolation binary is present.""" + + def __init__(self, backend: str, inner: Executor) -> None: + from jvagent.harness.runtime import SkillIsolationRefused + + if backend not in APPROVED: + raise SkillIsolationRefused( + f"unapproved isolation backend {backend!r}; subprocess is not a sandbox" + ) + if not isolation_available(backend): + raise SkillIsolationRefused( + f"{backend} binary {isolation_binary(backend)!r} not on PATH" + ) + self.backend = backend + self.inner = inner + + async def run(self, req: ExecRequest) -> ExecResult: + return await self.inner.run(wrap_isolated_command(self.backend, req)) + + +def executor_for_backend(backend: str, inner: Executor) -> Executor: + if not backend: + return inner + return IsolatedExecutor(backend, inner) + + +__all__ = [ + "BACKEND_BINS", + "IsolatedExecutor", + "executor_for_backend", + "isolation_available", + "isolation_binary", + "wrap_isolated_command", +] diff --git a/jvagent/harness/leases.py b/jvagent/harness/leases.py new file mode 100644 index 00000000..4ca07aeb --- /dev/null +++ b/jvagent/harness/leases.py @@ -0,0 +1,340 @@ +"""Session lease backends (HP-07). + +In-process dict is the default. File-backed leases let two processes contend +without Redis. Optional Redis/Dynamo adapters use SET NX / PutItem when those +clients are installed; missing clients raise, they do not silently fall back. +""" + +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Optional, Protocol + +DEFAULT_LEASE_TTL_S = 30.0 + + +class LeaseBackend(Protocol): + def acquire( + self, session_id: str, worker_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S + ) -> Dict[str, Any]: ... + + def bind(self, session_id: str, worker_id: str, correlation_id: str) -> None: ... + + def renew( + self, session_id: str, worker_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S + ) -> None: ... + + def release(self, session_id: str, worker_id: str) -> None: ... + + def get(self, session_id: str) -> Optional[Dict[str, Any]]: ... + + +@dataclass +class InProcessLeaseBackend: + """Wraps ``HarnessStore.leases``.""" + + leases: Dict[str, Dict[str, Any]] + + def acquire( + self, session_id: str, worker_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S + ) -> Dict[str, Any]: + now = time.monotonic() + held = self.leases.get(session_id) + if held and held["worker_id"] != worker_id and held["expires_at"] > now: + from jvagent.harness.runtime import SessionBusy + + raise SessionBusy(f"session {session_id} leased by {held['worker_id']}") + rec = { + "worker_id": worker_id, + "expires_at": now + ttl_s, + "correlation_id": (held or {}).get("correlation_id", ""), + } + if held and held["expires_at"] <= now: + rec["expired_correlation_id"] = held.get("correlation_id") or "" + self.leases[session_id] = rec + return rec + + def bind(self, session_id: str, worker_id: str, correlation_id: str) -> None: + held = self.leases.get(session_id) + if held and held["worker_id"] == worker_id: + held["correlation_id"] = correlation_id + + def renew( + self, session_id: str, worker_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S + ) -> None: + held = self.leases.get(session_id) + if not held or held["worker_id"] != worker_id: + from jvagent.harness.runtime import SessionBusy + + raise SessionBusy(f"session {session_id} not held by {worker_id}") + held["expires_at"] = time.monotonic() + ttl_s + + def release(self, session_id: str, worker_id: str) -> None: + held = self.leases.get(session_id) + if held and held["worker_id"] == worker_id: + self.leases.pop(session_id, None) + + def get(self, session_id: str) -> Optional[Dict[str, Any]]: + return self.leases.get(session_id) + + +class FileLeaseBackend: + """JSON file + exclusive create. Two OS processes can contend.""" + + def __init__(self, path: Path) -> None: + self.path = path + self.path.parent.mkdir(parents=True, exist_ok=True) + if not self.path.exists(): + self.path.write_text("{}", encoding="utf-8") + + def _load(self) -> Dict[str, Any]: + try: + return json.loads(self.path.read_text(encoding="utf-8") or "{}") + except json.JSONDecodeError: + return {} + + def _save(self, data: Dict[str, Any]) -> None: + tmp = self.path.with_suffix(self.path.suffix + ".tmp") + tmp.write_text(json.dumps(data), encoding="utf-8") + os.replace(tmp, self.path) + + def acquire( + self, session_id: str, worker_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S + ) -> Dict[str, Any]: + now = time.time() + data = self._load() + held = data.get(session_id) + if held and held["worker_id"] != worker_id and held["expires_at"] > now: + from jvagent.harness.runtime import SessionBusy + + raise SessionBusy(f"session {session_id} leased by {held['worker_id']}") + rec = { + "worker_id": worker_id, + "expires_at": now + ttl_s, + "correlation_id": (held or {}).get("correlation_id", ""), + } + if held and held["expires_at"] <= now: + rec["expired_correlation_id"] = held.get("correlation_id") or "" + data[session_id] = rec + self._save(data) + return rec + + def bind(self, session_id: str, worker_id: str, correlation_id: str) -> None: + data = self._load() + held = data.get(session_id) + if held and held["worker_id"] == worker_id: + held["correlation_id"] = correlation_id + self._save(data) + + def renew( + self, session_id: str, worker_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S + ) -> None: + data = self._load() + held = data.get(session_id) + if not held or held["worker_id"] != worker_id: + from jvagent.harness.runtime import SessionBusy + + raise SessionBusy(f"session {session_id} not held by {worker_id}") + held["expires_at"] = time.time() + ttl_s + self._save(data) + + def release(self, session_id: str, worker_id: str) -> None: + data = self._load() + held = data.get(session_id) + if held and held["worker_id"] == worker_id: + data.pop(session_id, None) + self._save(data) + + def get(self, session_id: str) -> Optional[Dict[str, Any]]: + return self._load().get(session_id) + + +class RedisLeaseBackend: + """Optional. Requires ``redis`` package. SET key NX EX.""" + + def __init__(self, client: Any, *, prefix: str = "jvagent:lease:") -> None: + self.client = client + self.prefix = prefix + + def _key(self, session_id: str) -> str: + return f"{self.prefix}{session_id}" + + def acquire( + self, session_id: str, worker_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S + ) -> Dict[str, Any]: + key = self._key(session_id) + ok = self.client.set(key, worker_id, nx=True, ex=int(max(ttl_s, 1))) + if not ok: + holder = self.client.get(key) + holder_s = holder.decode() if isinstance(holder, bytes) else holder + if holder_s != worker_id: + from jvagent.harness.runtime import SessionBusy + + raise SessionBusy(f"session {session_id} leased by {holder_s}") + return { + "worker_id": worker_id, + "expires_at": time.time() + ttl_s, + "correlation_id": "", + } + + def bind(self, session_id: str, worker_id: str, correlation_id: str) -> None: + self.client.set(self._key(session_id) + ":corr", correlation_id, xx=True) + + def renew( + self, session_id: str, worker_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S + ) -> None: + key = self._key(session_id) + holder = self.client.get(key) + holder_s = holder.decode() if isinstance(holder, bytes) else holder + if holder_s != worker_id: + from jvagent.harness.runtime import SessionBusy + + raise SessionBusy(f"session {session_id} not held by {worker_id}") + self.client.expire(key, int(max(ttl_s, 1))) + + def release(self, session_id: str, worker_id: str) -> None: + key = self._key(session_id) + holder = self.client.get(key) + holder_s = holder.decode() if isinstance(holder, bytes) else holder + if holder_s == worker_id: + self.client.delete(key) + + def get(self, session_id: str) -> Optional[Dict[str, Any]]: + holder = self.client.get(self._key(session_id)) + if not holder: + return None + holder_s = holder.decode() if isinstance(holder, bytes) else holder + return {"worker_id": holder_s, "correlation_id": "", "expires_at": 0} + + +class DynamoLeaseBackend: + """Optional. Requires a DynamoDB-like client with put_item/get_item/delete_item. + + Missing client is a raise, not a silent in-process fallback. + """ + + def __init__(self, client: Any, *, table: str = "jvagent-leases") -> None: + self.client = client + self.table = table + + def acquire( + self, session_id: str, worker_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S + ) -> Dict[str, Any]: + expires = int(time.time() + ttl_s) + existing = self.client.get_item( + TableName=self.table, Key={"session_id": {"S": session_id}} + ) + item = (existing or {}).get("Item") or {} + holder = ((item.get("worker_id") or {}).get("S")) or "" + exp = int(((item.get("expires_at") or {}).get("N")) or 0) + now = int(time.time()) + if holder and holder != worker_id and exp > now: + from jvagent.harness.runtime import SessionBusy + + raise SessionBusy(f"session {session_id} leased by {holder}") + rec = { + "worker_id": worker_id, + "expires_at": float(expires), + "correlation_id": ((item.get("correlation_id") or {}).get("S")) or "", + } + if holder and exp <= now: + rec["expired_correlation_id"] = rec["correlation_id"] + self.client.put_item( + TableName=self.table, + Item={ + "session_id": {"S": session_id}, + "worker_id": {"S": worker_id}, + "expires_at": {"N": str(expires)}, + "correlation_id": {"S": rec["correlation_id"]}, + }, + ) + return rec + + def bind(self, session_id: str, worker_id: str, correlation_id: str) -> None: + held = self.get(session_id) + if held and held["worker_id"] == worker_id: + held["correlation_id"] = correlation_id + self.client.put_item( + TableName=self.table, + Item={ + "session_id": {"S": session_id}, + "worker_id": {"S": worker_id}, + "expires_at": {"N": str(int(held.get("expires_at") or 0))}, + "correlation_id": {"S": correlation_id}, + }, + ) + + def renew( + self, session_id: str, worker_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S + ) -> None: + held = self.get(session_id) + if not held or held["worker_id"] != worker_id: + from jvagent.harness.runtime import SessionBusy + + raise SessionBusy(f"session {session_id} not held by {worker_id}") + held["expires_at"] = time.time() + ttl_s + self.bind(session_id, worker_id, held.get("correlation_id") or "") + + def release(self, session_id: str, worker_id: str) -> None: + held = self.get(session_id) + if held and held["worker_id"] == worker_id: + self.client.delete_item( + TableName=self.table, Key={"session_id": {"S": session_id}} + ) + + def get(self, session_id: str) -> Optional[Dict[str, Any]]: + existing = self.client.get_item( + TableName=self.table, Key={"session_id": {"S": session_id}} + ) + item = (existing or {}).get("Item") or {} + if not item: + return None + return { + "worker_id": ((item.get("worker_id") or {}).get("S")) or "", + "correlation_id": ((item.get("correlation_id") or {}).get("S")) or "", + "expires_at": float(((item.get("expires_at") or {}).get("N")) or 0), + } + + +def lease_backend_for( + kind: str, + *, + leases: Optional[Dict[str, Dict[str, Any]]] = None, + path: Optional[Path] = None, + redis_client: Any = None, + dynamo_client: Any = None, +) -> LeaseBackend: + if kind in ("", "memory", "inprocess"): + return InProcessLeaseBackend(leases if leases is not None else {}) + if kind == "file": + if path is None: + raise ValueError("file lease backend requires path") + return FileLeaseBackend(path) + if kind == "redis": + if redis_client is None: + raise ValueError( + "redis lease backend requires a client; no silent fallback" + ) + return RedisLeaseBackend(redis_client) + if kind in ("dynamo", "dynamodb"): + if dynamo_client is None: + raise ValueError( + "dynamo lease backend requires a client; no silent fallback" + ) + return DynamoLeaseBackend(dynamo_client) + raise ValueError(f"unknown lease backend {kind!r}") + + +__all__ = [ + "DEFAULT_LEASE_TTL_S", + "DynamoLeaseBackend", + "FileLeaseBackend", + "InProcessLeaseBackend", + "LeaseBackend", + "RedisLeaseBackend", + "lease_backend_for", +] diff --git a/jvagent/harness/persist.py b/jvagent/harness/persist.py new file mode 100644 index 00000000..bfa611b4 --- /dev/null +++ b/jvagent/harness/persist.py @@ -0,0 +1,235 @@ +"""Durable dump/load of a HarnessStore (HP-06 transport, HP-11 retention). + +JSON file on disk. Two workers share a path. Not Redis. Not a claim of +exactly-once channel send. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Tuple + +from jvagent.harness.contracts import ( + EventEnvelope, + IdempotencyClass, + InvocationRecord, + ToolSurfaceSnapshot, + TurnRunState, + native_caller_from_mapping, +) +from jvagent.harness.runtime import ( + HarnessStore, + SkillManifest, + StageRecord, + TurnRunJournal, +) + +STORE_FORMAT = 1 + + +def _tuple_key(parts: Tuple[str, ...]) -> str: + return "\x1f".join(parts) + + +def _split_pair(raw: str) -> Tuple[str, str]: + a, b = raw.split("\x1f", 1) + return a, b + + +def _split_triple(raw: str) -> Tuple[str, str, str]: + a, rest = raw.split("\x1f", 1) + b, c = rest.split("\x1f", 1) + return a, b, c + + +def dump_store(store: HarnessStore, path: Path) -> None: + """Write identities, snapshots, journals, ledger, outbox, traces, skills.""" + with store.lock: + snapshots = {} + for sid, snap in store.snapshots.items(): + blob = asdict(snap) + blob["caller"] = snap.caller.to_mapping() + snapshots[sid] = blob + current = {_tuple_key(k): v for k, v in store.current_snapshot.items()} + generation = {_tuple_key(k): v for k, v in store.generation.items()} + identities = {_tuple_key(k): v for k, v in store.identities.items()} + conversations = {_tuple_key(k): v for k, v in store.conversations.items()} + runs: Dict[str, Dict[str, Any]] = {} + for corr, journal in store.runs.items(): + runs[corr] = { + "correlation_id": journal.correlation_id, + "caller": journal.caller.to_mapping(), + "state": journal.state.value, + "snapshot_id": journal.snapshot_id, + "interaction_id": journal.interaction_id, + "seq": journal.seq, + "worker_id": journal.worker_id, + "entries": list(journal.entries), + "completed_invocation_ids": list(journal.completed_invocation_ids), + "observation_refs": list(journal.observation_refs), + "plan_phase": journal.plan_phase, + "reason": journal.reason, + } + invocations: Dict[str, Dict[str, Any]] = {} + for key, rec in store.invocations.items(): + rec_map = asdict(rec) + klass = rec.idempotency_class + rec_map["idempotency_class"] = klass.value if klass else None + invocations[key] = rec_map + outbox = { + sid: [asdict(e) for e in events] for sid, events in store.outbox.items() + } + manifests = {digest: asdict(m) for digest, m in store.skill_manifests.items()} + stages = {} + for p, rec in store.stages.items(): + stages[p] = { + "caller": rec.caller.to_mapping(), + "snapshot_id": rec.snapshot_id, + "digest": rec.digest, + "path": rec.path, + "active": rec.active, + } + payload = { + "format": STORE_FORMAT, + "dumped_at": datetime.now(timezone.utc).isoformat(), + "identities": identities, + "conversations": conversations, + "snapshots": snapshots, + "current_snapshot": current, + "generation": generation, + "runs": runs, + "runs_by_interaction": dict(store.runs_by_interaction), + "invocations": invocations, + "invocation_results": dict(store.invocation_results), + "outbox": outbox, + "traces": {k: list(v) for k, v in store.traces.items()}, + "skill_manifests": manifests, + "stages": stages, + "host_tools": {k: list(v) for k, v in store.host_tools.items()}, + "host_skills": {k: list(v) for k, v in store.host_skills.items()}, + "revoked_host_tools": { + k: sorted(v) for k, v in store.revoked_host_tools.items() + }, + "revoked_manifests": sorted(getattr(store, "revoked_manifests", set())), + "leases": dict(store.leases), + "draining": store.draining, + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, default=str), encoding="utf-8") + + +def load_store(path: Path, store: Optional[HarnessStore] = None) -> HarnessStore: + """Hydrate a store from :func:`dump_store` JSON. Callables are not restored.""" + target = store or HarnessStore() + raw: Mapping[str, Any] = json.loads(path.read_text(encoding="utf-8")) + with target.lock: + target.identities = { + _split_pair(k): v for k, v in (raw.get("identities") or {}).items() + } + target.conversations = { + _split_pair(k): v for k, v in (raw.get("conversations") or {}).items() + } + snaps: Dict[str, ToolSurfaceSnapshot] = {} + for sid, blob in (raw.get("snapshots") or {}).items(): + snaps[sid] = ToolSurfaceSnapshot( + snapshot_id=str(blob["snapshot_id"]), + caller=native_caller_from_mapping(blob["caller"]), + native_tool_names=tuple(blob.get("native_tool_names") or ()), + native_skill_keys=tuple(blob.get("native_skill_keys") or ()), + host_tool_names=tuple(blob.get("host_tool_names") or ()), + host_skill_keys=tuple(blob.get("host_skill_keys") or ()), + created_at=str(blob.get("created_at") or ""), + expires_at=str(blob.get("expires_at") or ""), + revoked=bool(blob.get("revoked")), + ) + target.snapshots = snaps + target.current_snapshot = { + _split_triple(k): v for k, v in (raw.get("current_snapshot") or {}).items() + } + target.generation = { + _split_triple(k): int(v) for k, v in (raw.get("generation") or {}).items() + } + runs: Dict[str, TurnRunJournal] = {} + for corr, blob in (raw.get("runs") or {}).items(): + runs[corr] = TurnRunJournal( + correlation_id=str(blob["correlation_id"]), + caller=native_caller_from_mapping(blob["caller"]), + state=TurnRunState(str(blob["state"])), + snapshot_id=str(blob.get("snapshot_id") or ""), + interaction_id=str(blob.get("interaction_id") or ""), + seq=int(blob.get("seq") or 0), + worker_id=str(blob.get("worker_id") or ""), + entries=list(blob.get("entries") or []), + completed_invocation_ids=list( + blob.get("completed_invocation_ids") or [] + ), + observation_refs=list(blob.get("observation_refs") or []), + plan_phase=str(blob.get("plan_phase") or ""), + reason=str(blob.get("reason") or ""), + ) + target.runs = runs + target.runs_by_interaction = dict(raw.get("runs_by_interaction") or {}) + invocations: Dict[str, InvocationRecord] = {} + for key, rec_map in (raw.get("invocations") or {}).items(): + klass_raw = rec_map.get("idempotency_class") + invocations[key] = InvocationRecord( + invocation_id=str(rec_map["invocation_id"]), + snapshot_id=str(rec_map.get("snapshot_id") or ""), + tool_name=str(rec_map.get("tool_name") or ""), + input_digest=str(rec_map.get("input_digest") or ""), + idempotency_class=(IdempotencyClass(klass_raw) if klass_raw else None), + attempt=int(rec_map.get("attempt") or 1), + outcome=rec_map.get("outcome"), + ) + target.invocations = invocations + target.invocation_results = { + k: str(v) for k, v in (raw.get("invocation_results") or {}).items() + } + outbox: Dict[str, List[EventEnvelope]] = {} + for sid, events in (raw.get("outbox") or {}).items(): + outbox[sid] = [EventEnvelope(**e) for e in events] + target.outbox = outbox + target.traces = {k: list(v) for k, v in (raw.get("traces") or {}).items()} + manifests: Dict[str, SkillManifest] = {} + for digest, blob in (raw.get("skill_manifests") or {}).items(): + manifests[digest] = SkillManifest( + skill_key=str(blob["skill_key"]), + source=str(blob.get("source") or ""), + digest=str(blob["digest"]), + declared_tools=tuple(blob.get("declared_tools") or ()), + capabilities=tuple(blob.get("capabilities") or ()), + trust_tier=str(blob.get("trust_tier") or "trusted"), + spec=str(blob.get("spec") or "jv"), + signature=str(blob.get("signature") or ""), + body=str(blob.get("body") or ""), + ) + target.skill_manifests = manifests + stages: Dict[str, StageRecord] = {} + for p, blob in (raw.get("stages") or {}).items(): + stages[p] = StageRecord( + caller=native_caller_from_mapping(blob["caller"]), + snapshot_id=str(blob["snapshot_id"]), + digest=str(blob["digest"]), + path=str(blob["path"]), + active=bool(blob.get("active", True)), + ) + target.stages = stages + target.host_tools = { + k: list(v) for k, v in (raw.get("host_tools") or {}).items() + } + target.host_skills = { + k: list(v) for k, v in (raw.get("host_skills") or {}).items() + } + target.revoked_host_tools = { + k: set(v) for k, v in (raw.get("revoked_host_tools") or {}).items() + } + target.revoked_manifests = set(raw.get("revoked_manifests") or []) + target.leases = dict(raw.get("leases") or {}) + target.draining = bool(raw.get("draining")) + return target + + +__all__ = ["STORE_FORMAT", "dump_store", "load_store"] diff --git a/jvagent/harness/provider.py b/jvagent/harness/provider.py index 01e7efb8..b1558bb1 100644 --- a/jvagent/harness/provider.py +++ b/jvagent/harness/provider.py @@ -36,17 +36,31 @@ async def invoke( ) -> ToolResult: reject_model_authority_fields(payload) snap = self.runtime.require_usable(snapshot_id) + if tool_name in snap.host_tool_names: + runner = self.runtime.host_runner(snap.caller.session_id, tool_name) + if runner is None: + raise HarnessContractError( + f"no runner registered for host tool {tool_name!r}" + ) + result = runner(dict(payload)) + if hasattr(result, "__await__"): + result = await result # type: ignore[misc] + return ToolResult( + invocation_id=invocation_id, + ok=True, + payload=( + dict(result) if isinstance(result, Mapping) else {"result": result} + ), + ) if ( - tool_name not in snap.host_tool_names - and tool_name not in snap.native_tool_names + tool_name not in snap.native_tool_names + and tool_name not in snap.host_tool_names ): raise HarnessContractError( f"tool {tool_name!r} not on snapshot {snapshot_id}" ) - return ToolResult( - invocation_id=invocation_id, - ok=True, - payload={"echo": dict(payload), "tool": tool_name}, + raise HarnessContractError( + f"native tool {tool_name!r} is dispatched by wrap_action_tool, not HostCapabilityProvider" ) async def load_skill( diff --git a/jvagent/harness/release.py b/jvagent/harness/release.py index d112c81d..22f1eeb5 100644 --- a/jvagent/harness/release.py +++ b/jvagent/harness/release.py @@ -63,9 +63,12 @@ def release_record(*, digest: str, topology: str) -> dict: "matrix": DEPLOYMENT_MATRIX, "limitations": [ "JSON and SQLite are single-writer; do not claim active-active.", - "Outbox is store-backed in HarnessStore; durable transport swap is deferred.", + "Outbox dump/load is JSON on disk via dump_store; Redis/Dynamo streams are not implied.", + "Session leases: in-process default; file/redis/dynamo adapters require an explicit client or path (no silent fallback).", "Subprocess skill execution is development containment, not a sandbox.", + "Untrusted skills refuse unless gvisor/firecracker/nsjail is on PATH.", "Exactly-once third-party effects require an idempotency mechanism.", + "HostCapabilityProvider.invoke runs a registered host runner; native tools stay on wrap_action_tool.", ], "rollback": "revert to process-local caches/bus; disable drain and shared store.", } diff --git a/jvagent/harness/runtime.py b/jvagent/harness/runtime.py index b8ded2c0..74fd2649 100644 --- a/jvagent/harness/runtime.py +++ b/jvagent/harness/runtime.py @@ -7,7 +7,9 @@ from __future__ import annotations import hashlib +import hmac import json +import logging import threading import time import uuid @@ -37,8 +39,12 @@ DEFAULT_LEASE_TTL_S = 30.0 MAX_EVENTS_PER_SESSION = 10_000 MAX_OBSERVATION_CHARS = 8_000 +MAX_TRACE_SPANS = 10_000 APPROVED_ISOLATION_BACKENDS = frozenset({"gvisor", "firecracker", "nsjail"}) CHECKPOINT_KIND = "harness.turn_run" +TRACE_KIND = "harness.trace" + +_log = logging.getLogger("jvagent.harness") _runtime_guard = threading.Lock() _runtime: Optional["HarnessRuntime"] = None @@ -117,6 +123,7 @@ class HarnessStore: host_tools: Dict[str, List[str]] = field(default_factory=dict) host_skills: Dict[str, List[str]] = field(default_factory=dict) revoked_host_tools: Dict[str, set] = field(default_factory=dict) + revoked_manifests: set = field(default_factory=set) breaker_states: Dict[str, Any] = field(default_factory=dict) draining: bool = False lock: threading.Lock = field(default_factory=threading.Lock) @@ -131,11 +138,19 @@ def __init__( *, worker_id: str = "", isolation_backend: str = "", + skill_signing_key: str = "", + lease_backend: Any = None, + max_trace_spans: int = MAX_TRACE_SPANS, ) -> None: self.store = store or HarnessStore() self.worker_id = worker_id or f"worker-{uuid.uuid4().hex[:8]}" self.isolation_backend = isolation_backend + self.skill_signing_key = skill_signing_key + self.lease_backend = lease_backend + self.max_trace_spans = max_trace_spans self.contract_version = CONTRACT_VERSION + self._host_runners: Dict[Tuple[str, str], Any] = {} + self._compensators: Dict[str, Any] = {} # -- identity (HP-02) ------------------------------------------------- @@ -362,7 +377,10 @@ def peek_completed_result( def correlation_for_session(self, session_id: str) -> Optional[str]: if not session_id: return None - held = self.store.leases.get(session_id) + if self.lease_backend is not None: + held = self.lease_backend.get(session_id) + else: + held = self.store.leases.get(session_id) if held: corr = str(held.get("correlation_id") or "") if corr: @@ -496,6 +514,13 @@ def persist_to_interaction(self, interaction: Any, correlation_id: str) -> None: if not (isinstance(m, dict) and m.get("kind") == CHECKPOINT_KIND) ] metrics.append({"kind": CHECKPOINT_KIND, "payload": payload}) + spans = self.traces_for(correlation_id) + metrics = [ + m + for m in metrics + if not (isinstance(m, dict) and m.get("kind") == TRACE_KIND) + ] + metrics.append({"kind": TRACE_KIND, "spans": spans}) interaction.observability_metrics = metrics def checkpoint_from_interaction(self, interaction: Any) -> Optional[Dict[str, Any]]: @@ -581,6 +606,9 @@ def finish_invocation( if journal is not None: journal.completed_invocation_ids.append(record.invocation_id) journal.observation_refs.append(f"inv:{record.invocation_id}") + if not ok and record.idempotency_class is IdempotencyClass.COMPENSATABLE: + if record.tool_name in self._compensators: + self.compensate(record.invocation_id) if not ok and record.idempotency_class is IdempotencyClass.NON_RETRYABLE: self.mark_recovery(correlation_id, reason=f"failed:{record.tool_name}") return @@ -593,7 +621,24 @@ def finish_invocation( ) def compensate(self, invocation_id: str) -> str: - return f"compensated:{invocation_id}" + record = None + with self.store.lock: + for rec in self.store.invocations.values(): + if rec.invocation_id == invocation_id: + record = rec + break + if record is None: + raise HarnessContractError(f"unknown invocation {invocation_id}") + fn = self._compensators.get(record.tool_name) + if fn is None: + raise HarnessContractError( + f"no compensator registered for {record.tool_name}" + ) + result = fn(record) + return str(result) + + def register_compensator(self, tool_name: str, fn: Any) -> None: + self._compensators[tool_name] = fn # -- outbox (HP-06) --------------------------------------------------- @@ -642,11 +687,28 @@ def replay_from( out = [e for e in stream if e.sequence > after] return out[:limit] + def journal_entries( + self, correlation_id: str, *, after_seq: int = 0, limit: int = 100 + ) -> List[Dict[str, Any]]: + journal = self.get_run(correlation_id) + if journal is None: + return [] + rows = [e for e in journal.entries if int(e.get("seq") or 0) > after_seq] + return rows[: max(limit, 0)] + # -- leases (HP-07) --------------------------------------------------- def acquire_session_lease( self, session_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S ) -> None: + if self.lease_backend is not None: + rec = self.lease_backend.acquire(session_id, self.worker_id, ttl_s=ttl_s) + expired = rec.get("expired_correlation_id") or "" + if expired and expired in self.store.runs: + run = self.store.runs[expired] + if run.state not in TURN_RUN_TERMINAL: + self.mark_recovery(expired, reason="lease_expired") + return now = time.monotonic() with self.store.lock: held = self.store.leases.get(session_id) @@ -683,6 +745,9 @@ def acquire_session_lease( } def bind_lease(self, session_id: str, correlation_id: str) -> None: + if self.lease_backend is not None: + self.lease_backend.bind(session_id, self.worker_id, correlation_id) + return with self.store.lock: held = self.store.leases.get(session_id) if held and held["worker_id"] == self.worker_id: @@ -691,6 +756,9 @@ def bind_lease(self, session_id: str, correlation_id: str) -> None: def renew_session_lease( self, session_id: str, *, ttl_s: float = DEFAULT_LEASE_TTL_S ) -> None: + if self.lease_backend is not None: + self.lease_backend.renew(session_id, self.worker_id, ttl_s=ttl_s) + return now = time.monotonic() with self.store.lock: held = self.store.leases.get(session_id) @@ -699,6 +767,9 @@ def renew_session_lease( held["expires_at"] = now + ttl_s def release_session_lease(self, session_id: str) -> None: + if self.lease_backend is not None: + self.lease_backend.release(session_id, self.worker_id) + return with self.store.lock: held = self.store.leases.get(session_id) if held and held["worker_id"] == self.worker_id: @@ -750,16 +821,43 @@ def revoke_host_tool(self, session_id: str, name: str) -> None: with self.store.lock: self.store.revoked_host_tools.setdefault(session_id, set()).add(name) + def register_host_runner(self, session_id: str, name: str, fn: Any) -> None: + self._host_runners[(session_id, name)] = fn + + def host_runner(self, session_id: str, name: str) -> Any: + return self._host_runners.get((session_id, name)) + # -- skills (HP-09) --------------------------------------------------- + def sign_digest(self, digest: str) -> str: + if not self.skill_signing_key: + return "" + return hmac.new( + self.skill_signing_key.encode(), digest.encode(), hashlib.sha256 + ).hexdigest() + def register_manifest(self, manifest: SkillManifest) -> None: if manifest.spec not in ("jv", "claude"): raise HarnessContractError( f"unsupported skill spec {manifest.spec!r}; only jv and claude" ) + if manifest.digest in self.store.revoked_manifests: + raise HarnessContractError(f"revoked skill digest {manifest.digest}") + if self.skill_signing_key: + expected = self.sign_digest(manifest.digest) + if not hmac.compare_digest(manifest.signature or "", expected): + raise HarnessContractError("invalid skill signature") with self.store.lock: self.store.skill_manifests[manifest.digest] = manifest + def publish_manifest(self, manifest: SkillManifest) -> None: + self.register_manifest(manifest) + + def revoke_manifest(self, digest: str) -> None: + with self.store.lock: + self.store.revoked_manifests.add(digest) + self.store.skill_manifests.pop(digest, None) + def activate_skill( self, caller: NativeCaller, @@ -769,6 +867,8 @@ def activate_skill( trust_tier: str = "trusted", ) -> StageRecord: snap = self.require_usable(snapshot_id) + if digest in self.store.revoked_manifests: + raise HarnessContractError(f"revoked skill digest {digest}") manifest = self.store.skill_manifests.get(digest) if manifest is None: raise HarnessContractError(f"unknown skill digest {digest}") @@ -808,8 +908,16 @@ def record_span(self, correlation_id: str, name: str, **fields: Any) -> None: fields = dict(fields) fields["caller"] = fields["caller"].as_tuple() redacted = {k: v for k, v in fields.items() if k not in ("secret", "password")} + _log.info( + "harness.span %s", + json.dumps( + {"correlation_id": correlation_id, "name": name, **redacted}, + default=str, + ), + ) with self.store.lock: - self.store.traces.setdefault(correlation_id, []).append( + spans = self.store.traces.setdefault(correlation_id, []) + spans.append( { "name": name, "ts": datetime.now(timezone.utc).isoformat(), @@ -817,6 +925,9 @@ def record_span(self, correlation_id: str, name: str, **fields: Any) -> None: **redacted, } ) + overflow = len(spans) - self.max_trace_spans + if overflow > 0: + del spans[:overflow] def traces_for(self, correlation_id: str) -> List[Dict[str, Any]]: return list(self.store.traces.get(correlation_id, [])) @@ -839,6 +950,21 @@ def replay_document(self, correlation_id: str) -> Dict[str, Any]: "invocations": list(journal.completed_invocation_ids) if journal else [], } + def mark_background(self, correlation_id: str) -> None: + journal = self.get_run(correlation_id) + if journal is None or journal.state in TURN_RUN_TERMINAL: + return + journal.plan_phase = "background" + + def prune_retention(self) -> None: + with self.store.lock: + for sid, stream in list(self.store.outbox.items()): + if len(stream) > MAX_EVENTS_PER_SESSION: + self.store.outbox[sid] = stream[-MAX_EVENTS_PER_SESSION:] + for corr, spans in list(self.store.traces.items()): + if len(spans) > self.max_trace_spans: + self.store.traces[corr] = spans[-self.max_trace_spans :] + # -- internals -------------------------------------------------------- def _require_run(self, correlation_id: str) -> TurnRunJournal: @@ -902,6 +1028,8 @@ def set_runtime(runtime: HarnessRuntime) -> None: "HarnessStore", "MAX_EVENTS_PER_SESSION", "MAX_OBSERVATION_CHARS", + "MAX_TRACE_SPANS", + "TRACE_KIND", "SAME_SESSION_POLICY", "SessionBusy", "SkillIsolationRefused", diff --git a/pyproject.toml b/pyproject.toml index 065a7a30..a9c8d583 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -181,6 +181,8 @@ asyncio_default_fixture_loop_scope = "function" markers = [ "asyncio: mark test as an asyncio test", "harness_conformance: host-neutral harness contract suite (HP-00)", + "harness_isolation: skill isolation refuse/wrap (HP-09)", + "harness_load: HP-11 capacity benches", ] filterwarnings = [ "ignore::DeprecationWarning:pydantic.*", diff --git a/tests/conformance/cucs/recovery.yaml b/tests/conformance/cucs/recovery.yaml new file mode 100644 index 00000000..b2f6cfe4 --- /dev/null +++ b/tests/conformance/cucs/recovery.yaml @@ -0,0 +1,26 @@ +schema: jvagent.use-case/v1 +id: harness.recovery.non-retryable +title: Failed non-retryable tool marks the TurnRun recovery_required +priority: P0 +tags: [harness, recovery] +given: + channel: default + new_user: false +turns: + - id: send + when: + user: "Send the mail" + harness: + decisions: + - action: tool + tool: send_mail + args: + to: a@b.c + - action: final + answer: "queued for recovery" + then: + loop: + tools_called: + - send_mail + context: + recovery_required: true diff --git a/tests/conformance/cucs/safety-untrusted.yaml b/tests/conformance/cucs/safety-untrusted.yaml new file mode 100644 index 00000000..1fb3a914 --- /dev/null +++ b/tests/conformance/cucs/safety-untrusted.yaml @@ -0,0 +1,24 @@ +schema: jvagent.use-case/v1 +id: harness.safety.untrusted +title: Untrusted script skill refuses without an approved isolation backend +priority: P0 +tags: [harness, safety] +given: + channel: default + new_user: false +turns: + - id: refuse + when: + user: "Run this untrusted script skill" + harness: + decisions: + - action: tool + tool: activate_skill + args: + digest: digest-untrusted + trust_tier: untrusted + - action: final + answer: "refused" + then: + context: + isolation_refused: true diff --git a/tests/conformance/cucs/skill-activation.yaml b/tests/conformance/cucs/skill-activation.yaml new file mode 100644 index 00000000..13760227 --- /dev/null +++ b/tests/conformance/cucs/skill-activation.yaml @@ -0,0 +1,25 @@ +schema: jvagent.use-case/v1 +id: harness.skill.activation +title: Trusted skill activates from digest under the current snapshot +priority: P0 +tags: [harness, skill-activation] +given: + channel: default + new_user: false +turns: + - id: activate + when: + user: "Use the host skill" + harness: + decisions: + - action: tool + tool: activate_skill + args: + digest: digest-host + trust_tier: trusted + - action: final + answer: "activated" + then: + loop: + tools_called: + - activate_skill diff --git a/tests/conformance/cucs/tool-selection.yaml b/tests/conformance/cucs/tool-selection.yaml new file mode 100644 index 00000000..c873dd7d --- /dev/null +++ b/tests/conformance/cucs/tool-selection.yaml @@ -0,0 +1,27 @@ +schema: jvagent.use-case/v1 +id: harness.tool.selection +title: Host tool is selected from the admitted snapshot, not a live host import +priority: P0 +tags: [harness, tool-selection] +given: + channel: default + new_user: false +turns: + - id: lookup + when: + user: "Look up the host record" + harness: + decisions: + - action: tool + tool: host_lookup + args: + q: x + - action: final + answer: "found" + then: + loop: + tools_called: + - host_lookup + tools_surface: + includes: + - host_lookup diff --git a/tests/conformance/cucs/uniqueness.yaml b/tests/conformance/cucs/uniqueness.yaml new file mode 100644 index 00000000..b225fe88 --- /dev/null +++ b/tests/conformance/cucs/uniqueness.yaml @@ -0,0 +1,32 @@ +schema: jvagent.use-case/v1 +id: harness.response.uniqueness +title: Idempotent replay returns the cached observation; final event is unique by message id +priority: P0 +tags: [harness, uniqueness] +given: + channel: default + new_user: false +turns: + - id: echo-once + when: + user: "Echo this" + harness: + decisions: + - action: tool + tool: echo + args: + x: 1 + - action: tool + tool: echo + args: + x: 1 + - action: final + answer: "hello" + then: + loop: + tools_called: + - echo + - echo + context: + cached_replay: true + unique_final: true diff --git a/tests/conformance/fixtures/fake_host/__init__.py b/tests/conformance/fixtures/fake_host/__init__.py index 03217117..e09cf5d6 100644 --- a/tests/conformance/fixtures/fake_host/__init__.py +++ b/tests/conformance/fixtures/fake_host/__init__.py @@ -14,6 +14,11 @@ def fake_host_runtime() -> HarnessRuntime: rt = HarnessRuntime(store, worker_id="fake-host") rt.put_host_tools("sess-conformance", ["host_lookup"]) rt.put_host_skills("sess-conformance", ["host_skill"]) + + async def _lookup(payload: dict) -> dict: + return {"ok": True, "q": payload.get("q")} + + rt.register_host_runner("sess-conformance", "host_lookup", _lookup) return rt diff --git a/tests/conformance/test_cucs_harness.py b/tests/conformance/test_cucs_harness.py new file mode 100644 index 00000000..4df8443c --- /dev/null +++ b/tests/conformance/test_cucs_harness.py @@ -0,0 +1,138 @@ +"""HP-10 CUCS evals against HarnessRuntime doubles (not a live model).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from jvagent.harness.contracts import IdempotencyClass, NativeCaller, TurnRunState +from jvagent.harness.runtime import ( + HarnessRuntime, + HarnessStore, + SkillIsolationRefused, + SkillManifest, +) +from jvagent.testing.use_case_loader import discover_use_cases, load_use_case + +pytestmark = pytest.mark.harness_conformance + +CUCS_ROOT = Path(__file__).resolve().parent / "cucs" + + +def _run_scenario(data: dict) -> dict: + rt = HarnessRuntime(HarnessStore(), worker_id="cucs") + caller = NativeCaller("ag", "cucs-user", "cucs-sess") + rt.put_host_tools(caller.session_id, ["host_lookup"]) + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rt.register_manifest( + SkillManifest( + skill_key="host_skill", + source="cucs", + digest="digest-host", + declared_tools=(), + capabilities=(), + trust_tier="trusted", + ) + ) + rt.register_manifest( + SkillManifest( + skill_key="scripty", + source="cucs", + digest="digest-untrusted", + declared_tools=(), + capabilities=(), + trust_tier="untrusted", + spec="claude", + ) + ) + called: list[str] = [] + isolation_refused = False + recovery_required = False + cached_replay = False + for turn in data["turns"]: + for dec in (turn.get("harness") or {}).get("decisions") or []: + if dec.get("action") != "tool": + if dec.get("action") == "final": + rt.append_event( + session_id=caller.session_id, + kind="final", + message_id=turn["id"], + correlation_id=corr, + snapshot_id=snap.snapshot_id, + ) + continue + name = str(dec["tool"]) + args = dict(dec.get("args") or {}) + called.append(name) + if name == "activate_skill": + try: + rt.activate_skill( + caller, + snap.snapshot_id, + str(args.get("digest") or ""), + trust_tier=str(args.get("trust_tier") or "trusted"), + ) + except SkillIsolationRefused: + isolation_refused = True + continue + klass = None + if name == "send_mail": + klass = IdempotencyClass.NON_RETRYABLE + elif name == "echo": + klass = IdempotencyClass.IDEMPOTENT + rec, cached = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name=name, + payload=args, + idempotency_class=klass, + ) + if cached is not None: + cached_replay = True + continue + ok = name != "send_mail" + rt.finish_invocation( + correlation_id=corr, + record=rec, + result="ok" if ok else "failed", + ok=ok, + ) + run = rt.get_run(corr) + if run is not None and run.state is TurnRunState.RECOVERY_REQUIRED: + recovery_required = True + finals = [e for e in rt.replay_from(caller.session_id) if e.kind == "final"] + return { + "tools_called": called, + "isolation_refused": isolation_refused, + "recovery_required": recovery_required, + "cached_replay": cached_replay, + "unique_final": len({e.message_id for e in finals}) == len(finals), + "host_tools": list(snap.host_tool_names), + } + + +@pytest.mark.parametrize("path", discover_use_cases(CUCS_ROOT), ids=lambda p: p.stem) +def test_cucs_harness_eval(path: Path): + data = load_use_case(path) + observed = _run_scenario(data) + for turn in data["turns"]: + then = turn.get("then") or {} + loop = then.get("loop") or {} + expected_tools = loop.get("tools_called") + if expected_tools: + assert observed["tools_called"] == expected_tools + surface = then.get("tools_surface") or {} + for name in surface.get("includes") or []: + assert name in observed["host_tools"] + ctx = then.get("context") or {} + for key in ( + "isolation_refused", + "recovery_required", + "cached_replay", + "unique_final", + ): + if key in ctx: + assert observed[key] is ctx[key] diff --git a/tests/harness/test_gap_close.py b/tests/harness/test_gap_close.py new file mode 100644 index 00000000..288d5532 --- /dev/null +++ b/tests/harness/test_gap_close.py @@ -0,0 +1,260 @@ +"""Gap-close proofs: dump/load, leases, signatures, isolation wrap, retention.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from jvagent.action.code_execution.executor import ExecRequest +from jvagent.harness.contracts import ( + HarnessContractError, + IdempotencyClass, + NativeCaller, +) +from jvagent.harness.isolation import wrap_isolated_command +from jvagent.harness.leases import FileLeaseBackend, lease_backend_for +from jvagent.harness.persist import dump_store, load_store +from jvagent.harness.provider import provider_for +from jvagent.harness.runtime import ( + TRACE_KIND, + HarnessRuntime, + HarnessStore, + SessionBusy, + SkillIsolationRefused, + SkillManifest, +) + +pytestmark = pytest.mark.harness_conformance + + +@pytest.fixture +def caller() -> NativeCaller: + return NativeCaller("ag", "u1", "s1") + + +def test_dump_store_round_trip(tmp_path: Path, caller: NativeCaller): + rt = HarnessRuntime(HarnessStore(), worker_id="w1") + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rec, _ = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="echo", + payload={"x": 1}, + idempotency_class=IdempotencyClass.IDEMPOTENT, + ) + rt.finish_invocation(correlation_id=corr, record=rec, result="hello") + rt.append_event( + session_id=caller.session_id, + kind="final", + message_id="m1", + correlation_id=corr, + snapshot_id=snap.snapshot_id, + ) + path = tmp_path / "store.json" + dump_store(rt.store, path) + restored = load_store(path) + other = HarnessRuntime(restored, worker_id="w2") + journal = other.get_run(corr) + assert journal is not None + assert rec.invocation_id in journal.completed_invocation_ids + assert [e.message_id for e in other.replay_from(caller.session_id)] == ["m1"] + + +def test_file_lease_backend_contends(tmp_path: Path): + path = tmp_path / "leases.json" + a = FileLeaseBackend(path) + b = FileLeaseBackend(path) + a.acquire("s1", "w1") + with pytest.raises(SessionBusy): + b.acquire("s1", "w2") + a.release("s1", "w1") + b.acquire("s1", "w2") + assert b.get("s1")["worker_id"] == "w2" + + +def test_redis_and_dynamo_backends_require_clients(): + with pytest.raises(ValueError, match="no silent fallback"): + lease_backend_for("redis") + with pytest.raises(ValueError, match="no silent fallback"): + lease_backend_for("dynamodb") + + +def test_runtime_file_lease_backend(tmp_path: Path, caller: NativeCaller): + backend = lease_backend_for("file", path=tmp_path / "leases.json") + a = HarnessRuntime(HarnessStore(), worker_id="w1", lease_backend=backend) + b = HarnessRuntime(a.store, worker_id="w2", lease_backend=backend) + a.acquire_session_lease(caller.session_id) + with pytest.raises(SessionBusy): + b.acquire_session_lease(caller.session_id) + + +def test_skill_signature_and_revoke(caller: NativeCaller): + rt = HarnessRuntime(HarnessStore(), skill_signing_key="k") + digest = "abc123" + good = SkillManifest( + skill_key="k", + source="app", + digest=digest, + declared_tools=(), + capabilities=(), + trust_tier="trusted", + signature=rt.sign_digest(digest), + ) + rt.register_manifest(good) + with pytest.raises(HarnessContractError, match="signature"): + rt.register_manifest( + SkillManifest( + skill_key="k", + source="app", + digest="other", + declared_tools=(), + capabilities=(), + trust_tier="trusted", + signature="deadbeef", + ) + ) + snap = rt.admit_snapshot(caller) + rt.revoke_manifest(digest) + with pytest.raises(HarnessContractError, match="revoked"): + rt.activate_skill(caller, snap.snapshot_id, digest) + + +@pytest.mark.harness_isolation +def test_wrap_isolated_command_refuses_missing_binary(): + req = ExecRequest(command="echo hi", cwd="/tmp") + with pytest.raises(SkillIsolationRefused): + wrap_isolated_command("nsjail", req) + with pytest.raises(SkillIsolationRefused): + wrap_isolated_command("docker", req) + + +@pytest.mark.harness_isolation +def test_wrap_isolated_command_prefixes_when_binary_present(monkeypatch): + monkeypatch.setattr( + "jvagent.harness.isolation.shutil.which", lambda name: f"/usr/bin/{name}" + ) + req = ExecRequest(command="echo hi", cwd="/tmp") + wrapped = wrap_isolated_command("nsjail", req) + assert wrapped.command.startswith("nsjail -Mo --cwd /tmp -- /bin/sh -c ") + assert "echo hi" in wrapped.command + + +def test_prune_retention_trims_traces(caller: NativeCaller): + rt = HarnessRuntime(HarnessStore(), max_trace_spans=3) + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + for i in range(8): + rt.record_span(corr, f"tick-{i}") + rt.prune_retention() + assert len(rt.traces_for(corr)) == 3 + + +def test_journal_and_event_pagination(caller: NativeCaller): + rt = HarnessRuntime(HarnessStore()) + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + for i in range(5): + rt.append_event( + session_id=caller.session_id, + kind="chunk", + message_id=f"m{i}", + correlation_id=corr, + snapshot_id=snap.snapshot_id, + ) + page = rt.replay_from(caller.session_id, limit=2) + assert [e.message_id for e in page] == ["m0", "m1"] + page2 = rt.replay_from(caller.session_id, cursor=page[-1].cursor, limit=2) + assert [e.message_id for e in page2] == ["m2", "m3"] + entries = rt.journal_entries(corr, after_seq=0, limit=1) + assert len(entries) == 1 + + +@pytest.mark.asyncio +async def test_host_provider_invokes_registered_runner(caller: NativeCaller): + rt = HarnessRuntime(HarnessStore()) + rt.put_host_tools(caller.session_id, ["host_lookup"]) + + async def _lookup(payload: dict) -> dict: + return {"ok": True, "q": payload.get("q")} + + rt.register_host_runner(caller.session_id, "host_lookup", _lookup) + snap = rt.admit_snapshot(caller) + provider = provider_for("native", rt) + result = await provider.invoke(snap.snapshot_id, "inv-1", "host_lookup", {"q": "x"}) + assert result.ok is True + assert result.payload["q"] == "x" + + +def test_persist_writes_trace_kind(caller: NativeCaller): + rt = HarnessRuntime(HarnessStore()) + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rt.record_span(corr, "model_tick", caller=caller.as_tuple()) + + class _Ix: + def __init__(self) -> None: + self.observability_metrics: list = [] + + ix = _Ix() + rt.persist_to_interaction(ix, corr) + kinds = {m.get("kind") for m in ix.observability_metrics if isinstance(m, dict)} + assert TRACE_KIND in kinds + + +def test_background_phase(caller: NativeCaller): + rt = HarnessRuntime(HarnessStore()) + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rt.mark_background(corr) + assert rt.get_run(corr).plan_phase == "background" + + +def test_in_memory_redis_adapter_set_nx(caller: NativeCaller): + class _Redis: + def __init__(self) -> None: + self.d: dict = {} + + def set(self, k, v, nx=False, ex=None, xx=False): + if nx and k in self.d: + return False + self.d[k] = v + return True + + def get(self, k): + return self.d.get(k) + + def expire(self, k, ttl): + return k in self.d + + def delete(self, k): + self.d.pop(k, None) + + backend = lease_backend_for("redis", redis_client=_Redis()) + a = HarnessRuntime(HarnessStore(), worker_id="w1", lease_backend=backend) + b = HarnessRuntime(a.store, worker_id="w2", lease_backend=backend) + a.acquire_session_lease(caller.session_id) + with pytest.raises(SessionBusy): + b.acquire_session_lease(caller.session_id) + + +def test_compensate_without_registration_raises(caller: NativeCaller): + rt = HarnessRuntime(HarnessStore()) + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rec, _ = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="book", + payload={"n": 1}, + idempotency_class=IdempotencyClass.COMPENSATABLE, + ) + with pytest.raises(HarnessContractError, match="compensator"): + rt.compensate(rec.invocation_id) diff --git a/tests/harness/test_load.py b/tests/harness/test_load.py new file mode 100644 index 00000000..a2b00d68 --- /dev/null +++ b/tests/harness/test_load.py @@ -0,0 +1,98 @@ +"""HP-11 named benches. Not a claim of production p95; bounds the in-process store.""" + +from __future__ import annotations + +import asyncio +import time + +import pytest + +from jvagent.harness.contracts import NativeCaller +from jvagent.harness.runtime import HarnessRuntime, HarnessStore + +pytestmark = [pytest.mark.harness_conformance, pytest.mark.harness_load] + + +def test_snapshot_creation_budget(): + rt = HarnessRuntime(HarnessStore()) + t0 = time.perf_counter() + for i in range(200): + rt.admit_snapshot(NativeCaller("ag", f"u{i}", f"s{i}")) + assert (time.perf_counter() - t0) * 1000 < 2000 + + +def test_graph_session_and_turnrun_budget(): + rt = HarnessRuntime(HarnessStore()) + t0 = time.perf_counter() + for i in range(100): + caller = NativeCaller("ag", f"u{i}", f"s{i}") + rt.upsert_user("mem", caller.user_id) + rt.upsert_conversation("mem", caller.session_id) + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rt.complete_turn(corr) + assert (time.perf_counter() - t0) * 1000 < 2000 + + +def test_streaming_fan_out_budget(): + rt = HarnessRuntime(HarnessStore()) + caller = NativeCaller("ag", "u", "s") + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + t0 = time.perf_counter() + for i in range(500): + rt.append_event( + session_id=caller.session_id, + kind="chunk", + message_id=f"m{i}", + correlation_id=corr, + snapshot_id=snap.snapshot_id, + ) + page = rt.replay_from(caller.session_id, limit=50) + assert len(page) == 50 + assert (time.perf_counter() - t0) * 1000 < 2000 + + +def test_long_session_pruning_budget(): + rt = HarnessRuntime(HarnessStore(), max_trace_spans=50) + caller = NativeCaller("ag", "u", "s") + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + t0 = time.perf_counter() + for i in range(400): + rt.record_span(corr, "tick") + rt.append_event( + session_id=caller.session_id, + kind="chunk", + message_id=f"m{i}", + correlation_id=corr, + snapshot_id=snap.snapshot_id, + ) + rt.prune_retention() + assert len(rt.traces_for(corr)) <= 50 + assert (time.perf_counter() - t0) * 1000 < 2000 + + +@pytest.mark.asyncio +async def test_parallel_tool_execution_budget(): + store = HarnessStore() + + async def _one(i: int) -> None: + rt = HarnessRuntime(store, worker_id=f"w{i}") + caller = NativeCaller("ag", f"u{i}", f"s{i}") + snap = rt.admit_snapshot(caller) + corr = rt.new_correlation() + rt.start_turn(corr, caller, snap) + rec, _ = rt.begin_invocation( + correlation_id=corr, + snapshot_id=snap.snapshot_id, + tool_name="echo", + payload={"i": i}, + ) + rt.finish_invocation(correlation_id=corr, record=rec, result="ok") + + t0 = time.perf_counter() + await asyncio.gather(*[_one(i) for i in range(40)]) + assert (time.perf_counter() - t0) * 1000 < 2000 diff --git a/tests/harness/test_runtime.py b/tests/harness/test_runtime.py index 17db76e9..4f769535 100644 --- a/tests/harness/test_runtime.py +++ b/tests/harness/test_runtime.py @@ -336,6 +336,7 @@ def test_compensatable_path(rt: HarnessRuntime, caller: NativeCaller): snap = rt.admit_snapshot(caller) corr = rt.new_correlation() rt.start_turn(corr, caller, snap) + rt.register_compensator("book", lambda rec: f"compensated:{rec.invocation_id}") rec, _ = rt.begin_invocation( correlation_id=corr, snapshot_id=snap.snapshot_id, @@ -344,6 +345,7 @@ def test_compensatable_path(rt: HarnessRuntime, caller: NativeCaller): idempotency_class=IdempotencyClass.COMPENSATABLE, ) assert rt.compensate(rec.invocation_id).startswith("compensated:") + rt.finish_invocation(correlation_id=corr, record=rec, result="failed", ok=False) def test_completed_run_not_resumed(rt: HarnessRuntime, caller: NativeCaller): From 1fa70ff45abab7233f2b896fc64ae3665ad86948 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Fri, 18 Sep 2026 00:29:13 -0400 Subject: [PATCH 11/20] feat(harness): surface host tools and restore durable reply frames Host capabilities join the standard tool catalog, skill bodies are real materializations, recovery_required stops re-entry, and outbox replay carries the original response payload. --- .../orchestrator_interact_action.py | 27 ++++++- .../action/orchestrator/skill_providers.py | 14 +++- jvagent/action/orchestrator/tools.py | 72 +++++++++++++++++++ jvagent/action/response/response_bus.py | 1 + jvagent/action/response/streaming.py | 41 +++++++---- jvagent/harness/contracts.py | 5 +- jvagent/harness/persist.py | 14 ++++ jvagent/harness/provider.py | 9 +++ jvagent/harness/runtime.py | 37 ++++++++++ .../orchestrator/test_harness_recovery.py | 43 +++++++++++ .../fixtures/fake_host/__init__.py | 8 ++- tests/conformance/test_delivery_replay.py | 43 ++++++++++- tests/conformance/test_provider_contract.py | 1 + tests/harness/test_runtime.py | 66 +++++++++++++++++ 14 files changed, 364 insertions(+), 17 deletions(-) create mode 100644 tests/action/orchestrator/test_harness_recovery.py diff --git a/jvagent/action/orchestrator/orchestrator_interact_action.py b/jvagent/action/orchestrator/orchestrator_interact_action.py index c60f285d..8069ddcc 100644 --- a/jvagent/action/orchestrator/orchestrator_interact_action.py +++ b/jvagent/action/orchestrator/orchestrator_interact_action.py @@ -122,6 +122,7 @@ salvage_tool_call_text, truncate_thought, wrap_action_tool, + wrap_host_tool, ) from jvagent.action.orchestrator.turn_cache import ( bind_turn_cache, @@ -973,6 +974,18 @@ async def execute(self, visitor: "InteractWalker") -> None: payload = rt.checkpoint_from_interaction(interaction) if payload: restored = rt.import_checkpoint(payload) + if ( + restored is not None + and restored.state is TurnRunState.RECOVERY_REQUIRED + ): + await visitor.report( + { + "recovery_required": True, + "correlation_id": restored.correlation_id, + "reason": restored.reason, + } + ) + return if restored is not None and restored.state not in ( TurnRunState.COMPLETED, TurnRunState.FAILED, @@ -1414,6 +1427,19 @@ async def _egress_exec( visible.add("reply") visible.add("respond") + # Host capabilities are snapshot-bound tools, not descriptor metadata. + # Native tools win name collisions, and host tools remain discoverable + # through the standard lean catalogue. + snap = (get_turn_cache() or {}).get("snapshot") + if snap is not None: + for name in getattr(snap, "host_tool_names", ()) or (): + if not name or name in tools: + if name in tools: + logger.warning("host tool %r conflicts with native tool", name) + continue + tools[name] = wrap_host_tool(name) + longtail.add(name) + # Skill-only gating (ADR-0043), part 1 — the GLOB MATCH. It runs HERE, # before the lean policy, because a gated name must not win a lean # pre-surface slot only to be discarded again at install time: gating one @@ -1636,7 +1662,6 @@ async def _egress_exec( longtail.discard(name) snap = None try: - from jvagent.action.orchestrator.turn_cache import get_turn_cache from jvagent.harness.runtime import get_runtime turn = get_turn_cache() or {} diff --git a/jvagent/action/orchestrator/skill_providers.py b/jvagent/action/orchestrator/skill_providers.py index 8b70a9d8..49513f62 100644 --- a/jvagent/action/orchestrator/skill_providers.py +++ b/jvagent/action/orchestrator/skill_providers.py @@ -55,15 +55,27 @@ def collect_host_skill_docs(agent: Any) -> List[SkillDoc]: turn = get_turn_cache() or {} snap = turn.get("snapshot") if snap is not None: + from jvagent.harness.runtime import get_runtime + existing = {d.name for d in docs} for key in getattr(snap, "host_skill_keys", ()) or (): if key and key not in existing: + materialization = get_runtime().host_skill_materialization( + snap.caller.session_id, key + ) + if materialization is None: + logger.warning( + "host skill %r has no registered materialization", key + ) + continue docs.append( SkillDoc( name=key, description=f"Host skill {key}", - body="", + body=materialization.body, source="host", + spec=materialization.spec, + digest=materialization.digest, ) ) except Exception as exc: diff --git a/jvagent/action/orchestrator/tools.py b/jvagent/action/orchestrator/tools.py index 6fb770d8..9dd97e74 100644 --- a/jvagent/action/orchestrator/tools.py +++ b/jvagent/action/orchestrator/tools.py @@ -206,6 +206,78 @@ async def _run(args: Dict[str, Any], _tool: Any = tool) -> str: ) +def wrap_host_tool(name: str, *, description: str = "") -> SkillTool: + """Adapt one snapshot-declared host capability into the model tool surface.""" + + async def _run(args: Dict[str, Any]) -> str: + from jvagent.action.orchestrator.turn_cache import get_turn_cache + from jvagent.harness.contracts import ( + HarnessContractError, + reject_model_authority_fields, + ) + from jvagent.harness.provider import provider_for + from jvagent.harness.runtime import get_runtime + + payload = dict(args or {}) + record = None + runtime = None + correlation_id = "" + interaction = None + try: + reject_model_authority_fields(payload) + turn = get_turn_cache() or {} + snap = turn.get("snapshot") + correlation_id = str(turn.get("correlation_id") or "") + interaction = turn.get("interaction") + if snap is None or not correlation_id: + raise HarnessContractError("host tool called outside an admitted turn") + runtime = get_runtime() + record, cached = runtime.begin_invocation( + correlation_id=correlation_id, + snapshot_id=snap.snapshot_id, + tool_name=name, + payload=payload, + ) + if cached is not None: + return cached + provider = turn.get("provider") or provider_for("native", runtime) + result = await provider.invoke( + snap.snapshot_id, record.invocation_id, name, payload + ) + content = json.dumps(dict(result.payload), default=str) + ok = bool(result.ok) + except HarnessContractError as exc: + if record is None: + return f"(tool error: {exc})" + content = f"(tool error: {exc})" + ok = False + except Exception as exc: + logger.warning("host tool %r raised: %s", name, exc) + content = f"(tool error: {exc})" + ok = False + if runtime is not None and record is not None: + runtime.finish_invocation( + correlation_id=correlation_id, + record=record, + result=content, + ok=ok, + ) + if interaction is not None: + runtime.persist_to_interaction(interaction, correlation_id) + return content + + return SkillTool( + name=name, + description=description or f"Host-provided capability: {name}", + run=_run, + parameters_schema={ + "type": "object", + "properties": {}, + "additionalProperties": True, + }, + ) + + def render_tools_section(tools: List[Any], *, lean: bool = False) -> str: """Render ``[{name, description}]`` (or objects) as a bulleted list. diff --git a/jvagent/action/response/response_bus.py b/jvagent/action/response/response_bus.py index 0be3bd99..ae2bf04e 100644 --- a/jvagent/action/response/response_bus.py +++ b/jvagent/action/response/response_bus.py @@ -321,6 +321,7 @@ async def _enqueue_and_notify( ), correlation_id=str(turn.get("correlation_id") or ""), snapshot_id=str(getattr(snap, "snapshot_id", "") or ""), + payload=message.to_dict(), ) except Exception: pass diff --git a/jvagent/action/response/streaming.py b/jvagent/action/response/streaming.py index 0c70c304..93ae7ac5 100644 --- a/jvagent/action/response/streaming.py +++ b/jvagent/action/response/streaming.py @@ -12,6 +12,14 @@ def _sse_dedup_key(message: Any) -> tuple: """Dedup key for SSE replay overlap — (id, message_type, sequence).""" + if isinstance(message, dict): + mid = message.get("id") or message.get("message_id") or "" + mtype = message.get("message_type") or "" + meta = message.get("metadata") or {} + seq = meta.get("sequence") if isinstance(meta, dict) else None + if seq is None: + seq = message.get("content") or "" + return (mid, mtype, seq) mid = getattr(message, "id", None) or getattr(message, "message_id", None) or "" mtype = getattr(message, "message_type", "") or "" meta = getattr(message, "metadata", None) or {} @@ -88,6 +96,9 @@ async def message_callback(message: Any) -> None: await response_bus.subscribe(session_id, message_callback, receive_chunks=True) try: + # IDs emitted from durable replay must not also be emitted when an + # in-process queue still has the same response during reconnect. + replayed_ids: set = set() # Durable outbox replay (HP-06) when a cursor is supplied. Live bus # backlog still covers in-process overlap; message ids remain the # dedup key (test_streaming_dedup). @@ -96,22 +107,28 @@ async def message_callback(message: Any) -> None: from jvagent.harness.runtime import get_runtime for env in get_runtime().replay_from(session_id, cursor): - yield format_sse_chunk( - { - "session_id": env.session_id, - "sequence": env.sequence, - "cursor": env.cursor, - "message_id": env.message_id, - "correlation_id": env.correlation_id, - "snapshot_id": env.snapshot_id, - "kind": env.kind, - } - ) + frame = dict(env.payload) + if not frame: + logger.warning( + "outbox event %s has no replayable payload", env.cursor + ) + continue + if not frame.get("id") and not frame.get("message_id"): + frame["id"] = env.message_id + if not frame.get("message_type"): + frame["message_type"] = env.kind + frame["harness"] = { + "sequence": env.sequence, + "cursor": env.cursor, + "correlation_id": env.correlation_id, + "snapshot_id": env.snapshot_id, + } + replayed_ids.add(_sse_dedup_key(frame)) + yield format_sse_chunk(frame) except Exception as exc: logger.debug("outbox cursor replay skipped: %s", exc) # Send any existing messages first, recording their ids for dedup. - replayed_ids: set = set() existing_messages = await response_bus.get_messages(session_id) if max_replay is not None and len(existing_messages) > max_replay: existing_messages = existing_messages[-max_replay:] diff --git a/jvagent/harness/contracts.py b/jvagent/harness/contracts.py index 2ac97269..f3a99da9 100644 --- a/jvagent/harness/contracts.py +++ b/jvagent/harness/contracts.py @@ -5,7 +5,7 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone from enum import Enum from typing import Any, Mapping, Optional, Protocol, Tuple @@ -209,6 +209,9 @@ class EventEnvelope: correlation_id: str snapshot_id: str kind: str + # The original transport frame. Metadata alone cannot reconstruct an + # assistant reply after the process-local ResponseBus has disappeared. + payload: Mapping[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: if self.sequence < 1: diff --git a/jvagent/harness/persist.py b/jvagent/harness/persist.py index bfa611b4..2a3fa0fd 100644 --- a/jvagent/harness/persist.py +++ b/jvagent/harness/persist.py @@ -16,6 +16,7 @@ EventEnvelope, IdempotencyClass, InvocationRecord, + SkillMaterialization, ToolSurfaceSnapshot, TurnRunState, native_caller_from_mapping, @@ -110,6 +111,10 @@ def dump_store(store: HarnessStore, path: Path) -> None: "stages": stages, "host_tools": {k: list(v) for k, v in store.host_tools.items()}, "host_skills": {k: list(v) for k, v in store.host_skills.items()}, + "host_skill_materializations": { + _tuple_key(key): asdict(value) + for key, value in store.host_skill_materializations.items() + }, "revoked_host_tools": { k: sorted(v) for k, v in store.revoked_host_tools.items() }, @@ -223,6 +228,15 @@ def load_store(path: Path, store: Optional[HarnessStore] = None) -> HarnessStore target.host_skills = { k: list(v) for k, v in (raw.get("host_skills") or {}).items() } + target.host_skill_materializations = { + _split_pair(key): SkillMaterialization( + skill_key=str(value["skill_key"]), + digest=str(value["digest"]), + spec=str(value["spec"]), + body=str(value["body"]), + ) + for key, value in (raw.get("host_skill_materializations") or {}).items() + } target.revoked_host_tools = { k: set(v) for k, v in (raw.get("revoked_host_tools") or {}).items() } diff --git a/jvagent/harness/provider.py b/jvagent/harness/provider.py index b1558bb1..d993f4b4 100644 --- a/jvagent/harness/provider.py +++ b/jvagent/harness/provider.py @@ -74,6 +74,15 @@ async def load_skill( raise HarnessContractError( f"skill {skill_key!r} not on snapshot {snapshot_id}" ) + if skill_key in snap.host_skill_keys: + materialization = self.runtime.host_skill_materialization( + snap.caller.session_id, skill_key + ) + if materialization is None: + raise HarnessContractError( + f"host skill {skill_key!r} has no registered materialization" + ) + return materialization digest = f"digest-{skill_key}" return SkillMaterialization( skill_key=skill_key, diff --git a/jvagent/harness/runtime.py b/jvagent/harness/runtime.py index 74fd2649..ffed006c 100644 --- a/jvagent/harness/runtime.py +++ b/jvagent/harness/runtime.py @@ -25,6 +25,7 @@ IdempotencyClass, InvocationRecord, NativeCaller, + SkillMaterialization, SnapshotSelector, ToolSurfaceSnapshot, TurnRunState, @@ -122,6 +123,9 @@ class HarnessStore: stages: Dict[str, StageRecord] = field(default_factory=dict) host_tools: Dict[str, List[str]] = field(default_factory=dict) host_skills: Dict[str, List[str]] = field(default_factory=dict) + host_skill_materializations: Dict[Tuple[str, str], SkillMaterialization] = field( + default_factory=dict + ) revoked_host_tools: Dict[str, set] = field(default_factory=dict) revoked_manifests: set = field(default_factory=set) breaker_states: Dict[str, Any] = field(default_factory=dict) @@ -650,6 +654,7 @@ def append_event( message_id: str, correlation_id: str, snapshot_id: str, + payload: Optional[Mapping[str, Any]] = None, ) -> EventEnvelope: with self.store.lock: stream = self.store.outbox.setdefault(session_id, []) @@ -664,6 +669,7 @@ def append_event( correlation_id=correlation_id, snapshot_id=snapshot_id, kind=kind, + payload=dict(payload or {}), ) stream.append(env) self.record_span( @@ -817,6 +823,37 @@ def put_host_skills(self, session_id: str, keys: List[str]) -> None: with self.store.lock: self.store.host_skills[session_id] = list(keys) + def register_host_skill( + self, + session_id: str, + skill_key: str, + *, + digest: str, + spec: str, + body: str, + ) -> None: + """Register the immutable materialization a host exposes for one session.""" + if spec not in ("jv", "claude"): + raise HarnessContractError(f"unsupported host skill spec {spec!r}") + materialization = SkillMaterialization( + skill_key=skill_key, + digest=digest, + spec=spec, + body=body, + ) + with self.store.lock: + keys = self.store.host_skills.setdefault(session_id, []) + if skill_key not in keys: + keys.append(skill_key) + self.store.host_skill_materializations[(session_id, skill_key)] = ( + materialization + ) + + def host_skill_materialization( + self, session_id: str, skill_key: str + ) -> Optional[SkillMaterialization]: + return self.store.host_skill_materializations.get((session_id, skill_key)) + def revoke_host_tool(self, session_id: str, name: str) -> None: with self.store.lock: self.store.revoked_host_tools.setdefault(session_id, set()).add(name) diff --git a/tests/action/orchestrator/test_harness_recovery.py b/tests/action/orchestrator/test_harness_recovery.py new file mode 100644 index 00000000..2a2f3bea --- /dev/null +++ b/tests/action/orchestrator/test_harness_recovery.py @@ -0,0 +1,43 @@ +"""TurnRun recovery states must stop the orchestrator before any new effects.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from jvagent.harness.contracts import NativeCaller +from jvagent.harness.runtime import HarnessRuntime, HarnessStore, reset_runtime + + +@pytest.mark.asyncio +async def test_recovery_required_checkpoint_never_reenters_turn_execution( + make_orchestrator, make_visitor, monkeypatch +): + visitor = make_visitor() + visitor.agent_id = "agent-1" + visitor.session_id = "session-1" + runtime = HarnessRuntime(HarnessStore()) + reset_runtime(runtime) + caller = NativeCaller("agent-1", "u", "session-1") + snapshot = runtime.admit_snapshot(caller) + correlation_id = runtime.new_correlation() + runtime.start_turn(correlation_id, caller, snapshot, interaction_id="int_1") + runtime.mark_recovery(correlation_id, reason="crash_after_dispatch") + visitor.interaction.observability_metrics = [ + { + "kind": "harness.turn_run", + "payload": runtime.export_checkpoint(correlation_id), + } + ] + visitor.report = AsyncMock() + orchestrator = make_orchestrator() + execute_turn = AsyncMock() + monkeypatch.setattr(orchestrator, "_execute_turn", execute_turn) + + await orchestrator.execute(visitor) + + execute_turn.assert_not_awaited() + visitor.report.assert_awaited_once() + assert visitor.report.await_args.args[0]["recovery_required"] is True + reset_runtime() diff --git a/tests/conformance/fixtures/fake_host/__init__.py b/tests/conformance/fixtures/fake_host/__init__.py index e09cf5d6..2a916ffb 100644 --- a/tests/conformance/fixtures/fake_host/__init__.py +++ b/tests/conformance/fixtures/fake_host/__init__.py @@ -13,7 +13,13 @@ def fake_host_runtime() -> HarnessRuntime: store = HarnessStore() rt = HarnessRuntime(store, worker_id="fake-host") rt.put_host_tools("sess-conformance", ["host_lookup"]) - rt.put_host_skills("sess-conformance", ["host_skill"]) + rt.register_host_skill( + "sess-conformance", + "host_skill", + digest="fake-host-skill-v1", + spec="jv", + body="# host_skill\n\nUse host_lookup before replying.", + ) async def _lookup(payload: dict) -> dict: return {"ok": True, "q": payload.get("q")} diff --git a/tests/conformance/test_delivery_replay.py b/tests/conformance/test_delivery_replay.py index 9789129d..b1c79566 100644 --- a/tests/conformance/test_delivery_replay.py +++ b/tests/conformance/test_delivery_replay.py @@ -4,8 +4,9 @@ import pytest +from jvagent.action.response.streaming import stream_messages from jvagent.harness.contracts import EventEnvelope, HarnessContractError, NativeCaller -from jvagent.harness.runtime import HarnessRuntime, HarnessStore +from jvagent.harness.runtime import HarnessRuntime, HarnessStore, reset_runtime pytestmark = pytest.mark.harness_conformance @@ -70,3 +71,43 @@ def test_reconnecting_client_replays_missed_frames_in_order(): replayed = w2.replay_from("s1", "s1:1") assert [e.message_id for e in replayed] == ["m2"] assert [e.sequence for e in replayed] == [2] + + +@pytest.mark.asyncio +async def test_restart_replay_contains_original_response_frame(): + """A fresh ResponseBus replays text from the durable outbox alone.""" + + class EmptyBus: + async def subscribe(self, *_args, **_kwargs): + return None + + async def get_messages(self, _session_id): + return [] + + async def unsubscribe(self, *_args, **_kwargs): + return None + + runtime = HarnessRuntime(HarnessStore()) + caller = NativeCaller("ag", "u1", "s1") + snapshot = runtime.admit_snapshot(caller) + runtime.append_event( + session_id=caller.session_id, + kind="final", + message_id="m-final", + correlation_id="corr-1", + snapshot_id=snapshot.snapshot_id, + payload={ + "id": "m-final", + "session_id": caller.session_id, + "message_type": "final", + "content": "Recovered answer", + "metadata": {}, + }, + ) + reset_runtime(runtime) + generator = stream_messages(caller.session_id, EmptyBus(), cursor="s1:0") + frame = await generator.__anext__() + await generator.aclose() + assert "Recovered answer" in frame + assert '"cursor": "s1:1"' in frame + reset_runtime() diff --git a/tests/conformance/test_provider_contract.py b/tests/conformance/test_provider_contract.py index 722df49b..06afffaa 100644 --- a/tests/conformance/test_provider_contract.py +++ b/tests/conformance/test_provider_contract.py @@ -42,6 +42,7 @@ async def test_provider_revocation_takes_effect_on_next_snapshot(transport): assert first.snapshot_id != second.snapshot_id skill = await provider.load_skill(second.snapshot_id, "host_skill") assert skill.skill_key == "host_skill" + assert "host_lookup" in skill.body encode = getattr(provider, "encode_caller", None) if callable(encode): blob = encode(caller) diff --git a/tests/harness/test_runtime.py b/tests/harness/test_runtime.py index 4f769535..5ec2624b 100644 --- a/tests/harness/test_runtime.py +++ b/tests/harness/test_runtime.py @@ -425,3 +425,69 @@ def test_peek_completed_skips_non_idempotent(rt: HarnessRuntime, caller: NativeC ) is None ) + + +@pytest.mark.asyncio +async def test_snapshot_host_tool_is_callable_from_the_standard_tool_surface( + caller: NativeCaller, +): + from jvagent.action.orchestrator.tools import wrap_host_tool + from jvagent.action.orchestrator.turn_cache import bind_turn_cache + + runtime = HarnessRuntime(HarnessStore()) + reset_runtime(runtime) + runtime.put_host_tools(caller.session_id, ["host_lookup"]) + runtime.register_host_runner( + caller.session_id, + "host_lookup", + lambda payload: {"answer": payload["q"]}, + ) + snapshot = runtime.admit_snapshot(caller) + corr = runtime.new_correlation() + runtime.start_turn(corr, caller, snapshot) + with bind_turn_cache() as turn: + turn["snapshot"] = snapshot + turn["correlation_id"] = corr + result = await wrap_host_tool("host_lookup").run({"q": "found"}) + assert '"answer": "found"' in result + assert runtime.get_run(corr).completed_invocation_ids + reset_runtime() + + +@pytest.mark.asyncio +async def test_host_tool_invoke_error_finishes_the_ledger(caller: NativeCaller): + from jvagent.action.orchestrator.tools import wrap_host_tool + from jvagent.action.orchestrator.turn_cache import bind_turn_cache + + runtime = HarnessRuntime(HarnessStore()) + reset_runtime(runtime) + runtime.put_host_tools(caller.session_id, ["host_lookup"]) + snapshot = runtime.admit_snapshot(caller) + corr = runtime.new_correlation() + runtime.start_turn(corr, caller, snapshot) + with bind_turn_cache() as turn: + turn["snapshot"] = snapshot + turn["correlation_id"] = corr + result = await wrap_host_tool("host_lookup").run({"q": "x"}) + assert result.startswith("(tool error:") + assert runtime.get_run(corr).state is TurnRunState.RUNNING + reset_runtime() + + +@pytest.mark.asyncio +async def test_host_skill_materialization_is_not_a_placeholder(caller: NativeCaller): + from jvagent.harness.provider import LocalHostProvider + + runtime = HarnessRuntime(HarnessStore()) + runtime.register_host_skill( + caller.session_id, + "host_procedure", + digest="host-procedure-v1", + spec="jv", + body="# Host procedure\n\nCall host_lookup first.", + ) + snapshot = runtime.admit_snapshot(caller) + materialization = await LocalHostProvider(runtime).load_skill( + snapshot.snapshot_id, "host_procedure" + ) + assert materialization.body == "# Host procedure\n\nCall host_lookup first." From 0dc28789b8313228faa5e13207ff99ba3221cbca Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Fri, 18 Sep 2026 01:20:14 -0400 Subject: [PATCH 12/20] fix(response): keep one assistant identity on the streamed turn Non-stream publish during an open accumulator minted a second Object id, and finalize_interaction emitted another final under a fresh uuid. Integral splits bubbles on those ids. --- CHANGELOG.md | 9 ++ jvagent/action/response/response_bus.py | 47 ++++--- jvchat/src/hooks/useStreaming.test.tsx | 63 ++++++++++ jvchat/src/hooks/useStreaming.ts | 73 ++++++----- tests/action/response/test_emitted_latch.py | 128 ++++++++++++++++++++ 5 files changed, 277 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ab742d7..a364a294 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,15 @@ and this project adheres to [PEP 440](https://peps.python.org/pep-0440/) / `Test jvagent` run has succeeded. The `jvchat` job has a 20-minute job timeout, an 8-minute install timeout and one `npm ci` retry. +### Fixed + +- **One assistant identity per streamed turn.** Non-stream `publish()` while a + user accumulator is open is suppressed (it minted a new Object id, which + Integral splits into a second bubble). `finalize_interaction` no longer + emits a second `message_type=final` under a fresh id when the stream already + finalized. `commit_pending_adhoc` reuses `acc.message_id`. jvchat merges a + same-id adhoc flush into the in-flight stream row. + ### Added - **Nightly reproduction of the #203 failure.** `scripts/live_smoke.py` gains diff --git a/jvagent/action/response/response_bus.py b/jvagent/action/response/response_bus.py index ae2bf04e..888e1c76 100644 --- a/jvagent/action/response/response_bus.py +++ b/jvagent/action/response/response_bus.py @@ -467,11 +467,14 @@ async def _deliver_flush( # ``Interaction.emitted`` is the framework's single-egress latch # (ADR-0025). A live user stream may continue after its first chunk set # the latch, but every separate non-transient user publish is rejected - # here at the delivery choke point. Consumers should never need to - # compare reply text or repair duplicate bubbles. - active_user_stream = bool( - stream and interaction_id and interaction_id in self._adhoc_accumulation + # here at the delivery choke point. A non-stream publish while the + # accumulator is already open mints a new Object id; Integral splits + # bubbles on that id, so it is suppressed even if the first chunk has + # not latched yet (gate-held / empty). + open_user_stream = bool( + interaction_id and interaction_id in self._adhoc_accumulation ) + active_user_stream = bool(stream and open_user_stream) has_emitted = getattr(interaction, "has_emitted", None) already_emitted = False if callable(has_emitted): @@ -484,8 +487,10 @@ async def _deliver_flush( and not transient and interaction is not None and content - and already_emitted - and not active_user_stream + and ( + (already_emitted and not active_user_stream) + or (not stream and open_user_stream) + ) ): logger.debug( "response bus: suppressed second user egress for interaction %s", @@ -854,6 +859,7 @@ async def commit_pending_adhoc( return now = await self._get_now() message = ResponseMessage( + id=acc.message_id, session_id=acc.session_id, user_id=acc.user_id or "", interaction_id=interaction_id, @@ -1157,15 +1163,28 @@ async def finalize_interaction( # Token spend is computed in the endpoint after flush, when all model_call # events are present in observability_metrics. - # Emit final signal + # Streaming publish() already enqueued message_type=final under + # acc.message_id. A second final with a new Object id is a distinct + # assistant identity on the wire (Integral splits bubbles on that). user_id = getattr(interaction, "user_id", None) if interaction else None - await self._emit_final_signal( - session_id=session_id, - channel=channel, - interaction_id=interaction_id, - user_id=user_id, - metadata={}, - ) + last_user_id = None + already_final = False + for buffered in self._message_buffers.get(interaction_id) or []: + if (getattr(buffered, "category", "user") or "user") != "user": + continue + if buffered.id: + last_user_id = buffered.id + if buffered.message_type == "final": + already_final = True + if not already_final: + await self._emit_final_signal( + session_id=session_id, + channel=channel, + interaction_id=interaction_id, + user_id=user_id, + metadata={}, + message_id=last_user_id, + ) # Clean up request-scoped resources (adhoc, message buffers) self._adhoc_accumulation.pop(interaction_id, None) diff --git a/jvchat/src/hooks/useStreaming.test.tsx b/jvchat/src/hooks/useStreaming.test.tsx index 08e36764..697648fc 100644 --- a/jvchat/src/hooks/useStreaming.test.tsx +++ b/jvchat/src/hooks/useStreaming.test.tsx @@ -380,4 +380,67 @@ describe("useStreaming thought handling", () => { expect(result.current.error).toMatch(/unauthorized|failed/i); }); + + it("merges user adhoc flush into the streaming row instead of duplicating", async () => { + mockStreamInteract.mockImplementation(async (_agentId, _request, onChunk) => { + onChunk({ + type: "start", + interaction_id: "int-user-merge", + session_id: "sess-user-merge", + }); + onChunk({ + type: "message", + message: { + id: "o.ResponseMessage.user123", + session_id: "sess-user-merge", + interaction_id: "int-user-merge", + message_type: "stream_chunk", + content: "Ships ", + channel: "default", + category: "user", + metadata: {}, + }, + }); + onChunk({ + type: "message", + message: { + id: "o.ResponseMessage.user123", + session_id: "sess-user-merge", + interaction_id: "int-user-merge", + message_type: "adhoc", + content: "Ships Tuesday.", + channel: "default", + category: "user", + metadata: {}, + }, + }); + onChunk({ + type: "final", + interaction: { + id: "int-user-merge", + utterance: "when", + actions: [], + directives: [], + parameters: [], + model_log: [], + messages: [], + streamed: true, + }, + }); + }); + + const { result } = renderHook(() => useStreaming("agent-1", "sess-user-merge")); + + await act(async () => { + await result.current.sendMessage("when"); + }); + + await waitFor(() => { + expect(result.current.isStreaming).toBe(false); + }); + + const assistants = result.current.messages.filter((m) => m.role === "assistant"); + expect(assistants).toHaveLength(1); + expect(assistants[0]?.content).toBe("Ships Tuesday."); + }); }); diff --git a/jvchat/src/hooks/useStreaming.ts b/jvchat/src/hooks/useStreaming.ts index b3ccb715..921231e4 100644 --- a/jvchat/src/hooks/useStreaming.ts +++ b/jvchat/src/hooks/useStreaming.ts @@ -644,32 +644,37 @@ export function useStreaming(agentId: string, sessionId?: string) { }) // Don't set isStreaming to false here - wait for chunk.type='final' for complete payload } else if (msg.message_type === 'adhoc') { - const adhocMessage: Message = { - id: msg.id || `adhoc-${Date.now()}-${Math.random()}`, - role: 'assistant', - interactionId: msg.interaction_id, - content: msg.content || '', - timestamp: msg.timestamp || new Date().toISOString(), - streaming: false, - metadata: mergeResponseMetadata(undefined, msg.metadata), - } - const streamSessionId = streamSessionIdRef.current - const currentView = sessionIdRef.current - const viewingStreamSession = isViewingStreamSession(currentView, streamSessionId) - - if (!viewingStreamSession && streamSessionId) { - const stored = getMessages(streamSessionId) - const updated = [...stored, adhocMessage] - saveMessages(streamSessionId, updated) - return + const messageId = msg.id || `adhoc-${Date.now()}-${Math.random()}` + const upsertAdhoc = (list: Message[]): Message[] => { + const existingIndex = list.findIndex((m) => m.id === messageId) + if (existingIndex >= 0) { + return list.map((m, idx) => + idx === existingIndex + ? { + ...m, + content: msg.content || m.content || '', + streaming: false, + interactionId: m.interactionId || msg.interaction_id, + metadata: mergeResponseMetadata(m.metadata, msg.metadata), + timestamp: msg.timestamp || m.timestamp, + } + : m + ) + } + return [ + ...list, + { + id: messageId, + role: 'assistant' as const, + interactionId: msg.interaction_id, + content: msg.content || '', + timestamp: msg.timestamp || new Date().toISOString(), + streaming: false, + metadata: mergeResponseMetadata(undefined, msg.metadata), + }, + ] } - - setMessages((prev) => { - // Append as new message (don't update existing messages) - let updated = [...prev, adhocMessage] - - // Ensure only the last message of each interaction has debugData - // Group messages by interactionId and keep debugData only on the last message per interaction + const stripDebugToLast = (updated: Message[]): Message[] => { const interactionGroups = new Map() updated.forEach((m, idx) => { if (m.role === 'assistant' && m.interactionId) { @@ -678,9 +683,7 @@ export function useStreaming(agentId: string, sessionId?: string) { interactionGroups.set(m.interactionId, indices) } }) - - // Remove debugData from all messages except the last one per interaction - updated = updated.map((m, idx) => { + return updated.map((m, idx) => { if (m.role === 'assistant' && m.interactionId && m.debugData) { const indices = interactionGroups.get(m.interactionId) || [] const lastIndexForInteraction = indices.length > 0 ? indices[indices.length - 1] : -1 @@ -691,8 +694,20 @@ export function useStreaming(agentId: string, sessionId?: string) { } return m }) + } + const streamSessionId = streamSessionIdRef.current + const currentView = sessionIdRef.current + const viewingStreamSession = isViewingStreamSession(currentView, streamSessionId) - // Save adhoc message immediately if we have a session ID + if (!viewingStreamSession && streamSessionId) { + const stored = getMessages(streamSessionId) + const updated = stripDebugToLast(upsertAdhoc(stored)) + saveMessages(streamSessionId, updated) + return + } + + setMessages((prev) => { + const updated = stripDebugToLast(upsertAdhoc(prev)) const activeSessionId = sessionIdRef.current if (activeSessionId) { saveMessages(activeSessionId, updated) diff --git a/tests/action/response/test_emitted_latch.py b/tests/action/response/test_emitted_latch.py index 44013a8f..5a576f54 100644 --- a/tests/action/response/test_emitted_latch.py +++ b/tests/action/response/test_emitted_latch.py @@ -130,3 +130,131 @@ async def on_message(message): assert [message.message_type for message in seen] == ["stream_chunk", "final"] assert "".join(message.content for message in seen) == "Your order ships Tuesday." assert interaction.response == "Your order ships Tuesday." + + +@pytest.mark.asyncio +async def test_nonstream_during_open_stream_does_not_mint_a_second_identity(): + """A live user accumulator owns the turn. + + Non-stream publish() mints a fresh Object id. If that is allowed while + chunks are in flight, Integral's translator splits on the new id and the + browser shows two assistant bubbles for one answer. + """ + bus = ResponseBus() + interaction = Interaction() + seen = [] + + async def on_message(message): + seen.append(message) + + await bus.subscribe("s1", on_message, receive_chunks=True) + await bus.publish( + session_id="s1", + content="Hello from the stream.", + channel="default", + stream=True, + streaming_complete=False, + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + stream_id = seen[0].id + + await bus.publish( + session_id="s1", + content="Hello from the stream.", + channel="default", + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + + user_ids = {message.id for message in seen if message.category == "user"} + assert user_ids == {stream_id} + assert [message.message_type for message in seen] == ["stream_chunk"] + + +@pytest.mark.asyncio +async def test_nonstream_while_gate_holds_does_not_mint_a_second_identity(): + """First stream call may create the accumulator without latching (empty / + withheld chunk). Non-stream publish must still not invent a second id. + """ + bus = ResponseBus() + interaction = Interaction() + seen = [] + + async def on_message(message): + seen.append(message) + + await bus.subscribe("s1", on_message, receive_chunks=True) + await bus.publish( + session_id="s1", + content="", + channel="default", + stream=True, + streaming_complete=False, + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + assert interaction.has_emitted() is False + assert interaction.id in bus._adhoc_accumulation + + await bus.publish( + session_id="s1", + content="The withheld answer.", + channel="default", + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + + assert seen == [] + assert interaction.has_emitted() is False + + +@pytest.mark.asyncio +async def test_finalize_does_not_mint_a_second_user_identity(): + """Streaming already enqueues message_type=final under acc.message_id. + finalize_interaction must not emit another user-category frame with a + new Object id — Integral treats that as a message-boundary. + """ + bus = ResponseBus() + interaction = Interaction() + seen = [] + + async def on_message(message): + seen.append(message) + + await bus.subscribe("s1", on_message, receive_chunks=True) + await bus.publish( + session_id="s1", + content="Ships Tuesday.", + channel="default", + stream=True, + streaming_complete=False, + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + await bus.publish( + session_id="s1", + content="", + channel="default", + stream=True, + streaming_complete=True, + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + ids_after_stream = {message.id for message in seen} + assert len(ids_after_stream) == 1 + + await bus.finalize_interaction( + interaction_id=interaction.id, + interaction=interaction, + session_id="s1", + channel="default", + ) + + assert {message.id for message in seen} == ids_after_stream From 6f383ee775b62e3f009886b238e116e5979cc1de Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Fri, 18 Sep 2026 06:59:32 -0400 Subject: [PATCH 13/20] fix(response): atomically claim one final per interaction --- CHANGELOG.md | 7 + jvagent/action/response/__init__.py | 2 + jvagent/action/response/response_bus.py | 251 +++++++++++++++--- tests/action/response/conftest.py | 13 + .../response/test_atomic_final_emission.py | 147 ++++++++++ 5 files changed, 385 insertions(+), 35 deletions(-) create mode 100644 tests/action/response/conftest.py create mode 100644 tests/action/response/test_atomic_final_emission.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a364a294..b4ceacc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,13 @@ and this project adheres to [PEP 440](https://peps.python.org/pep-0440/) / ### Fixed +- **Atomic per-interaction egress claim across ResponseBus instances.** A + process-wide `InteractionEgressRecord` (keyed by `interaction_id`) is the + single durable latch for user delivery and `message_type=final`. Rematerialized + `Interaction` objects and a second bus instance can no longer emit a second + Hello. Fresh session → Hello → exactly one persisted response and one + delivered final (`test_atomic_final_emission.py`). + - **One assistant identity per streamed turn.** Non-stream `publish()` while a user accumulator is open is suppressed (it minted a new Object id, which Integral splits into a second bubble). `finalize_interaction` no longer diff --git a/jvagent/action/response/__init__.py b/jvagent/action/response/__init__.py index 799738b5..24036260 100644 --- a/jvagent/action/response/__init__.py +++ b/jvagent/action/response/__init__.py @@ -13,6 +13,7 @@ from jvagent.action.response.response_bus import ( ResponseBus, clear_agent_response_bus, + clear_interaction_egress, get_agent_response_bus, ) from jvagent.action.response.streaming import ( @@ -31,4 +32,5 @@ "stream_messages", "get_agent_response_bus", "clear_agent_response_bus", + "clear_interaction_egress", ] diff --git a/jvagent/action/response/response_bus.py b/jvagent/action/response/response_bus.py index 888e1c76..90f02d40 100644 --- a/jvagent/action/response/response_bus.py +++ b/jvagent/action/response/response_bus.py @@ -80,6 +80,106 @@ def _governed(category: str, transient: bool) -> bool: _agent_bus_registry: Dict[str, "ResponseBus"] = {} +@dataclass +class InteractionEgressRecord: + """Process-wide single-egress claim for one interaction. + + ``Interaction.emitted`` is per-Python-object and ``_message_buffers`` is + per-ResponseBus. A rematerialized Interaction or a second bus instance + cannot see those latches, so Hello can still emit twice. This table is + the shared record every bus consults. + """ + + interaction_id: str + message_id: str = "" + session_id: str = "" + delivered: bool = False + finalized: bool = False + + +_interaction_egress: Dict[str, InteractionEgressRecord] = {} + + +def _interaction_egress_lock() -> asyncio.Lock: + from jvagent.core.async_locks import get_loop_lock + + return get_loop_lock("interaction_egress") + + +def clear_interaction_egress(interaction_id: Optional[str] = None) -> None: + """Test helper: drop one or all process-wide egress claims.""" + if interaction_id is None: + _interaction_egress.clear() + return + _interaction_egress.pop(str(interaction_id), None) + + +async def try_claim_user_egress( + interaction_id: str, + *, + message_id: str, + session_id: str = "", + continue_stream: bool = False, +) -> Tuple[bool, str]: + """Atomically claim the first user delivery for ``interaction_id``. + + Returns ``(allowed, canonical_message_id)``. Stream continuation of the + already-claimed identity is allowed; any other user delivery is not. + """ + key = str(interaction_id or "").strip() + if not key: + return True, message_id + async with _interaction_egress_lock(): + rec = _interaction_egress.get(key) + if rec is None: + _interaction_egress[key] = InteractionEgressRecord( + interaction_id=key, + message_id=message_id, + session_id=session_id, + delivered=True, + ) + return True, message_id + if continue_stream and rec.message_id and rec.message_id == message_id: + rec.delivered = True + return True, rec.message_id + if rec.delivered: + return False, rec.message_id or message_id + rec.delivered = True + rec.message_id = rec.message_id or message_id + rec.session_id = rec.session_id or session_id + return True, rec.message_id + + +async def try_claim_final( + interaction_id: str, + *, + message_id: Optional[str] = None, + session_id: str = "", +) -> Tuple[bool, str]: + """Atomically claim the single ``message_type=final`` for ``interaction_id``.""" + key = str(interaction_id or "").strip() + fallback = message_id or f"o.ResponseMessage.{uuid.uuid4().hex[:24]}" + if not key: + return True, fallback + async with _interaction_egress_lock(): + rec = _interaction_egress.get(key) + if rec is None: + rec = InteractionEgressRecord( + interaction_id=key, + message_id=fallback, + session_id=session_id, + ) + _interaction_egress[key] = rec + if rec.finalized: + return False, rec.message_id or fallback + rec.finalized = True + rec.delivered = True + rec.session_id = rec.session_id or session_id + if not rec.message_id: + rec.message_id = fallback + return True, rec.message_id + + def _agent_bus_lock() -> asyncio.Lock: from jvagent.core.async_locks import get_loop_lock @@ -103,6 +203,7 @@ def clear_agent_response_bus(agent_id: Optional[str] = None) -> None: """Test helper: drop one or all registry entries.""" if agent_id is None: _agent_bus_registry.clear() + clear_interaction_egress() return _agent_bus_registry.pop(str(agent_id), None) @@ -471,6 +572,10 @@ async def _deliver_flush( # accumulator is already open mints a new Object id; Integral splits # bubbles on that id, so it is suppressed even if the first chunk has # not latched yet (gate-held / empty). + # + # The process-wide InteractionEgressRecord is the latch that survives + # a rematerialized Interaction OR a second ResponseBus instance for + # the same agent (fresh-session Hello duplicates). open_user_stream = bool( interaction_id and interaction_id in self._adhoc_accumulation ) @@ -482,16 +587,35 @@ async def _deliver_flush( already_emitted = has_emitted() is True except Exception: already_emitted = False + suppress_second = False + claimed_user_id = "" if ( message_category == "user" and not transient and interaction is not None - and content - and ( - (already_emitted and not active_user_stream) - or (not stream and open_user_stream) - ) + and (not stream and open_user_stream) ): + suppress_second = True + elif ( + message_category == "user" and not transient and content and interaction_id + ): + if already_emitted and not active_user_stream: + suppress_second = True + else: + acc_id = "" + if active_user_stream: + acc_id = self._adhoc_accumulation[interaction_id].message_id + allowed, claimed_user_id = await try_claim_user_egress( + interaction_id, + message_id=acc_id or f"o.ResponseMessage.{uuid.uuid4().hex[:24]}", + session_id=session_id, + continue_stream=active_user_stream, + ) + if not allowed: + suppress_second = True + if hasattr(interaction, "mark_emitted"): + interaction.mark_emitted() + if suppress_second: logger.debug( "response bus: suppressed second user egress for interaction %s", interaction_id or getattr(interaction, "id", ""), @@ -512,19 +636,22 @@ async def _deliver_flush( if not stream: # Non-streaming: immediate filters, adapter, accumulation, one adhoc message - message = ResponseMessage( - session_id=session_id, - user_id=user_id or "", - interaction_id=interaction_id or "", - content=content, - channel=channel, - message_type="adhoc", - metadata=metadata or {}, - timestamp=now, - category=message_category, - thought_type=thought_type, - segment_id=message_segment_id, - ) + message_kwargs: Dict[str, Any] = { + "session_id": session_id, + "user_id": user_id or "", + "interaction_id": interaction_id or "", + "content": content, + "channel": channel, + "message_type": "adhoc", + "metadata": metadata or {}, + "timestamp": now, + "category": message_category, + "thought_type": thought_type, + "segment_id": message_segment_id, + } + if claimed_user_id: + message_kwargs["id"] = claimed_user_id + message = ResponseMessage(**message_kwargs) await _deliver_flush(message, content, transient) await self._enqueue_and_notify(message, session_id) if interaction_id: @@ -566,6 +693,8 @@ async def _deliver_flush( segment_id=message_segment_id, relay_to_adapters=relay_to_adapters, ) + if claimed_user_id and message_category == "user": + acc.message_id = claimed_user_id for chunk in chunk_text_by_lm_tokens(content): acc.chunks.append(chunk) acc.last_activity = time.time() @@ -630,8 +759,12 @@ async def _deliver_flush( thought_type=acc.thought_type, segment_id=acc.segment_id, ) - await self._enqueue_and_notify(final_message, session_id) - self._append_to_message_buffers(interaction_id, final_message) + await self._enqueue_claimed_final( + interaction_id=interaction_id, + session_id=session_id, + final_message=final_message, + category=message_category, + ) if message_category == "thought": self._thought_accumulation.pop( (interaction_id, acc.segment_id or "default"), None @@ -652,6 +785,8 @@ async def _deliver_flush( segment_id=message_segment_id, relay_to_adapters=relay_to_adapters, ) + if claimed_user_id and message_category == "user" and not acc.chunks: + acc.message_id = claimed_user_id # Incremental chunks are released through the accumulator's gate: it # withholds anything a later chunk could still change (a trailing # closer, an unfinished sentence) and returns only settled text. @@ -783,8 +918,12 @@ async def _deliver_flush( thought_type=acc.thought_type, segment_id=acc.segment_id, ) - await self._enqueue_and_notify(final_message, session_id) - self._append_to_message_buffers(interaction_id, final_message) + await self._enqueue_claimed_final( + interaction_id=interaction_id, + session_id=session_id, + final_message=final_message, + category=message_category, + ) if message_category == "thought": self._thought_accumulation.pop( (interaction_id, acc.segment_id or "default"), None @@ -938,6 +1077,42 @@ async def commit_pending_thoughts( ) self._thought_accumulation.pop(key, None) + async def _enqueue_claimed_final( + self, + *, + interaction_id: str, + session_id: str, + final_message: ResponseMessage, + category: str, + ) -> bool: + """Enqueue a stream-complete final if this interaction has not already finalized.""" + if category == "user": + allowed, canon = await try_claim_final( + interaction_id, + message_id=final_message.id, + session_id=session_id, + ) + if not allowed: + return False + if canon and canon != getattr(final_message, "id", ""): + final_message = ResponseMessage( + id=canon, + session_id=final_message.session_id, + user_id=final_message.user_id, + interaction_id=final_message.interaction_id, + content=final_message.content, + channel=final_message.channel, + message_type=final_message.message_type, + metadata=final_message.metadata or {}, + timestamp=final_message.timestamp, + category=final_message.category, + thought_type=final_message.thought_type, + segment_id=final_message.segment_id, + ) + await self._enqueue_and_notify(final_message, session_id) + self._append_to_message_buffers(interaction_id, final_message) + return True + async def _emit_final_signal( self, session_id: str, @@ -948,9 +1123,14 @@ async def _emit_final_signal( message_id: Optional[str] = None, ) -> None: """Internal: enqueue a final ResponseMessage and notify subscribers (no filters/adapters).""" + allowed, canon = await try_claim_final( + interaction_id, message_id=message_id, session_id=session_id + ) + if not allowed: + return now = await self._get_now() final_message = ResponseMessage( - id=message_id or f"o.ResponseMessage.{uuid.uuid4().hex[:24]}", + id=canon, session_id=session_id, user_id=user_id or "", interaction_id=interaction_id, @@ -1166,25 +1346,26 @@ async def finalize_interaction( # Streaming publish() already enqueued message_type=final under # acc.message_id. A second final with a new Object id is a distinct # assistant identity on the wire (Integral splits bubbles on that). + # try_claim_final is process-wide so a second ResponseBus cannot + # emit another one just because this instance's buffers are empty. user_id = getattr(interaction, "user_id", None) if interaction else None last_user_id = None - already_final = False for buffered in self._message_buffers.get(interaction_id) or []: if (getattr(buffered, "category", "user") or "user") != "user": continue if buffered.id: last_user_id = buffered.id - if buffered.message_type == "final": - already_final = True - if not already_final: - await self._emit_final_signal( - session_id=session_id, - channel=channel, - interaction_id=interaction_id, - user_id=user_id, - metadata={}, - message_id=last_user_id, - ) + rec = _interaction_egress.get(interaction_id) + if rec is not None and rec.message_id: + last_user_id = rec.message_id + await self._emit_final_signal( + session_id=session_id, + channel=channel, + interaction_id=interaction_id, + user_id=user_id, + metadata={}, + message_id=last_user_id, + ) # Clean up request-scoped resources (adhoc, message buffers) self._adhoc_accumulation.pop(interaction_id, None) diff --git a/tests/action/response/conftest.py b/tests/action/response/conftest.py new file mode 100644 index 00000000..983929cf --- /dev/null +++ b/tests/action/response/conftest.py @@ -0,0 +1,13 @@ +"""Isolation for process-wide response egress state.""" + +import pytest + +from jvagent.action.response.response_bus import clear_interaction_egress + + +@pytest.fixture(autouse=True) +def _clear_interaction_egress(): + """Keep placeholder interaction IDs from leaking across unit tests.""" + clear_interaction_egress() + yield + clear_interaction_egress() diff --git a/tests/action/response/test_atomic_final_emission.py b/tests/action/response/test_atomic_final_emission.py new file mode 100644 index 00000000..b825cd43 --- /dev/null +++ b/tests/action/response/test_atomic_final_emission.py @@ -0,0 +1,147 @@ +"""Process-wide egress claim: one delivered user final per interaction.""" + +from __future__ import annotations + +import pytest + +from jvagent.action.response.response_bus import ( + ResponseBus, + clear_interaction_egress, +) +from jvagent.memory.interaction import Interaction + + +@pytest.fixture(autouse=True) +def _clear_egress(): + clear_interaction_egress() + yield + clear_interaction_egress() + + +def _clone_interaction(src: Interaction) -> Interaction: + """Fresh Python object with the same id and an unset emitted latch.""" + clone = Interaction() + object.__setattr__(clone, "id", src.id) + clone.emitted = False + clone.response = "" + return clone + + +@pytest.mark.asyncio +async def test_rematerialized_interaction_cannot_emit_a_second_hello(): + bus = ResponseBus() + interaction = Interaction() + seen = [] + + async def on_message(message): + seen.append(message) + + await bus.subscribe("s1", on_message, receive_chunks=True) + await bus.publish( + session_id="s1", + content="Hello! I'm Integral's assistant.", + channel="default", + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + twin = _clone_interaction(interaction) + assert twin.has_emitted() is False + await bus.publish( + session_id="s1", + content="Hello! I'm Integral's assistant.", + channel="default", + interaction=twin, + interaction_id=interaction.id, + category="user", + ) + + user_text = [ + m.content + for m in seen + if m.category == "user" and m.message_type != "final" and m.content + ] + assert user_text == ["Hello! I'm Integral's assistant."] + + +@pytest.mark.asyncio +async def test_two_buses_fresh_session_hello_one_final(): + """Fresh session Hello: two ResponseBus instances, one interaction. + + Exactly one persisted response and one delivered final across both + subscriber lists. + """ + bus_a = ResponseBus() + bus_b = ResponseBus() + interaction = Interaction() + seen_a = [] + seen_b = [] + + async def on_a(message): + seen_a.append(message) + + async def on_b(message): + seen_b.append(message) + + await bus_a.subscribe("s1", on_a, receive_chunks=True) + await bus_b.subscribe("s1", on_b, receive_chunks=True) + + await bus_a.publish( + session_id="s1", + content="Hello! How can I help?", + channel="default", + stream=True, + streaming_complete=False, + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + await bus_a.publish( + session_id="s1", + content="", + channel="default", + stream=True, + streaming_complete=True, + interaction=interaction, + interaction_id=interaction.id, + category="user", + ) + twin = _clone_interaction(interaction) + await bus_b.publish( + session_id="s1", + content="Hello! How can I help?", + channel="default", + interaction=twin, + interaction_id=interaction.id, + category="user", + ) + await bus_a.finalize_interaction( + interaction_id=interaction.id, + interaction=interaction, + session_id="s1", + channel="default", + ) + await bus_b.finalize_interaction( + interaction_id=interaction.id, + interaction=twin, + session_id="s1", + channel="default", + ) + + combined = seen_a + seen_b + user_text = [ + m.content + for m in combined + if m.category == "user" and m.message_type == "stream_chunk" and m.content + ] + finals = [m for m in combined if m.category == "user" and m.message_type == "final"] + adhoc = [ + m.content + for m in combined + if m.category == "user" and m.message_type == "adhoc" and m.content + ] + assert user_text == ["Hello! How can I help?"] + assert adhoc == [] + assert len(finals) == 1 + assert interaction.response == "Hello! How can I help?" + assert finals[0].id == seen_a[0].id From 9084d4a8f1376c8121497d55bb7c0dfd407df1a7 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Fri, 18 Sep 2026 07:24:11 -0400 Subject: [PATCH 14/20] test(response): isolate process-wide egress claims across tests Placeholder interaction ids like i1 leaked between cases and suppressed legitimate publishes in the governance suite. --- .../orchestrator/test_egress_governance.py | 18 +++++++++--------- tests/conftest.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/tests/action/orchestrator/test_egress_governance.py b/tests/action/orchestrator/test_egress_governance.py index 2a794823..82a47abf 100644 --- a/tests/action/orchestrator/test_egress_governance.py +++ b/tests/action/orchestrator/test_egress_governance.py @@ -73,10 +73,10 @@ async def test_streamed_and_non_streamed_replies_are_governed_identically(): "Hello! How can I assist you today?" ) - async def _run(stream: bool) -> str: + async def _run(stream: bool, interaction_id: str) -> str: bus = ResponseBus() interaction = MagicMock() - interaction.id = "i1" + interaction.id = interaction_id interaction.response = None interaction.parameters = [] interaction.set_response = MagicMock(return_value=True) @@ -88,7 +88,7 @@ async def _run(stream: bool) -> str: content=ch, channel="default", stream=True, - interaction_id="i1", + interaction_id=interaction_id, interaction=interaction, user_id="u1", streaming_complete=False, @@ -98,14 +98,14 @@ async def _run(stream: bool) -> str: content="", channel="default", stream=True, - interaction_id="i1", + interaction_id=interaction_id, interaction=interaction, user_id="u1", streaming_complete=True, ) return "".join( m.content - for m in bus._message_buffers.get("i1", []) + for m in bus._message_buffers.get(interaction_id, []) if m.message_type == "stream_chunk" ) await bus.publish( @@ -113,16 +113,16 @@ async def _run(stream: bool) -> str: content=text, channel="default", stream=False, - interaction_id="i1", + interaction_id=interaction_id, interaction=interaction, user_id="u1", ) return "".join( - m.content for m in bus._message_buffers.get("i1", []) if m.content + m.content for m in bus._message_buffers.get(interaction_id, []) if m.content ) - streamed = await _run(True) - plain = await _run(False) + streamed = await _run(True, "i-stream") + plain = await _run(False, "i-plain") assert streamed == plain == vet_egress(text) assert streamed.count("Hello") == 1 diff --git a/tests/conftest.py b/tests/conftest.py index 22bee9f3..d2725655 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,6 +19,21 @@ def _clear_jvspatial_load_env_cache(): yield +@pytest.fixture(autouse=True) +def _clear_interaction_egress(): + """Process-wide egress claims must not leak across tests. + + ``InteractionEgressRecord`` is keyed by interaction_id. Many unit tests + reuse placeholders like ``i1``, so a claimed Hello in one file would + suppress user publish in another. + """ + from jvagent.action.response.response_bus import clear_interaction_egress + + clear_interaction_egress() + yield + clear_interaction_egress() + + @pytest.fixture(autouse=True) def _ensure_default_graph_context(tmp_path_factory, monkeypatch): """Bind a per-test default GraphContext for any test that touches jvspatial. From d2eec8415a064df3b4fd2daecd2d975dd90dbaf1 Mon Sep 17 00:00:00 2001 From: Tharick Jairam Date: Thu, 17 Sep 2026 12:58:38 -0400 Subject: [PATCH 15/20] feat(jvchat): replay debug ticks with first-tick tool schemas Co-authored-by: Cursor --- jvagent/action/model/base.py | 62 +- jvagent/action/model/language/base.py | 8 + jvchat/src/components/DebugInteractions.tsx | 562 ++++++++++-------- jvchat/src/config/api.ts | 4 +- jvchat/src/lib/debugReplay.test.ts | 243 ++++++++ jvchat/src/lib/debugReplay.ts | 365 ++++++++++++ .../test_observability_tool_definitions.py | 106 +++- 7 files changed, 1085 insertions(+), 265 deletions(-) create mode 100644 jvchat/src/lib/debugReplay.test.ts create mode 100644 jvchat/src/lib/debugReplay.ts diff --git a/jvagent/action/model/base.py b/jvagent/action/model/base.py index 0be26a17..69c3c9df 100644 --- a/jvagent/action/model/base.py +++ b/jvagent/action/model/base.py @@ -42,6 +42,37 @@ def _tool_definition_names(tools: Any) -> List[str]: return names +def _tool_names_fingerprint(names: Any) -> tuple: + if not isinstance(names, list): + return () + return tuple(n for n in names if isinstance(n, str) and n) + + +def _first_tick_for_tool_surface(interaction: Any, tool_names: List[str]) -> bool: + """True when this interaction has not yet stored ``tools`` for this name set. + + Debug replay needs the schemas once per tool surface. Later ticks of the + same names keep ``tool_names`` only so an agentic turn does not multiply + the same kilobytes by the tick count. + """ + names = _tool_names_fingerprint(tool_names) + if not names: + return False + events = getattr(interaction, "observability_metrics", None) or [] + for event in events: + if not isinstance(event, dict): + continue + event_type = event.get("event_type") + if event_type not in ("model_call", None, ""): + continue + ev_data = event.get("data") + if not isinstance(ev_data, dict) or "tools" not in ev_data: + continue + if _tool_names_fingerprint(ev_data.get("tool_names")) == names: + return False + return True + + T = TypeVar("T") @@ -158,12 +189,12 @@ class BaseModelAction(Action, ABC): telemetry_tool_definitions: bool = attribute( default=False, description=( - "Store the full tool/function definitions sent with each call on " - "the model_call observability event, so a debug UI can replay the " - "request exactly. Off by default: the schemas are identical on " - "every tick of a turn, so an agentic loop persists the same " - "kilobytes once per tick per interaction. Turn on per model action " - "while debugging. Tool NAMES are always recorded and cost nothing." + "Store the full tool/function definitions on every model_call " + "observability event. Off by default: the first tick of each unique " + "tool_names surface still records the schemas so a debug UI can " + "replay the request; later ticks of the same surface keep names " + "only. Turn this on per model action to persist schemas on every " + "tick. Tool NAMES are always recorded and cost nothing." ), ) @@ -628,9 +659,26 @@ async def _emit_observability( result_tools = getattr(result, "tools", None) if result_tools: data["tool_names"] = _tool_definition_names(result_tools) - if self.telemetry_tool_definitions: + if self.telemetry_tool_definitions or _first_tick_for_tool_surface( + interaction, data["tool_names"] + ): data["tools"] = result_tools + temperature = getattr(result, "temperature", None) + if isinstance(temperature, (int, float)) and not isinstance( + temperature, bool + ): + data["temperature"] = temperature + max_tokens = getattr(result, "max_tokens", None) + if isinstance(max_tokens, int) and not isinstance(max_tokens, bool): + data["max_tokens"] = max_tokens + tool_choice = getattr(result, "tool_choice", None) + if isinstance(tool_choice, (str, dict)): + data["tool_choice"] = tool_choice + parallel = getattr(result, "parallel_tool_calls", None) + if isinstance(parallel, bool): + data["parallel_tool_calls"] = parallel + # Build event and append directly to interaction event = { "event_type": event_type, diff --git a/jvagent/action/model/language/base.py b/jvagent/action/model/language/base.py index 95ca1f2f..81c22fe7 100644 --- a/jvagent/action/model/language/base.py +++ b/jvagent/action/model/language/base.py @@ -212,6 +212,10 @@ def __init__( self.thinking_tokens = thinking_tokens self.request_model = request_model self.tools = tools + self.temperature = None + self.max_tokens = None + self.tool_choice = None + self.parallel_tool_calls = None self._thinking_queue: Optional[asyncio.Queue] = thinking_queue self._thinking_closed: bool = False @@ -897,6 +901,10 @@ async def stream_with_retry() -> AsyncGenerator[str, None]: # (e.g. LiteLLM/OpenAI returns gpt-4.1-2025-04-14 for openai/gpt-4.1). result.request_model = kwargs.get("model") or getattr(self, "model", None) or "" result.tools = tools + result.temperature = kwargs.get("temperature") + result.max_tokens = kwargs.get("max_tokens") + result.tool_choice = kwargs.get("tool_choice") + result.parallel_tool_calls = kwargs.get("parallel_tool_calls") # Store calling_action_name in result for observability if calling_action_name: diff --git a/jvchat/src/components/DebugInteractions.tsx b/jvchat/src/components/DebugInteractions.tsx index b0df72b7..d5dfb8c5 100644 --- a/jvchat/src/components/DebugInteractions.tsx +++ b/jvchat/src/components/DebugInteractions.tsx @@ -28,6 +28,17 @@ import { toolCallsForMetric, type DebugToolCall, } from "../lib/debugToolCalls"; +import { + buildExportV2, + buildQueryPayload, + buildReplaySnapshot, + formatCopyPrompt, + formatImproveSystemPrompt, + parseImportFile, + unwrapQueryActionResponse, + type DebugExportSelection, + type ReplaySnapshot, +} from "../lib/debugReplay"; /** Code / text fields: black in dark theme, off-grey in light theme */ function debugCodePanelClass(isDark: boolean) { @@ -36,23 +47,6 @@ function debugCodePanelClass(isDark: boolean) { : "bg-zinc-100 border border-zinc-300 text-zinc-900 placeholder-zinc-600"; } -function parseJsonArray( - text: string, - label: string, -): { ok: true; value: unknown[] } | { ok: false; error: string } { - const trimmed = text.trim(); - if (!trimmed) return { ok: true, value: [] }; - try { - const parsed = JSON.parse(trimmed); - if (!Array.isArray(parsed)) { - return { ok: false, error: `Cannot retest: ${label} must be an array.` }; - } - return { ok: true, value: parsed }; - } catch { - return { ok: false, error: `Cannot retest: ${label} is not valid JSON.` }; - } -} - /** * Build a human-readable label for an observability metric. * @@ -244,6 +238,8 @@ export function DebugInteractions({ /** Editable tool definitions sent on retest. */ const [toolsText, setToolsText] = useState("[]"); const replaySyncKeyRef = useRef(""); + const pendingImportSelectionRef = useRef(null); + const skipEditorSyncRef = useRef(false); const [improveInstruction, setImproveInstruction] = useState(""); const [improveModel, setImproveModel] = useState("gpt-4o"); const [improving, setImproving] = useState(false); @@ -284,6 +280,7 @@ export function DebugInteractions({ // → data.provider). Falls back to the first available provider if the // recorded one isn't installed on this agent. useEffect(() => { + if (skipEditorSyncRef.current) return; if (!selectedInteraction) return; if (selectedInteraction.event_type !== "model_call") return; const metricProvider = selectedInteraction.data?.provider; @@ -357,9 +354,12 @@ export function DebugInteractions({ const pd = metric.data || {}; // Get history from metric data or parent's conversation history const history = pd.history || parent.conversationHistory || []; + const metricKey = `${parent.id}:${metricIdx}:${metric.timestamp ?? ""}`; setSelectedInteraction({ - id: metric.id, + id: metric.id || metricKey, + parentId: parent.id, + metricIndex: metricIdx, // ADR-0009 / observability: every metric carries event_type + // data. Surface both so the inspector can render type-specific // payloads (helm_shift, model_call, etc.) rather than treating @@ -380,6 +380,15 @@ export function DebugInteractions({ tool_names: Array.isArray(pd.tool_names) ? pd.tool_names : [], tool_calls: Array.isArray(pd.tool_calls) ? pd.tool_calls : [], finish_reason: pd.finish_reason || "", + called_by: pd.called_by || "", + usage: pd.usage || null, + temperature: typeof pd.temperature === "number" ? pd.temperature : undefined, + max_tokens: typeof pd.max_tokens === "number" ? pd.max_tokens : undefined, + tool_choice: pd.tool_choice, + parallel_tool_calls: + typeof pd.parallel_tool_calls === "boolean" + ? pd.parallel_tool_calls + : undefined, }, }); setTestResult(null); @@ -703,15 +712,18 @@ export function DebugInteractions({ } const currentParent = selectedParentIndex != null ? effectiveParents[selectedParentIndex] : null; - const metricId = selectedInteraction?.id; - const parentWithMetric = metricId - ? effectiveParents.find((p) => - p.metrics?.some((m: any) => m.id === metricId), - ) + const parentId = selectedInteraction?.parentId; + const storedMetricIdx = selectedInteraction?.metricIndex; + const parentWithMetric = parentId + ? effectiveParents.find((p) => p.id === parentId) : null; const metricIdx = - parentWithMetric?.metrics?.findIndex((m: any) => m.id === metricId) ?? -1; - if (parentWithMetric && metricIdx >= 0) { + typeof storedMetricIdx === "number" ? storedMetricIdx : -1; + if ( + parentWithMetric && + metricIdx >= 0 && + metricIdx < (parentWithMetric.metrics?.length || 0) + ) { const newParentIdx = effectiveParents.indexOf(parentWithMetric); if (newParentIdx !== selectedParentIndex || selectedMetricIndex !== metricIdx) { selectInteraction(newParentIdx, metricIdx, effectiveParents); @@ -733,6 +745,10 @@ export function DebugInteractions({ }, [selectedUserId, pageSize, refreshInteractionLogsPage1]); useEffect(() => { + if (skipEditorSyncRef.current) { + setShowHistory(true); + return; + } const historyData = selectedInteraction?.data?.history; const hasHistory = Array.isArray(historyData); @@ -770,6 +786,7 @@ export function DebugInteractions({ })(), ].join(":"); useEffect(() => { + if (skipEditorSyncRef.current) return; if (replaySyncKey === replaySyncKeyRef.current) return; replaySyncKeyRef.current = replaySyncKey; setReplayText( @@ -798,6 +815,103 @@ export function DebugInteractions({ adjustHeight(improveResultRef.current); }, [improveResult, loading]); + useEffect(() => { + const sel = pendingImportSelectionRef.current; + if (sel && selectedInteraction) { + pendingImportSelectionRef.current = null; + replaySyncKeyRef.current = replaySyncKey; + let parsedHistory = Array.isArray(selectedInteraction.data?.history) + ? selectedInteraction.data.history + : []; + try { + const parsed = sel.historyText.trim() + ? JSON.parse(sel.historyText) + : []; + if (Array.isArray(parsed)) parsedHistory = parsed; + } catch { + // Keep metric history when the exported editor JSON is invalid. + } + setHistoryText(sel.historyText); + setReplayText(sel.replayText); + setToolsText(sel.toolsText); + if (sel.replayModel) setReplayModel(sel.replayModel); + if (sel.provider && modelActions[sel.provider]) { + setSelectedProvider(sel.provider); + setModelAction(modelActions[sel.provider]); + } + setTestResult(sel.testResult ?? null); + setSelectedInteraction((si: any) => + si + ? { + ...si, + data: { + ...si.data, + user_prompt: sel.user_prompt, + system_prompt: sel.system_prompt, + history: parsedHistory, + }, + } + : si, + ); + return; + } + if (skipEditorSyncRef.current) { + skipEditorSyncRef.current = false; + } + }, [selectedInteraction, replaySyncKey, modelActions]); + + const liveSnapshot = useCallback((): + | { ok: true; snapshot: ReplaySnapshot } + | { ok: false; error: string } => { + if (!selectedInteraction) { + return { ok: false, error: "Cannot retest: no interaction selected." }; + } + return buildReplaySnapshot({ + user: selectedInteraction.data.user_prompt || "", + system: selectedInteraction.data.system_prompt || "", + historyText, + replayText, + toolsText, + model: (replayModel || "").trim(), + provider: selectedProvider || "", + response: selectedInteraction.data.response || "", + toolCalls: selectedInteraction.data.tool_calls, + finishReason: selectedInteraction.data.finish_reason || "", + calledBy: selectedInteraction.data.called_by, + usage: selectedInteraction.data.usage, + toolSource: retestTools.source, + temperature: selectedInteraction.data.temperature, + maxTokens: selectedInteraction.data.max_tokens, + toolChoice: selectedInteraction.data.tool_choice, + parallelToolCalls: selectedInteraction.data.parallel_tool_calls, + }); + }, [ + selectedInteraction, + historyText, + replayText, + toolsText, + replayModel, + selectedProvider, + retestTools.source, + ]); + + const copyLivePrompt = async (withImprove: boolean) => { + const built = liveSnapshot(); + if (!built.ok) { + setError(built.error); + return; + } + const text = formatCopyPrompt( + built.snapshot, + withImprove ? improveInstruction : undefined, + ); + try { + await navigator.clipboard.writeText(text); + } catch { + setError("Could not copy to clipboard."); + } + }; + const handleTest = async () => { if (!selectedInteraction) return; @@ -813,8 +927,16 @@ export function DebugInteractions({ return; } - const prompt = (selectedInteraction.data.user_prompt || "").trim(); - if (!prompt) { + const built = liveSnapshot(); + if (!built.ok) { + preserveScroll(() => + setTestResult({ success: false, error: built.error }), + ); + return; + } + const snapshot = built.snapshot; + + if (!(snapshot.user || "").trim()) { preserveScroll(() => setTestResult({ success: false, @@ -828,15 +950,7 @@ export function DebugInteractions({ const finishReason = selectedInteraction.data.finish_reason || ""; const needsTools = originalToolCalls.length > 0 || finishReason === "tool_calls"; - - const parsedTools = parseJsonArray(toolsText, "Tools (JSON)"); - if (!parsedTools.ok) { - preserveScroll(() => - setTestResult({ success: false, error: parsedTools.error }), - ); - return; - } - if (needsTools && parsedTools.value.length === 0) { + if (needsTools && snapshot.tools.length === 0) { preserveScroll(() => setTestResult({ success: false, @@ -846,10 +960,8 @@ export function DebugInteractions({ ); return; } - const tools = parsedTools.value; - const modelToSend = (replayModel || "").trim(); - if (!modelToSend) { + if (!snapshot.model) { preserveScroll(() => setTestResult({ success: false, @@ -860,68 +972,21 @@ export function DebugInteractions({ return; } - const parsedHistory = parseJsonArray(historyText, "History (JSON)"); - if (!parsedHistory.ok) { - preserveScroll(() => - setTestResult({ success: false, error: parsedHistory.error }), - ); - return; - } - const history = parsedHistory.value; - - const parsedReplay = parseJsonArray(replayText, "This-turn tool replay (JSON)"); - if (!parsedReplay.ok) { - preserveScroll(() => - setTestResult({ success: false, error: parsedReplay.error }), - ); - return; - } - const replay = parsedReplay.value; - preserveScroll(() => { setTesting(true); setTestResult(null); }); - const includeTools = tools.length > 0 && (needsTools || retestTools.source === "recorded"); - try { - const payload: Record = { - model: modelToSend, - provider: selectedProvider || undefined, - }; - if (replay.length > 0 || includeTools) { - const messages: Record[] = []; - const system = selectedInteraction.data.system_prompt; - if (system) { - messages.push({ role: "system", content: system }); - } - if (history.length > 0) { - messages.push(...(history as Record[])); - } - messages.push({ - role: "user", - content: selectedInteraction.data.user_prompt, - }); - messages.push(...(replay as Record[])); - payload.messages = messages; - payload.tool_choice = "auto"; - payload.parallel_tool_calls = false; - if (includeTools) { - payload.tools = tools; - } - } else { - payload.prompt = selectedInteraction.data.user_prompt; - payload.system = selectedInteraction.data.system_prompt; - payload.history = history; - } - - const data = await apiClient.queryAction(actionId, payload); + const payload = buildQueryPayload(snapshot); + const data = unwrapQueryActionResponse( + await apiClient.queryAction(actionId, payload), + ); preserveScroll(() => setTestResult({ success: true, response: data.response, - data: data, + data, }), ); } catch (error: any) { @@ -939,6 +1004,12 @@ export function DebugInteractions({ const handleImprovePrompt = async () => { if (!selectedInteraction || !modelAction || !improveInstruction) return; + const built = liveSnapshot(); + if (!built.ok) { + preserveScroll(() => setImproveResult(`Error: ${built.error}`)); + return; + } + preserveScroll(() => { setImproving(true); setImproveResult(""); @@ -946,26 +1017,8 @@ export function DebugInteractions({ try { const improvePayload = { - prompt: `Given the following context, improve the prompts based on the instruction. - -User Prompt: -${selectedInteraction.data.user_prompt} - -System Prompt: -${selectedInteraction.data.system_prompt} - -Conversation History: -${JSON.stringify(selectedInteraction.data.history || [], null, 2)} - -RESULT: -${selectedInteraction.data.response} - -Improvement Instruction: -${improveInstruction} - -Provide improvement instruction on how to improve the prompt. Return a raw markdown.`, - system: - "You are a prompt engineering expert. Analyze the given prompts and improve them based on the instruction.", + prompt: formatCopyPrompt(built.snapshot, improveInstruction), + system: formatImproveSystemPrompt(), model: improveModel, provider: improveProvider || undefined, history: [], @@ -973,8 +1026,12 @@ Provide improvement instruction on how to improve the prompt. Return a raw markd const improveActionId = modelActions[improveProvider]?.id || modelAction?.id; - const data = await apiClient.queryAction(improveActionId, improvePayload); - preserveScroll(() => setImproveResult(data.response || "")); + const data = unwrapQueryActionResponse( + await apiClient.queryAction(improveActionId, improvePayload), + ); + preserveScroll(() => + setImproveResult(typeof data.response === "string" ? data.response : ""), + ); } catch (error: any) { preserveScroll(() => setImproveResult(`Error: ${error.message}`)); } finally { @@ -1046,16 +1103,25 @@ Provide improvement instruction on how to improve the prompt. Return a raw markd const handleExport = () => { if (parentInteractions.length === 0) return; - const dataToExport = { - parentInteractions: parentInteractions, - pagination: pagination, - selectedParentIndex: selectedParentIndex, - selectedMetricIndex: selectedMetricIndex, - metadata: { - exportedAt: new Date().toISOString(), - agentId: targetAgentId, - }, - }; + const dataToExport = buildExportV2({ + parentInteractions, + pagination, + selectedParentIndex, + selectedMetricIndex, + selection: selectedInteraction + ? { + user_prompt: selectedInteraction.data?.user_prompt || "", + system_prompt: selectedInteraction.data?.system_prompt || "", + historyText, + replayText, + toolsText, + replayModel, + provider: selectedProvider, + testResult, + } + : null, + agentId: targetAgentId, + }); const blob = new Blob([JSON.stringify(dataToExport, null, 2)], { type: "application/json", @@ -1078,45 +1144,39 @@ Provide improvement instruction on how to improve the prompt. Return a raw markd reader.onload = (event) => { try { const content = event.target?.result as string; - const parsed = JSON.parse(content); + const imported = parseImportFile(JSON.parse(content)); - // Check if it's the new format (full list) or legacy format (single interaction) - if ( - parsed.parentInteractions && - Array.isArray(parsed.parentInteractions) - ) { + if (imported.kind === "invalid") { + setError(imported.error); + e.target.value = ""; + return; + } + + if (imported.kind === "v2" || imported.kind === "v1") { + if (imported.kind === "v2" && imported.selection) { + skipEditorSyncRef.current = true; + pendingImportSelectionRef.current = imported.selection; + } preserveScroll(() => { - setParentInteractions(parsed.parentInteractions); - setPagination(parsed.pagination || null); - - const pIdx = - typeof parsed.selectedParentIndex === "number" - ? parsed.selectedParentIndex - : 0; - const mIdx = - typeof parsed.selectedMetricIndex === "number" - ? parsed.selectedMetricIndex - : 0; - - if (parsed.parentInteractions.length > 0) { - selectInteraction(pIdx, mIdx, parsed.parentInteractions); + setParentInteractions(imported.parentInteractions); + setPagination((imported.pagination as typeof pagination) || null); + if (imported.parentInteractions.length > 0) { + selectInteraction( + imported.selectedParentIndex, + imported.selectedMetricIndex, + imported.parentInteractions, + ); } }); } else { - // Legacy format or single interaction export - const interactionData = parsed.interaction || parsed; - const testResultData = parsed.testResult || null; - - if (interactionData?.data) { - preserveScroll(() => { + preserveScroll(() => { + if (parentInteractions.length === 0) { setSelectedParentIndex(null); setSelectedMetricIndex(null); - setSelectedInteraction(interactionData); - setTestResult(testResultData); - }); - } else { - setError("Invalid import file format"); - } + } + setSelectedInteraction(imported.interaction); + setTestResult(imported.testResult); + }); } } catch (err) { console.error("Import failed", err); @@ -1186,7 +1246,7 @@ Provide improvement instruction on how to improve the prompt. Return a raw markd +
+ + +
LM only — tools proposed, not executed - {retestTools.source === "stub" && - ((selectedInteraction.data.tool_calls || []).length > - 0 || - selectedInteraction.data.finish_reason === - "tool_calls") && ( + {retestTools.source === "stub" && ( )} - {/* Test Result — show tool_calls when present; else response text. */} + {/* Test Result — show response and tool_calls when present. */} {testResult && (
{ const toolCalls = testResult.data?.tool_calls ?? []; - if (Array.isArray(toolCalls) && toolCalls.length > 0) { - return ( - - ); - } const tr = testResult.data?.response ?? testResult.response ?? ""; - const tp = tryParseJsonDisplay(tr); - if (tp != null) { + const hasTools = + Array.isArray(toolCalls) && toolCalls.length > 0; + const tp = + typeof tr === "string" && tr + ? tryParseJsonDisplay(tr) + : null; + if (!hasTools && !tr) { return ( - + 
+                                (empty response, no tool_calls)
+
); } return ( - 
-                              {tr || "(empty response, no tool_calls)"}
-
+ <> + {hasTools && ( +
+

+ tool_calls +

+ +
+ )} + {!!tr && + (tp != null ? ( +
+

+ response +

+ +
+ ) : ( + 
+                                    {tr}
+
+ ))} + ); })() ) : ( @@ -1972,30 +2086,10 @@ Provide improvement instruction on how to improve the prompt. Return a raw markd