From 973d8ba84c8d5cbcf9b341632d9d7a544c10bbff Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 11:32:52 +0800 Subject: [PATCH 01/82] fix: restore source files after failed candidate adoption --- opencollab/adapters/_candidate_adoption.py | 97 +++++ opencollab/adapters/_candidate_increment.py | 44 ++- opencollab/application/workflow_candidates.py | 3 + .../duo/_candidate_records.py | 3 + .../test_candidate_adoption_recovery.py | 330 ++++++++++++++++++ 5 files changed, 464 insertions(+), 13 deletions(-) create mode 100644 opencollab/adapters/_candidate_adoption.py create mode 100644 tests/workflows/test_candidate_adoption_recovery.py diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py new file mode 100644 index 00000000..3e631234 --- /dev/null +++ b/opencollab/adapters/_candidate_adoption.py @@ -0,0 +1,97 @@ +"""Restore physical working files after a failed candidate publication.""" + +from __future__ import annotations + +import posixpath +import shlex +from collections.abc import AsyncIterator, Sequence +from contextlib import asynccontextmanager +from typing import Any + +from opencollab.adapters._git_patch_bytes import _complete +from opencollab.application.async_timeout import await_owned_operation +from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError + + +@asynccontextmanager +async def _adoption_transaction( + environment: Any, workspace: str, paths: Sequence[str], *, repository: bool, +) -> AsyncIterator[None]: + for path in paths: + if ( + not path or "\0" in path or posixpath.normpath(path) != path or posixpath.isabs(path) + or path in {".", ".."} or path.startswith("../") + or any(part.casefold() == ".git" for part in path.split("/")) + ): + raise RuntimeError(f"invalid path in candidate patch: {path!r}") + # Own a missing or non-directory ancestor too, so rollback also reverses + # directory creation and file/directory replacements without following links. + inspected = _complete(await environment.exec_cmd( + f"cd -- {shlex.quote(workspace)} && " + f"for path in {shlex.join(paths)}; do root=$path; parent=${{path%/*}}; " + 'while [ "$parent" != "$path" ]; do ' + 'if [ ! -d "$parent" ] || [ -L "$parent" ]; then root=$parent; fi; ' + 'path=$parent; parent=${path%/*}; done; printf "%s\\0" "$root"; done', + timeout=120, + ), "candidate adoption path inspection") + roots: list[str] = [] + for path in sorted(set(inspected.split("\0")) - {""}): + if not any(path.startswith(root + "/") for root in roots): + roots.append(path) + storage = '"${TMPDIR:-/tmp}"' + if repository: + storage = shlex.quote(_complete(await environment.exec_cmd( + f"git -C {shlex.quote(workspace)} rev-parse --absolute-git-dir", timeout=30, + ), "candidate recovery storage").removesuffix("\n")) + backup = _complete(await environment.exec_cmd( + f'mktemp -d {storage}/candidate-recovery-XXXXXXXX', timeout=30, + ), "candidate recovery storage").removesuffix("\n") + backup_quoted = shlex.quote(backup) + keep_backup = False + try: + copies = [] + restores = [] + for index, path in enumerate(roots): + source = shlex.quote(posixpath.join(workspace, path)) + saved = shlex.quote(posixpath.join(backup, str(index))) + copies.append( + f"printf '%s' {source} > {saved}.path || exit; " + f"if [ -e {source} ] || [ -L {source} ]; then " + f"cp -pPR -- {source} {saved} || exit; fi" + ) + restores.append( + f"if [ -e {saved} ] || [ -L {saved} ]; then " + f"if rm -rf -- {source} && mv -- {saved} {source} " + f"&& {{ [ -e {source} ] || [ -L {source} ]; }}; then :; else failed=1; fi; " + f"elif rm -rf -- {source} && [ ! -e {source} ] && [ ! -L {source} ]; " + "then :; else failed=1; fi" + ) + _complete(await environment.exec_cmd( + "; ".join(copies), timeout=900, + ), "candidate recovery snapshot") + try: + yield + except BaseException as failure: + try: + _complete(await await_owned_operation(environment.exec_cmd( + "failed=0; " + "; ".join(restores) + '; [ "$failed" -eq 0 ]', + timeout=900, + )), "candidate adoption recovery") + except BaseException as recovery: + keep_backup = True + revoke = getattr(environment, "revoke", None) + if callable(revoke): + revoke() + raise CandidateAdoptionRecoveryError( + f"candidate adoption failed ({failure}); recovery failed ({recovery}); " + f"original files retained in {backup}" + ) from recovery + raise + finally: + if not keep_backup: + try: + await await_owned_operation(environment.exec_cmd( + f"rm -rf -- {backup_quoted}", timeout=120, + )) + except Exception: + pass diff --git a/opencollab/adapters/_candidate_increment.py b/opencollab/adapters/_candidate_increment.py index 4272f226..51ebe361 100644 --- a/opencollab/adapters/_candidate_increment.py +++ b/opencollab/adapters/_candidate_increment.py @@ -5,9 +5,11 @@ import posixpath import shlex from collections.abc import Sequence +from contextlib import AsyncExitStack from typing import Any -from opencollab.adapters._candidate_physical import _apply_physical +from opencollab.adapters._candidate_adoption import _adoption_transaction +from opencollab.adapters._candidate_physical import _physical_git from opencollab.adapters._git_patch_bytes import _complete, _patch_temporary from opencollab.adapters.git_patch import _git_command_and_config_guard from opencollab.patches import patch_paths @@ -51,15 +53,31 @@ async def _apply_increment( guard = "" if strict and repository: git, guard = _git_command_and_config_guard() - if physical_paths and repository: - if guard: - _complete(await environment.exec_cmd( - f"cd -- {shlex.quote(workspace)} && {guard}:", timeout=120, - ), "physical candidate configuration inspection") - await _apply_physical(environment, workspace, candidate_file, physical_paths, strict=strict) - return - _complete(await environment.exec_cmd( - f"cd -- {shlex.quote(workspace)} && {guard}" - f"{git} apply --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}", - timeout=120, - ), "candidate adoption") + async with AsyncExitStack() as resources: + command = f"cd -- {shlex.quote(workspace)} && {guard}{git}" + operation = "candidate adoption" + if physical_paths and repository: + if guard: + _complete(await environment.exec_cmd( + f"cd -- {shlex.quote(workspace)} && {guard}:", timeout=120, + ), "physical candidate configuration inspection") + command, _ = await resources.enter_async_context(_physical_git( + environment, workspace, physical_paths, strict=strict, + )) + operation = "physical candidate adoption" + # Let Git reject unsafe path aliases before a recursive backup can + # encounter repository metadata. Publication can still fail later. + _complete(await environment.exec_cmd( + f"{command} apply --check --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}", + timeout=120, + ), operation) + async with _adoption_transaction( + environment, workspace, sorted(candidate_paths), repository=repository, + ): + try: + _complete(await environment.exec_cmd( + f"{command} apply --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}", + timeout=120, + ), operation) + finally: + await resources.aclose() diff --git a/opencollab/application/workflow_candidates.py b/opencollab/application/workflow_candidates.py index d54b5f3e..459cc3bd 100644 --- a/opencollab/application/workflow_candidates.py +++ b/opencollab/application/workflow_candidates.py @@ -74,6 +74,9 @@ class CandidateCaptureError(RuntimeError): class CandidateWorkspaceTrackingError(RuntimeError): """The source worktree changed while a candidate was executing.""" +class CandidateAdoptionRecoveryError(RuntimeError): + """Candidate publication failed and the source still needs recovery.""" + def _candidate_worktree_location(lease: Any) -> str: """Include an optional location in candidate retention diagnostics.""" workspace = getattr(lease, "candidate_workspace", None) diff --git a/opencollab/builtin_workflows/duo/_candidate_records.py b/opencollab/builtin_workflows/duo/_candidate_records.py index b08a83a5..bf7d0903 100644 --- a/opencollab/builtin_workflows/duo/_candidate_records.py +++ b/opencollab/builtin_workflows/duo/_candidate_records.py @@ -5,6 +5,7 @@ import copy from typing import Any +from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError from opencollab.workflows import CandidateRun MAX_PUBLIC_COMMAND_BYTES = 4_000 @@ -98,6 +99,8 @@ async def _adopt( attempts.append(label) try: await ctx.adopt_candidate(candidate, preserve_paths=preserve_paths) + except CandidateAdoptionRecoveryError: + raise except Exception as exc: # noqa: BLE001 await ctx.log(f"Duo adoption failed ({label}) after {type(exc).__name__}") continue diff --git a/tests/workflows/test_candidate_adoption_recovery.py b/tests/workflows/test_candidate_adoption_recovery.py new file mode 100644 index 00000000..dc9ce60f --- /dev/null +++ b/tests/workflows/test_candidate_adoption_recovery.py @@ -0,0 +1,330 @@ +from __future__ import annotations + +import json +import os +import shlex +import stat +import subprocess +import sys +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters._env_local import LocalEnvironment +from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace +from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError +from tests.workflows.test_workflow_candidate_workspace import _git + + +def _run(repo, *, cache=False): + return subprocess.run( + [sys.executable, *([] if cache else ["-B"]), "app.py"], cwd=repo, + capture_output=True, text=True, check=True, + ).stdout + + +def _contents(root): + result = {} + for path in sorted(root.rglob("*")): + relative = path.relative_to(root) + if ".git" in relative.parts: + continue + mode = path.lstat().st_mode + result[str(relative)] = (stat.S_IFMT(mode), stat.S_IMODE(mode), + os.readlink(path) if path.is_symlink() else + path.read_bytes() if path.is_file() else None) + return result + + +def _repository(tmp_path, *, physical=False, cache=False): + repo = tmp_path / "repo" + repo.mkdir() + (repo / "settings").mkdir() + (repo / "settings/defaults.py").write_text("VALUE = 1\n") + (repo / "loader.py").write_text("from settings.defaults import VALUE\ndef value():\n return VALUE\n") + (repo / "app.py").write_text("from loader import value\nprint(value())\n") + (repo / "other.py").write_text("VALUE = 1\n") + (repo / "note.txt").write_text("committed\n") + (repo / ".gitignore").write_text("__pycache__/\nignored-link\n") + if physical: + (repo / ".gitattributes").write_text("*.py text eol=crlf\n") + _git(repo, "init", "-q") + _git(repo, "config", "user.name", "Test") + _git(repo, "config", "user.email", "test@example.invalid") + _git(repo, "add", ".") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare candidate recovery") + for name in ("note.txt", "loader.py"): + with (repo / name).open("a") as stream: + stream.write("# staged user change\n") + _git(repo, "add", name) + with (repo / name).open("a") as stream: + stream.write("# unstaged user change\n") + (repo / "draft.txt").write_text("untracked user draft\n") + (repo / "draft.txt").chmod(0o600) + (repo / "loader.py").chmod(0o750) + (repo / "settings").chmod(0o750) + (repo / "settings/defaults.py").chmod(0o640) + assert _run(repo, cache=cache) == "1\n" + if cache: + (repo / "settings/ignored-link").symlink_to("../../outside-user-file") + assert list((repo / "settings/__pycache__").glob("*.pyc")) + return repo + + +def _flatten(folder): + (folder / "settings/defaults.py").unlink() + (folder / "settings").rmdir() + (folder / "settings").write_text("42\n") + (folder / "loader.py").write_text( + 'from pathlib import Path\ndef value():\n return int(Path("settings").read_text())\n' + ) + (folder / "draft.txt").write_text("candidate draft\n") + + +@pytest.mark.parametrize("physical", [False, True]) +@pytest.mark.parametrize("cache", [False, True]) +async def test_public_sdk_candidate_recovers_directory_replacement(tmp_path, physical, cache): + repo = _repository(tmp_path, physical=physical, cache=cache) + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + + async def edit(child, inputs): + _flatten(Path(child.workspace_root)) + assert _run(Path(child.workspace_root)) == "42\n" + return "tested" + + async def flow(ctx, inputs): + candidate = await ctx.candidate_workflow(edit, {}, label="flatten") + assert "+42" in candidate.effective_diff + if cache: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await ctx.adopt_candidate(candidate) + assert _contents(repo) == before + else: + await ctx.adopt_candidate(candidate) + return "finished" + + result = await OpenCollab(repo).workflow(flow, trace=False) + assert result.ok, result.reason + assert _run(repo) == ("1\n" if cache else "42\n") + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert _git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1 + + +async def test_git_write_failure_after_successful_check_restores_partial_application(tmp_path): + repo = _repository(tmp_path) + (repo / "z").mkdir(mode=0o555) + if os.access(repo / "z", os.W_OK): + pytest.skip("requires directory write permissions to be enforced") + patch = ( + "diff --git a/other.py b/other.py\n--- a/other.py\n+++ b/other.py\n" + "@@ -1 +1 @@\n-VALUE = 1\n+VALUE = 42\n" + "diff --git a/z/new.txt b/z/new.txt\nnew file mode 100644\n" + "--- /dev/null\n+++ b/z/new.txt\n@@ -0,0 +1 @@\n+new contents\n" + ) + checked = subprocess.run( + ["git", "-C", str(repo), "apply", "--check"], input=patch, capture_output=True, text=True, + ) + assert checked.returncode == 0, checked.stderr + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + try: + with pytest.raises(RuntimeError, match="unable to write file|Permission denied"): + await EnvCandidateWorkspace(environment).adopt(patch) + assert _contents(repo) == before + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not environment.revoked + assert (await environment.exec_cmd("printf usable")).stdout == "usable" + finally: + (repo / "z").chmod(0o755) + await environment.cleanup() + + +@pytest.mark.parametrize("path", [".", ".gi\u200ct/config"]) +async def test_git_rejects_metadata_aliases_before_candidate_backup(tmp_path, path): + repo = _repository(tmp_path) + _git(repo, "config", "core.protectHFS", "true") + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + patch = (f"diff --git a/{path} b/{path}\nnew file mode 100644\n" + f"--- /dev/null\n+++ b/{path}\n@@ -0,0 +1 @@\n+unsafe\n") + environment = LocalEnvironment(str(repo)) + try: + with pytest.raises(RuntimeError, match="invalid path"): + await EnvCandidateWorkspace(environment).adopt(patch) + assert _contents(repo) == before + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not environment.revoked + finally: + await environment.cleanup() + + +class _PublishingFailureEnvironment(LocalEnvironment): + def __init__(self, workspace, *, recovery_failure=False, publication_failure=True): + super().__init__(str(workspace)) + self.recovery_failure = recovery_failure + self.publication_failure = publication_failure + self.published = False + + async def exec_cmd(self, command, timeout=120.0, **kwargs): + if command.startswith("failed=0;") and self.recovery_failure: + raise OSError("recovery device unavailable") + result = await super().exec_cmd(command, timeout, **kwargs) + if (self.publication_failure and " apply --binary " in command + and ".candidate-adopt-" in command and result.returncode == 0): + self.published = True + raise OSError("publication receipt I/O failure") + return result + + +@pytest.mark.parametrize("physical", [False, True]) +@pytest.mark.parametrize("recover", [False, True]) +async def test_publication_error_restores_types_permissions_and_new_parents(tmp_path, physical, recover): + repo = _repository(tmp_path, physical=physical) + (repo / "link").symlink_to("draft.txt") + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + environment = _PublishingFailureEnvironment(repo, recovery_failure=not recover) + + async def edit(child, inputs): + folder = Path(child.workspace_root) + _flatten(folder) + (folder / "created/nested").mkdir(parents=True) + (folder / "created/nested/new.txt").write_text("new contents\n") + (folder / "link").unlink() + (folder / "link").write_text("regular replacement\n") + return "ready" + + async def flow(ctx, inputs): + candidate = await ctx.candidate_workflow(edit, {}, label="publication") + await ctx.adopt_candidate(candidate) + + try: + result = await OpenCollab(repo, environment=environment).workflow(flow, trace=False) + assert environment.published + assert not result.ok + if recover: + assert isinstance(result.error, OSError) + assert str(result.error) == "publication receipt I/O failure" + assert _contents(repo) == before + assert _run(repo) == "1\n" + assert not environment.revoked + else: + assert isinstance(result.error, CandidateAdoptionRecoveryError) + assert "publication receipt I/O failure" in result.reason + assert "recovery device unavailable" in result.reason + retained, = (repo / ".git").glob("candidate-recovery-*") + assert str(retained) in result.reason + originals = { + Path(path.read_text()).relative_to(repo).as_posix(): path.with_suffix("") + for path in retained.glob("*.path") + } + assert originals["loader.py"].read_bytes() == before["loader.py"][2] + assert originals["loader.py"].stat().st_mode & 0o777 == 0o750 + assert originals["settings"].is_dir() + assert originals["link"].is_symlink() + assert os.readlink(originals["link"]) == "draft.txt" + assert environment.revoked + with pytest.raises(RuntimeError, match="aborted"): + await environment.exec_cmd("printf unexpected > other.py") + assert (repo / ".git/index").read_bytes() == index + finally: + await environment.cleanup() + + +def _response(name, arguments, role, step): + return LLMResponse(tool_calls=[{ + "id": f"{role}-{step}", "type": "function", + "function": {"name": name, "arguments": json.dumps(arguments)}, + }], finish_reason="tool_calls", usage=Usage(1, 1)) + + +class _DuoModel: + def __init__(self, *, independent): + self.independent = independent + + async def complete(self, messages, **kwargs): + prompt = "\n".join(str(message.get("content") or "") for message in messages + if message.get("role") == "user") + role = "A" if "You are candidate A," in prompt else "B" if "You are candidate B," in prompt else "judge" + step = sum(message.get("role") == "tool" for message in messages) + if role in {"A", "B"}: + if step == 0: + if role == "A": + script = ( + 'from pathlib import Path; p=Path("settings"); (p/"defaults.py").unlink(); ' + 'p.rmdir(); p.write_text("42\\n"); ' + 'Path("loader.py").write_text("from pathlib import Path\\ndef value():\\n' + ' return int(Path(\\"settings\\").read_text())\\n")' + ) + else: + path = "other.py" if self.independent else "settings/defaults.py" + script = f"from pathlib import Path; Path({path!r}).write_text('VALUE = 42\\n')" + return _response("bash", {"command": shlex.join([sys.executable, "-B", "-c", script])}, role, step) + if step == 1: + return _response("bash", {"command": shlex.join([sys.executable, "-B", "app.py"])}, role, step) + return LLMResponse(content=f"Candidate {role} finished", usage=Usage(1, 1)) + return _response("structured_output", { + "winner": "A", "requirements_complete": True, + "requirements": [{ + "requirement": "Read a flat settings file and print 42", + "a_coverage": "covered", "b_coverage": "not_covered", + "a_evidence": ["loader.py reads the flat settings file containing 42"], + "b_evidence": ["settings/defaults.py still uses the original directory layout"], + }], + "rationale": "A replaces settings/defaults.py with settings and updates loader.py", + }, role, step) + + +@pytest.mark.parametrize("independent", [False, True]) +@pytest.mark.parametrize("recovery_failure", [False, True]) +async def test_duo_fallback_requires_recovered_source(tmp_path, independent, recovery_failure): + repo = _repository(tmp_path, cache=True) + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + environment = _PublishingFailureEnvironment( + repo, recovery_failure=recovery_failure, publication_failure=False, + ) + backend = EnvCandidateWorkspace(environment) + attempted = [] + adopt = backend.adopt_run + + async def recorded(candidate, preserve_paths=()): + attempted.append(candidate.label) + await adopt(candidate, preserve_paths) + + backend.adopt_run = recorded + try: + result = await OpenCollab(repo, environment=environment, model="local-scripted").workflow( + "duo", {"goal": "Read a flat settings file and print 42", "allow_unisolated_shell": True, + "candidate_evidence_dir": str(tmp_path / "evidence")}, + llm=_DuoModel(independent=independent), candidate_workspace=backend, + trace=False, max_steps=5, budget=10000, limit_mode="explicit", timeout=30, + ) + if recovery_failure: + assert not result.ok + assert isinstance(result.error, CandidateAdoptionRecoveryError) + assert len(attempted) == 1 + assert "Directory not empty" in result.reason + assert "original files retained" in result.reason + assert list((repo / ".git").glob("candidate-recovery-*")) + else: + assert result.ok, result.reason + assert len(attempted) == 2 + assert result.output["status"] == "done" + assert result.output["adopted"] == "B" + target = "other.py" if independent else "settings/defaults.py" + after = _contents(repo) + assert after.pop(target)[2] == b"VALUE = 42\n" + before.pop(target) + assert after == before + assert _run(repo) == ("1\n" if independent else "42\n") + assert (repo / ".git/index").read_bytes() == index + finally: + await environment.cleanup() From 04849770a9dd56d46be914ad126cf4a8af3c82e9 Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 11:33:45 +0800 Subject: [PATCH 02/82] fix(llm): recheck queued request budgets before dispatch --- opencollab/adapters/llm/client.py | 94 +++--- .../application/_session_run_completion.py | 42 ++- opencollab/application/session_run.py | 23 +- opencollab/application/workflow.py | 4 + .../test_pooled_request_budget_http.py | 291 ++++++++++++++++++ 5 files changed, 396 insertions(+), 58 deletions(-) create mode 100644 tests/runtime/test_pooled_request_budget_http.py diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index 67ece79f..9944b207 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -9,6 +9,7 @@ import threading import time import uuid +from contextlib import AsyncExitStack from typing import Any, Callable from urllib.parse import urlsplit @@ -122,6 +123,7 @@ def __init__( self.request_pool = request_pool self.request_capacity = request_capacity self.supports_request_admission = request_pool is not None + self.supports_request_revalidation = request_pool is not None self.model = model self.provider = provider self.max_retries = max(0, max_retries) @@ -290,6 +292,7 @@ async def complete( max_output_tokens: int | None = None, response_session_id: str | None = None, on_admitted: Callable[[], None] | None = None, + before_request: Callable[[], int] | None = None, ) -> LLMResponse: """Single-shot completion. Returns full response. @@ -314,8 +317,15 @@ async def complete( ) # Measure here so every provider path records the call timing. with recording() as first_token: - try: - async with request_slot(self.request_pool, self.request_capacity): + async with AsyncExitStack() as request_owner: + try: + await request_owner.enter_async_context(request_slot(self.request_pool, self.request_capacity)) + if before_request is not None: + current_limit = before_request() + max_output_tokens = ( + current_limit if max_output_tokens is None + else min(max_output_tokens, current_limit) + ) if on_admitted is not None: on_admitted() if self._anthropic: @@ -378,44 +388,44 @@ async def complete( first_event_timeout=self.first_event_timeout, stream_idle_timeout=self.stream_idle_timeout, ) - transport_timing = first_token.snapshot() - try: - response.transport_timing = transport_timing - except (AttributeError, TypeError): - # Custom provider responses may reject new attributes. The usage ledger - # still records their timing. - pass - await _record_api_usage_async( - provider=self.provider, - model=self.model, - wire_protocol=self.wire_protocol, - reasoning_effort=reasoning_effort, - base_url=self.base_url, - latency_s=time.monotonic() - start, - status="success", - response=response, - transport_timing=transport_timing, - ) - return response - except BaseException as exc: - if response is not None: - exc.usage = response.usage - else: + transport_timing = first_token.snapshot() try: - await _record_api_usage_async( - provider=self.provider, - model=self.model, - wire_protocol=self.wire_protocol, - reasoning_effort=reasoning_effort, - base_url=self.base_url, - latency_s=time.monotonic() - start, - status="error", - error=exc, - transport_timing=first_token.snapshot(), - ) - except asyncio.CancelledError as cancellation: - usage = getattr(exc, "usage", None) - if usage is not None: - cancellation.usage = usage - raise - raise + response.transport_timing = transport_timing + except (AttributeError, TypeError): + # Custom provider responses may reject new attributes. The usage ledger + # still records their timing. + pass + await _record_api_usage_async( + provider=self.provider, + model=self.model, + wire_protocol=self.wire_protocol, + reasoning_effort=reasoning_effort, + base_url=self.base_url, + latency_s=time.monotonic() - start, + status="success", + response=response, + transport_timing=transport_timing, + ) + return response + except BaseException as exc: + if response is not None: + exc.usage = response.usage + else: + try: + await _record_api_usage_async( + provider=self.provider, + model=self.model, + wire_protocol=self.wire_protocol, + reasoning_effort=reasoning_effort, + base_url=self.base_url, + latency_s=time.monotonic() - start, + status="error", + error=exc, + transport_timing=first_token.snapshot(), + ) + except asyncio.CancelledError as cancellation: + usage = getattr(exc, "usage", None) + if usage is not None: + cancellation.usage = usage + raise + raise diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index 7f696175..d4237411 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -777,6 +777,15 @@ def _completion_accepts_output_limit(self) -> bool: for parameter in parameters ) + def _revalidate_request( + self, messages: list[dict], tools: list[dict] | None, *, thinking: bool = False, + thinking_params: dict[str, Any] | None = None, + ) -> int: + self._raise_if_run_cancelled() + return self._request_output_limit( + messages, tools, thinking=thinking, thinking_params=thinking_params, + ) + async def _invoke_summary( self, complete: Callable[..., Awaitable[CompletionResponse]], messages: list[dict], *, _summary_output_limit: Callable[[list[dict]], int] | None = None, **kwargs: Any, @@ -799,6 +808,13 @@ async def _invoke_summary( admitted = asyncio.Event() if getattr(self.llm, "supports_request_admission", False) else None if admitted is not None: kwargs["on_admitted"] = admitted.set + if getattr(self.llm, "supports_request_revalidation", False): + def revalidate() -> int: + limit = self._revalidate_request( + messages, None, thinking=thinking, thinking_params=thinking_params, + ) + return min(limit, _summary_output_limit(messages)) if _summary_output_limit is not None else limit + kwargs["before_request"] = revalidate abandoned = False if self.tracer: self.tracer.log_step(step_type="llm_call_started", payload={ @@ -870,17 +886,33 @@ async def _invoke_llm(self, **kwargs: Any) -> CompletionResponse: admitted = asyncio.Event() if getattr(self.llm, "supports_request_admission", False) else None if admitted is not None: kwargs["on_admitted"] = admitted.set - operation = _complete_with_error_usage(self, self.llm.complete, protected_call=protected_call, **kwargs) + if getattr(self.llm, "supports_request_revalidation", False): + kwargs["before_request"] = lambda: self._revalidate_request( + kwargs["messages"], kwargs.get("tools"), thinking=kwargs.get("thinking", False), + thinking_params=kwargs.get("thinking_params"), + ) + async def complete_owned() -> CompletionResponse: + try: + response = await _complete_with_error_usage( + self, self.llm.complete, protected_call=protected_call, **kwargs, + ) + self._account_completion_usage( + response, protected_call=protected_call, + late=asyncio.current_task() in self._draining_provider_tasks, + ) + return response + finally: + self._draining_provider_tasks.discard(asyncio.current_task()) + self._finish_run_control_if_quiesced() + + operation = complete_owned() try: response = await operation if self._per_call_timeout is None else await abandon_on_timeout( operation, self._per_call_timeout, task_tracker=self._track_provider_task, late_task_tracker=self._mark_provider_task_draining, - late_result_handler=lambda task: self._record_late_provider_result( - task, - protected_call=protected_call, - ), + late_result_handler=self._provider_task_done, start_event=admitted, ) except CallerTimeoutError as exc: diff --git a/opencollab/application/session_run.py b/opencollab/application/session_run.py index f9184545..bd942887 100644 --- a/opencollab/application/session_run.py +++ b/opencollab/application/session_run.py @@ -254,19 +254,23 @@ def _record_late_provider_result( """Charge a provider response that survived caller cancellation or timeout.""" try: response = task.result() - _input_tokens, total_tokens = _normalize_completion_usage(response.usage) - self._late_provider_usage += (total_tokens,) - self.state.add_used_tokens(total_tokens) - self._record_usage_event(response.usage, total_tokens, purpose="completion", late=True) - self._mark_budget_reserve_consumed(protected_call=protected_call) - if self.late_provider_usage_checkpoint is not None: - self.late_provider_usage_checkpoint() + self._account_completion_usage(response, protected_call=protected_call, late=True) except BaseException: pass finally: self._draining_provider_tasks.discard(task) self._finish_run_control_if_quiesced() + def _account_completion_usage(self, response: Any, *, protected_call: bool, late: bool) -> None: + _input_tokens, total_tokens = _normalize_completion_usage(response.usage) + self.state.add_used_tokens(total_tokens) + self._record_usage_event(response.usage, total_tokens, purpose="completion", late=late) + self._mark_budget_reserve_consumed(protected_call=protected_call) + if late: + self._late_provider_usage += (total_tokens,) + if self.late_provider_usage_checkpoint is not None: + self.late_provider_usage_checkpoint() + async def run_loop(self, cancel_event: asyncio.Event | None = None) -> str: """Drive the phase FSM until the turn finishes or suspends. @@ -699,11 +703,8 @@ async def run_llm_call(self) -> None: await self._stop_precheck("team budget exceeded: aggregate spend reached the global cap") return latency = time.monotonic() - start - input_tokens, total_tokens = _normalize_completion_usage(response.usage) - self.state.add_used_tokens(total_tokens) - self._record_usage_event(response.usage, total_tokens, purpose="completion") + input_tokens, _total_tokens = _normalize_completion_usage(response.usage) self._run_final_prompt = None - self._mark_budget_reserve_consumed() self.state.add_markup_recovered(getattr(response.usage, "markup_recovered", 0)) self.state.set_context_tokens(input_tokens) diff --git a/opencollab/application/workflow.py b/opencollab/application/workflow.py index 491f8a27..25ed7efc 100644 --- a/opencollab/application/workflow.py +++ b/opencollab/application/workflow.py @@ -712,6 +712,10 @@ def _track_session(self, session: Any) -> None: lease = self._active_budget_lease.get() if lease is not None: lease.sessions.append(session) + runner = getattr(session, "runner", None) + if lease.reserved and runner is not None: + # Other leases keep their allocation when a sibling overspends. + runner.budget_reservations.append(lambda: max(0, -int(self.budget.remaining()))) def _active_call_has_pending_cleanup(self) -> bool: lease = self._active_budget_lease.get() diff --git a/tests/runtime/test_pooled_request_budget_http.py b/tests/runtime/test_pooled_request_budget_http.py new file mode 100644 index 00000000..04ae0327 --- /dev/null +++ b/tests/runtime/test_pooled_request_budget_http.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters.llm import client as transport +from opencollab.adapters.llm.client import LLMClient +from opencollab.adapters.llm.request_admission import request_slot +from opencollab.application._session_run_shared import _TeamBudgetStop, _TokenBudgetStop +from opencollab.application.run_control import _RunCancelled +from opencollab.bootstrap import build_session +from opencollab.domain.agent import Agent +from tests.runtime.test_llm_chat_error_usage_http import ReportedUsageStream +from tests.support.environment_test_support import local_environment_factory as local_environment_factory +from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport + + +@pytest.fixture(autouse=True) +def isolated_configuration(monkeypatch, tmp_path): + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + + +async def _wait_until(predicate): + async def wait(): + while not predicate(): + await asyncio.sleep(0) + await asyncio.wait_for(wait(), 5) + + +@pytest.mark.parametrize("method", ["agent", "agent_run"]) +@pytest.mark.parametrize("first_tokens, total, slow_ledger", [ + (5_000, 20_000, False), + ("reduced", 20_000, False), + (19_990, 20_000, False), + (20_210, 20_000, False), + (20_210, 20_000, True), + (20_210, 20_000, "handoff"), + (20_210, 20_000, "error_usage"), + (20_210, None, False), +]) +async def test_native_queue_rechecks_shared_workflow_budget( + tmp_path, monkeypatch, method, first_tokens, total, slow_ledger, +): + started, release = asyncio.Event(), asyncio.Event() + contexts, requests, limits = [], [], [] + + async def handler(request): + body = json.loads(request.content) + requests.append((body, contexts[0].tokens_spent())) + if len(requests) == 1: + started.set() + await release.wait() + if slow_ledger == "error_usage": + return http_response( + request, 200, headers={"content-type": "text/event-stream"}, + stream=ReportedUsageStream( + {"prompt_tokens": first_tokens - 10, "completion_tokens": 10, "total_tokens": first_tokens}, + disconnect=True, + ), + ) + response = completion_http_response(request, text="first answer", output_tokens=10) + payload = response.json() + payload["usage"].update(prompt_tokens=first_tokens - 10, total_tokens=first_tokens) + return http_response(request, 200, json=payload) + return completion_http_response(request, text="second answer") + + install_sdk_transport(monkeypatch, "openai", handler) + record = transport._record_api_usage_async + + async def delayed_record(**kwargs): + if slow_ledger is True or slow_ledger == "error_usage": + await asyncio.sleep(0.12) + await record(**kwargs) + + monkeypatch.setattr(transport, "_record_api_usage_async", delayed_record) + if slow_ledger == "handoff": + released_slot = asyncio.Event() + native_slot = transport.request_slot + entries = 0 + + @asynccontextmanager + async def ordered_slot(*args): + nonlocal entries + index = entries + entries += 1 + if index: + await released_slot.wait() + async with native_slot(*args): + yield + if not index: + released_slot.set() + + monkeypatch.setattr(transport, "request_slot", ordered_slot) + pool = tmp_path / "pool" + pool.mkdir() + async with LLMClient( + model="gpt-4o", api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", request_pool=str(pool), request_capacity=1, + max_retries=0, + stream_chat=slow_ledger == "error_usage", + ) as client: + async def flow(ctx, _args): + nonlocal first_tokens + contexts.append(ctx) + run = getattr(ctx, method) + tasks = [asyncio.create_task(run("first", budget=18_000, tools=[]))] + try: + await asyncio.wait_for(started.wait(), 5) + tasks.append(asyncio.create_task(run("second", budget=2_000, tools=[]))) + await _wait_until(lambda: len(ctx.sessions) == 2 and ctx.sessions[1].runner._llm_step_started) + assert len(requests) == 1 + second = ctx.sessions[1] + reserved = client.estimate_request_tokens(second.messages) + if first_tokens == "reduced": + first_tokens = total - reserved - 50 + expected = min(128, (2_000 if total is None else min(2_000, total - first_tokens)) - reserved) + limits.append(expected) + release.set() + values = await asyncio.wait_for(asyncio.gather(*tasks), 5) + if expected >= 1: + assert getattr(values[1], "output", values[1]) == "second answer" + assert requests[1][0]["max_tokens"] == expected + assert requests[1][1] == first_tokens + else: + assert getattr(values[1], "output", values[1]) == "" + assert "budget exhausted before model call" in second.state.terminal_reason + return "finished" + finally: + release.set() + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + result = await OpenCollab(tmp_path, model="gpt-4o", config={"max_output_tokens": 128}).workflow( + flow, llm=client, budget=total, concurrency=2, trace=False, limit_mode="explicit", + ) + assert result.status == "completed", result.reason + expected_calls = 2 if limits[0] >= 1 else 1 + assert len(requests) == expected_calls + assert result.tokens == first_tokens + (4 if expected_calls == 2 else 0) + assert contexts[0].budget._leases == [] + assert contexts[0].pending_cleanup_tasks == () + + +async def test_native_overspend_retains_requests_that_already_started(tmp_path, monkeypatch): + first_started, second_started = asyncio.Event(), asyncio.Event() + release_first, release_second = asyncio.Event(), asyncio.Event() + requests, contexts = [], [] + + async def handler(request): + body = json.loads(request.content) + prompt = body["messages"][-1]["content"].splitlines()[0] + requests.append(prompt) + if prompt == "first": + first_started.set() + await release_first.wait() + payload = completion_http_response(request, text="first answer", output_tokens=10).json() + payload["usage"].update(prompt_tokens=20_200, total_tokens=20_210) + return http_response(request, 200, json=payload) + second_started.set() + await release_second.wait() + return completion_http_response(request, text="second answer") + + install_sdk_transport(monkeypatch, "openai", handler) + pool = tmp_path / "pool" + pool.mkdir() + async with LLMClient( + model="gpt-4o", api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", request_pool=str(pool), request_capacity=2, + ) as client: + async def flow(ctx, _args): + contexts.append(ctx) + tasks = [asyncio.create_task(ctx.agent("first", tools=[], budget=18_000))] + try: + await asyncio.wait_for(first_started.wait(), 5) + tasks.append(asyncio.create_task(ctx.agent("second", tools=[], budget=2_000))) + await asyncio.wait_for(second_started.wait(), 5) + release_first.set() + assert await asyncio.wait_for(tasks[0], 5) == "first answer" + assert ctx.tokens_spent() == 20_210 + assert not tasks[1].done() + release_second.set() + assert await asyncio.wait_for(tasks[1], 5) == "second answer" + return "finished" + finally: + release_first.set() + release_second.set() + await asyncio.gather(*tasks, return_exceptions=True) + + result = await OpenCollab(tmp_path, model="gpt-4o", config={"max_output_tokens": 128}).workflow( + flow, llm=client, budget=20_000, concurrency=2, trace=False, limit_mode="explicit", + ) + assert result.status == "completed", result.reason + assert requests == ["first", "second"] + assert result.tokens == 20_214 + assert contexts[0].pending_cleanup_tasks == () + + +@pytest.mark.parametrize("configuration, field", [ + ({"provider": "openai", "wire_protocol": "chat_completions", "model": "gpt-4o"}, "max_tokens"), + ({"provider": "openai", "wire_protocol": "responses", "model": "gpt-4o"}, "max_output_tokens"), + ({"provider": "anthropic", "model": "claude-sonnet-4-6"}, "max_tokens"), +], ids=["chat", "responses", "anthropic"]) +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("change", ["reduced", "exhausted", "unbounded", "cancelled", "team"]) +async def test_admission_uses_current_session_limits_and_cancellation( + tmp_path, monkeypatch, local_environment_factory, configuration, field, summary, change, +): + requests = [] + + async def handler(request): + requests.append(json.loads(request.content)) + return completion_http_response(request) + + install_sdk_transport(monkeypatch, configuration["provider"], handler) + pool = tmp_path / "pool" + pool.mkdir() + team_exhausted, cancel = asyncio.Event(), asyncio.Event() + async with LLMClient( + **configuration, api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", request_pool=str(pool), request_capacity=1, + max_retries=0, + ) as client: + session = build_session( + agent=Agent(name="queued", system_prompt="system", max_tokens_per_step=128, **configuration), + llm=client, env=local_environment_factory(str(tmp_path)), max_budget_tokens=20_000, + team_budget_exhausted=team_exhausted.is_set, default_cancel_event=cancel, + ) + messages = [{"role": "user", "content": "Finish the task."}] + reserved = client.estimate_request_tokens(messages) + session.messages = messages + queued = asyncio.Event() + native_slot = transport.request_slot + + @asynccontextmanager + async def observed_slot(*args): + queued.set() + async with native_slot(*args): + yield + + monkeypatch.setattr(transport, "request_slot", observed_slot) + + async def complete_summary(messages, *, on_response, **kwargs): + response = await client.complete(messages, **kwargs) + on_response(response) + return response + + try: + async with request_slot(str(pool), 1): + operation = session.runner._invoke_summary( + complete_summary, messages, _summary_output_limit=lambda _: 80, + ) if summary else session.run_loop() + task = asyncio.create_task(operation) + await asyncio.wait_for(queued.wait(), 5) + assert requests == [] + if change == "reduced": + session.max_budget_tokens = reserved + 50 + elif change == "exhausted": + session.used_tokens = 20_000 + elif change == "unbounded": + session.max_budget_tokens = None + elif change == "cancelled": + cancel.set() + else: + team_exhausted.set() + if summary and change in {"exhausted", "cancelled", "team"}: + error = {"exhausted": _TokenBudgetStop, "cancelled": _RunCancelled, "team": _TeamBudgetStop}[change] + with pytest.raises(error): + await asyncio.wait_for(task, 5) + else: + answer = await asyncio.wait_for(task, 5) + if change in {"reduced", "unbounded"}: + assert getattr(answer, "content", answer) == "ok" + else: + assert answer == "" + if change in {"reduced", "unbounded"}: + expected = 50 if change == "reduced" else (80 if summary else 128) + assert len(requests) == 1 and requests[0][field] == expected + assert session.used_tokens == 4 + else: + assert requests == [] + assert session.used_tokens == (20_000 if change == "exhausted" else 0) + finally: + await session.aclose() From dc52061177c967d076fc3d9ceb105b1d344e862e Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 11:39:59 +0800 Subject: [PATCH 03/82] fix(session): settle cancellation races after eager usage accounting --- .../application/_session_run_completion.py | 17 ++++++++++++----- opencollab/application/session_run.py | 16 +++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index d4237411..22400efd 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -891,20 +891,27 @@ async def _invoke_llm(self, **kwargs: Any) -> CompletionResponse: kwargs["messages"], kwargs.get("tools"), thinking=kwargs.get("thinking", False), thinking_params=kwargs.get("thinking_params"), ) + late_accounted = False + async def complete_owned() -> CompletionResponse: + nonlocal late_accounted try: response = await _complete_with_error_usage( self, self.llm.complete, protected_call=protected_call, **kwargs, ) - self._account_completion_usage( - response, protected_call=protected_call, - late=asyncio.current_task() in self._draining_provider_tasks, - ) + late_accounted = asyncio.current_task() in self._draining_provider_tasks + self._account_completion_usage(response, protected_call=protected_call, late=late_accounted) return response finally: self._draining_provider_tasks.discard(asyncio.current_task()) self._finish_run_control_if_quiesced() + def record_abandoned(task: asyncio.Task[Any]) -> None: + if late_accounted: + self._provider_task_done(task) + else: + self._record_late_provider_result(task, protected_call=protected_call, usage_accounted=True) + operation = complete_owned() try: response = await operation if self._per_call_timeout is None else await abandon_on_timeout( @@ -912,7 +919,7 @@ async def complete_owned() -> CompletionResponse: self._per_call_timeout, task_tracker=self._track_provider_task, late_task_tracker=self._mark_provider_task_draining, - late_result_handler=self._provider_task_done, + late_result_handler=record_abandoned, start_event=admitted, ) except CallerTimeoutError as exc: diff --git a/opencollab/application/session_run.py b/opencollab/application/session_run.py index bd942887..7af543a0 100644 --- a/opencollab/application/session_run.py +++ b/opencollab/application/session_run.py @@ -250,22 +250,28 @@ def _record_late_provider_result( task: asyncio.Future[Any], *, protected_call: bool = False, + usage_accounted: bool = False, ) -> None: """Charge a provider response that survived caller cancellation or timeout.""" try: response = task.result() - self._account_completion_usage(response, protected_call=protected_call, late=True) + self._account_completion_usage( + response, protected_call=protected_call, late=True, usage_accounted=usage_accounted, + ) except BaseException: pass finally: self._draining_provider_tasks.discard(task) self._finish_run_control_if_quiesced() - def _account_completion_usage(self, response: Any, *, protected_call: bool, late: bool) -> None: + def _account_completion_usage( + self, response: Any, *, protected_call: bool, late: bool, usage_accounted: bool = False, + ) -> None: _input_tokens, total_tokens = _normalize_completion_usage(response.usage) - self.state.add_used_tokens(total_tokens) - self._record_usage_event(response.usage, total_tokens, purpose="completion", late=late) - self._mark_budget_reserve_consumed(protected_call=protected_call) + if not usage_accounted: + self.state.add_used_tokens(total_tokens) + self._record_usage_event(response.usage, total_tokens, purpose="completion", late=late) + self._mark_budget_reserve_consumed(protected_call=protected_call) if late: self._late_provider_usage += (total_tokens,) if self.late_provider_usage_checkpoint is not None: From 0cc90358ef57bf760e22c60e8b959430edae6294 Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 11:48:18 +0800 Subject: [PATCH 04/82] fix: restore pruned candidate directory ancestors --- opencollab/adapters/_candidate_adoption.py | 41 ++++++++++++++- .../test_candidate_adoption_recovery.py | 51 +++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index 3e631234..2dd7aeca 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -38,6 +38,27 @@ async def _adoption_transaction( for path in sorted(set(inspected.split("\0")) - {""}): if not any(path.startswith(root + "/") for root in roots): roots.append(path) + ancestors = set() + for path in roots: + parent = posixpath.dirname(path) + while parent: + ancestors.add(parent) + parent = posixpath.dirname(parent) + directory_modes: list[tuple[str, str]] = [] + if ancestors: + metadata = _complete(await environment.exec_cmd( + f"cd -- {shlex.quote(workspace)} && " + f"for path in {shlex.join(sorted(ancestors, key=lambda path: (path.count('/'), path)))}; do " + 'if [ -d "$path" ] && [ ! -L "$path" ]; then ' + 'mode=$(stat -c %a -- "$path" 2>/dev/null) || ' + 'mode=$(stat -f %Lp -- "$path") || exit; ' + 'printf "%s\\0%s\\0" "$path" "$mode"; fi; done', + timeout=120, + ), "candidate ancestor directory inspection").split("\0") + for path, mode in zip(metadata[::2], metadata[1::2]): + if not mode or len(mode) > 4 or any(digit not in "01234567" for digit in mode): + raise RuntimeError("candidate ancestor directory mode is invalid") + directory_modes.append((path, mode)) storage = '"${TMPDIR:-/tmp}"' if repository: storage = shlex.quote(_complete(await environment.exec_cmd( @@ -51,6 +72,22 @@ async def _adoption_transaction( try: copies = [] restores = [] + recreate_directories = [] + restore_directory_modes = [] + # Git prunes empty ancestors after deleting their last leaf. Recreate + # those directories with write access until their saved leaves return. + for index, (path, mode) in enumerate(directory_modes): + source = shlex.quote(posixpath.join(workspace, path)) + marker = shlex.quote(posixpath.join(backup, f"directory-{index}")) + copies.append(f"printf '%s\\0%s\\0' {source} {mode} > {marker}.mode || exit") + recreate_directories.append( + f"if [ ! -e {source} ] && [ ! -L {source} ]; then " + f"if mkdir -m 700 -- {source} && : > {marker}; then :; else failed=1; fi; " + f"elif [ ! -d {source} ] || [ -L {source} ]; then failed=1; fi" + ) + restore_directory_modes.append( + f"if [ -f {marker} ]; then chmod -- {mode} {source} || failed=1; fi" + ) for index, path in enumerate(roots): source = shlex.quote(posixpath.join(workspace, path)) saved = shlex.quote(posixpath.join(backup, str(index))) @@ -74,7 +111,9 @@ async def _adoption_transaction( except BaseException as failure: try: _complete(await await_owned_operation(environment.exec_cmd( - "failed=0; " + "; ".join(restores) + '; [ "$failed" -eq 0 ]', + "failed=0; " + "; ".join(( + *recreate_directories, *restores, *reversed(restore_directory_modes), + )) + '; [ "$failed" -eq 0 ]', timeout=900, )), "candidate adoption recovery") except BaseException as recovery: diff --git a/tests/workflows/test_candidate_adoption_recovery.py b/tests/workflows/test_candidate_adoption_recovery.py index dc9ce60f..8242f8cd 100644 --- a/tests/workflows/test_candidate_adoption_recovery.py +++ b/tests/workflows/test_candidate_adoption_recovery.py @@ -114,6 +114,57 @@ async def flow(ctx, inputs): assert _git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1 +@pytest.mark.parametrize("physical", [False, True]) +@pytest.mark.parametrize("pruned", ["both", "nested", "none"]) +@pytest.mark.parametrize("cache", [False, True]) +async def test_public_sdk_recovers_pruned_ancestors_and_modes(tmp_path, physical, pruned, cache): + repo = _repository(tmp_path, physical=physical, cache=cache) + (repo / "old/nested").mkdir(parents=True) + (repo / "old/nested/helper.py").write_text("VALUE = 1\n") + _git(repo, "add", "old/nested/helper.py") + (repo / "old").chmod(0o750) + (repo / "old/nested").chmod(0o710) + (repo / "old/nested/helper.py").chmod(0o640) + if pruned != "both": + sibling = "old/sibling.txt" if pruned == "nested" else "old/nested/sibling.txt" + (repo / sibling).write_text("user sibling\n") + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + + async def edit(child, inputs): + folder = Path(child.workspace_root) + (folder / "old/nested/helper.py").unlink() + _flatten(folder) + assert _run(folder) == "42\n" + return "tested" + + async def flow(ctx, inputs): + candidate = await ctx.candidate_workflow(edit, {}, label="pruned-parent") + if cache: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await ctx.adopt_candidate(candidate) + assert _contents(repo) == before + else: + await ctx.adopt_candidate(candidate) + assert not (repo / "old/nested/helper.py").exists() + assert (repo / "old").exists() is (pruned != "both") + assert (repo / "old/nested").exists() is (pruned == "none") + assert not environment.revoked + assert (await environment.exec_cmd("printf recovered")).stdout == "recovered" + return "recovered" + + try: + result = await OpenCollab(repo, environment=environment).workflow(flow, trace=False) + assert result.ok, result.reason + assert result.output == "recovered" + assert _run(repo) == ("1\n" if cache else "42\n") + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").glob("candidate-recovery-*")) + finally: + await environment.cleanup() + + async def test_git_write_failure_after_successful_check_restores_partial_application(tmp_path): repo = _repository(tmp_path) (repo / "z").mkdir(mode=0o555) From 12f543c6bf30172531cb2bb95c3b13f8e06d542d Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 11:58:06 +0800 Subject: [PATCH 05/82] fix(workflow): enforce authored child budgets at native admission --- docs/session-isolation.md | 4 + docs/session-isolation.zh-CN.md | 3 + opencollab/adapters/llm/client.py | 9 +- opencollab/bootstrap/authored_workflow.py | 15 +- .../bootstrap/authored_workflow_bridge.py | 66 +++++- ..._member_authored_workflow_queued_budget.py | 194 ++++++++++++++++ .../test_authored_request_budget_http.py | 212 ++++++++++++++++++ 7 files changed, 496 insertions(+), 7 deletions(-) create mode 100644 tests/interfaces/test_member_authored_workflow_queued_budget.py create mode 100644 tests/runtime/test_authored_request_budget_http.py diff --git a/docs/session-isolation.md b/docs/session-isolation.md index b1584e3c..524c5b29 100644 --- a/docs/session-isolation.md +++ b/docs/session-isolation.md @@ -241,3 +241,7 @@ to run its child agents in independent worktrees inside its own VM. Individual child calls can select `isolation="vm-session"` to request another guest. The [executable VM example](../examples/vm-authored-workflow/README.md) starts with a member writing source and finishes by collecting the native child result. + +The `budget` on `launch` is the child's total token allowance, shared across its +sessions. The host checks the remaining allowance before each model request, +including calls made with `over_budget_ok=True` inside the child. diff --git a/docs/session-isolation.zh-CN.md b/docs/session-isolation.zh-CN.md index e573dc4c..e55b72d1 100644 --- a/docs/session-isolation.zh-CN.md +++ b/docs/session-isolation.zh-CN.md @@ -174,3 +174,6 @@ Team 成员之间的通信继续遵循配置中的拓扑。 省略时继承成员策略。成员使用 `{"action":"launch","source":"generated.py","isolation":"worktree"}` 可让子 Agent 在自己的 VM 中使用独立工作树。单个子调用可通过 `isolation="vm-session"` 申请另一台来宾。 [可执行 VM 示例](../examples/vm-authored-workflow/README.zh-CN.md) 从成员写出源码开始,执行到取得原生子运行结果。 + +`launch` 的 `budget` 是子运行各 Session 共享的总额度。控制端在每个模型请求开始前检查剩余额度, +子代码使用 `over_budget_ok=True` 的调用也受该额度约束。 diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index 9944b207..92ed8972 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -123,7 +123,7 @@ def __init__( self.request_pool = request_pool self.request_capacity = request_capacity self.supports_request_admission = request_pool is not None - self.supports_request_revalidation = request_pool is not None + self.supports_request_revalidation = True self.model = model self.provider = provider self.max_retries = max(0, max_retries) @@ -293,6 +293,7 @@ async def complete( response_session_id: str | None = None, on_admitted: Callable[[], None] | None = None, before_request: Callable[[], int] | None = None, + on_usage: Callable[[Any], None] | None = None, ) -> LLMResponse: """Single-shot completion. Returns full response. @@ -310,6 +311,7 @@ async def complete( """ start = time.monotonic() response = None + settled_usage = None retry_budget = ( self.provider_retry_budget if self._retry_budget_explicit @@ -388,6 +390,7 @@ async def complete( first_event_timeout=self.first_event_timeout, stream_idle_timeout=self.stream_idle_timeout, ) + settled_usage = response.usage transport_timing = first_token.snapshot() try: response.transport_timing = transport_timing @@ -408,6 +411,7 @@ async def complete( ) return response except BaseException as exc: + settled_usage = response.usage if response is not None else getattr(exc, "usage", None) if response is not None: exc.usage = response.usage else: @@ -429,3 +433,6 @@ async def complete( cancellation.usage = usage raise raise + finally: + if on_usage is not None and settled_usage is not None: + on_usage(settled_usage) diff --git a/opencollab/bootstrap/authored_workflow.py b/opencollab/bootstrap/authored_workflow.py index b6cb00da..d5d55cf2 100644 --- a/opencollab/bootstrap/authored_workflow.py +++ b/opencollab/bootstrap/authored_workflow.py @@ -212,9 +212,18 @@ def _snapshot(self, child: _Child, *, include_result: bool) -> dict[str, Any]: return record def _reserved_tokens(self) -> int: - return sum(max(0, (item.budget or 0) - item.tokens) + # Retain native spend while the child's usage file catches up. + return sum(max(0, (item.budget or 0) - item.tokens, + (item.bridge.tokens if item.bridge is not None else item.tokens) - item.tokens) for item in self._children.values() if item.status == "running") + def _remaining_child_budget(self, child: _Child) -> int | None: + remaining = self._get_session().runner.remaining_budget_tokens + if remaining is None: + return None + spent = max(child.tokens, child.bridge.tokens if child.bridge is not None else 0) + return remaining + max(0, (child.budget or 0) - spent) + async def _launch(self, params: dict[str, Any], runtime: ToolRuntime) -> _Child: parent = self._get_session() environment = runtime.environment @@ -291,6 +300,10 @@ async def _launch(self, params: dict[str, Any], runtime: ToolRuntime) -> _Child: await runtime.safety_policy.check_cmd_interactive(command, runtime.confirm_fn()) child.bridge = AuthoredWorkflowBridge( environment, artifacts, pool, config, runtime, + budget=budget, + remaining_budget=lambda: self._remaining_child_budget(child), + team_budget_exhausted=parent.runner._team_budget_exhausted, + cancel_requested=lambda: child.stop_requested or parent.runner._run_cancellation_requested(), allow_unisolated_shell=self._allow_unisolated_shell and not getattr(shell, "require_process_isolation", False), ) diff --git a/opencollab/bootstrap/authored_workflow_bridge.py b/opencollab/bootstrap/authored_workflow_bridge.py index 5fc2f6e7..a35934ee 100644 --- a/opencollab/bootstrap/authored_workflow_bridge.py +++ b/opencollab/bootstrap/authored_workflow_bridge.py @@ -12,16 +12,19 @@ from contextvars import ContextVar from dataclasses import asdict from pathlib import Path -from typing import Any +from typing import Any, Callable from opencollab.adapters._env_base import Environment, ExecResult, TextFileRange from opencollab.adapters.llm import errors as llm_errors from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.llm.types import LLMResponse, Usage, to_plain_data from opencollab.adapters.tools.base import host_write_lock +from opencollab.application._session_run_shared import _TeamBudgetStop, _TokenBudgetStop from opencollab.application._session_run_usage import _normalize_completion_usage from opencollab.application.async_timeout import await_owned_operation, consume_task_result from opencollab.application.exception_notes import add_exception_note +from opencollab.application.run_control import _RunCancelled +from opencollab.domain.agent import DEFAULT_MAX_TOKENS_PER_STEP _METADATA = {"context_window", "estimate_request_tokens", "estimate_context_tokens", "minimum_output_tokens"} _COMPLETION = {"messages", "tools", "temperature", "thinking", "thinking_params", "reasoning_effort", @@ -37,6 +40,13 @@ def _write_json(path: Path, value: Any) -> None: def _restore_error(value: dict[str, Any]) -> Exception: name = value.pop("type") + if name in {"_RunCancelled", "_TeamBudgetStop"}: + return {"_RunCancelled": _RunCancelled, "_TeamBudgetStop": _TeamBudgetStop}[name]() + if name == "_TokenBudgetStop": + return _TokenBudgetStop( + reserved_input_tokens=value["reserved_input_tokens"], remaining_budget=value["remaining_budget"], + minimum_output_tokens=value["minimum_output_tokens"], + ) cls = getattr(llm_errors, name, getattr(builtins, name, RuntimeError)) if not isinstance(cls, type) or not issubclass(cls, Exception): cls = RuntimeError @@ -102,7 +112,10 @@ class AuthoredWorkflowBridge: """Own only this authored run's strategy pools and admitted operations.""" def __init__(self, environment: Any, artifacts: str, pool: Any, llm: Any, runtime: Any, - *, allow_unisolated_shell: bool) -> None: + *, allow_unisolated_shell: bool, budget: int | None = None, + remaining_budget: Callable[[], int | None] | None = None, + team_budget_exhausted: Callable[[], bool] | None = None, + cancel_requested: Callable[[], bool] | None = None) -> None: relative = posixpath.relpath(artifacts, environment.workspace) if relative == ".." or relative.startswith("../") or posixpath.isabs(relative): raise ValueError("child mailbox must be inside the member workspace") @@ -114,6 +127,10 @@ def __init__(self, environment: Any, artifacts: str, pool: Any, llm: Any, runtim self._environments = {0: environment} self._next_environment = 0 self.tokens = 0 + self._budget = budget + self._remaining_budget = remaining_budget + self._team_budget_exhausted = team_budget_exhausted + self._cancel_requested = cancel_requested self._tasks: dict[str, asyncio.Task] = {} self._handles: dict[str, int] = {} self._edits: dict[str, _FileEdit] = {} @@ -142,6 +159,39 @@ def options(self) -> dict[str, Any]: "supports_request_admission": self._config.get("request_pool") is not None, "supports_response_session_identity": self._config.get("wire_protocol") == "responses"} + def _request_output_limit(self, arguments: dict[str, Any]) -> int: + if self._cancel_requested is not None and self._cancel_requested(): + raise _RunCancelled + if self._team_budget_exhausted is not None and self._team_budget_exhausted(): + raise _TeamBudgetStop + configured = arguments.get("max_output_tokens") + if configured is None: + configured = self._config.get("max_output_tokens", DEFAULT_MAX_TOKENS_PER_STEP) + remaining = None if self._budget is None else self._budget - self.tokens + if self._remaining_budget is not None: + parent_remaining = self._remaining_budget() + if parent_remaining is not None: + remaining = parent_remaining if remaining is None else min(remaining, parent_remaining) + if remaining is None: + return configured + model = self._model() + thinking = arguments.get("thinking", False) + thinking_params = arguments.get("thinking_params") + reserved = model.estimate_request_tokens( + arguments["messages"], arguments.get("tools"), thinking=thinking, thinking_params=thinking_params, + ) + minimum = model.minimum_output_tokens( + max_output_tokens=configured, thinking=thinking, thinking_params=thinking_params, + ) + if remaining - reserved < minimum: + raise _TokenBudgetStop( + reserved_input_tokens=reserved, remaining_budget=remaining, minimum_output_tokens=minimum, + ) + return min(configured, remaining - reserved) + + def _account_native_usage(self, usage: Any) -> None: + self.tokens += _normalize_completion_usage(usage)[1] + async def poll(self) -> None: try: requests = json.loads(await self._environment.read_file(posixpath.join(self._directory, "requests.json"))) @@ -183,7 +233,8 @@ async def _respond(self, identity: str, request: dict[str, Any]) -> None: if not isinstance(exc, (Exception, asyncio.CancelledError)): raise error = {"type": type(exc).__name__, "message": str(exc)} - for name in ("usage", "status_code", "code", "param", "body"): + for name in ("usage", "status_code", "code", "param", "body", "reserved_input_tokens", + "remaining_budget", "minimum_output_tokens"): value = getattr(exc, name, None) if value is not None: error[name] = to_plain_data(value) @@ -233,6 +284,10 @@ async def _dispatch(self, identity: str, request: dict[str, Any]) -> Any: if set(arguments) - _COMPLETION: raise ValueError("unknown completion controls") notifications = [] + native_accounting = getattr(self._model(), "supports_request_revalidation", False) + if native_accounting: + arguments["before_request"] = lambda: self._request_output_limit(arguments) + arguments["on_usage"] = self._account_native_usage if getattr(self._model(), "supports_request_admission", False): def admitted() -> None: notifications.append(asyncio.create_task(self._environment.write_file( @@ -243,10 +298,11 @@ def admitted() -> None: try: response = await self._model().complete(**arguments) except BaseException as exc: - if getattr(exc, "usage", None) is not None: + if not native_accounting and getattr(exc, "usage", None) is not None: self.tokens += _normalize_completion_usage(exc.usage)[1] raise - self.tokens += _normalize_completion_usage(response.usage)[1] + if not native_accounting: + self.tokens += _normalize_completion_usage(response.usage)[1] return to_plain_data(response) finally: if notifications: diff --git a/tests/interfaces/test_member_authored_workflow_queued_budget.py b/tests/interfaces/test_member_authored_workflow_queued_budget.py new file mode 100644 index 00000000..21cda938 --- /dev/null +++ b/tests/interfaces/test_member_authored_workflow_queued_budget.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import asyncio +import json +import sys +import threading +from contextlib import asynccontextmanager + +import pytest + +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm import client as transport +from tests.interfaces.test_member_authored_workflow import _manager, _serve_model +from tests.support.paths import REPO_ROOT + + +@pytest.mark.parametrize("first_tokens, parent_stop, child_escape", [ + (5_000, None, False), (20_210, None, False), (5_000, "team", False), (5_000, "cancel", False), + (20_210, None, True), +]) +async def test_authored_child_stops_a_queued_request_after_shared_budget_exhaustion( + tmp_path, monkeypatch, first_tokens, parent_stop, child_escape, +): + monkeypatch.setenv("PYTHONPATH", str(REPO_ROOT)) + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + (tmp_path / "authored.py").write_text('''import asyncio +from pathlib import Path + +async def main(ctx, args): + first = asyncio.create_task(ctx.agent("first", budget=18000, tools=[])) + if args.get("escape"): + await first + while not Path("first-network").exists(): + await asyncio.sleep(0.01) + second = asyncio.create_task(ctx.agent("second", budget=2000, tools=[], over_budget_ok=args.get("escape", False))) + answers = await asyncio.gather(first, second) + return {"answers": answers, "reason": ctx.sessions[1].state.terminal_reason} +''', encoding="utf-8") + pool = tmp_path / "pool" + pool.mkdir() + release, first_entered, queued = threading.Event(), threading.Event(), asyncio.Event() + native_slot = transport.request_slot + entries = 0 + + @asynccontextmanager + async def observed_slot(*args): + nonlocal entries + entries += 1 + if entries == 2: + queued.set() + async with native_slot(*args): + yield + + monkeypatch.setattr(transport, "request_slot", observed_slot) + + def reply(body): + first = body["messages"][-1]["content"].startswith("first") + if first: + (tmp_path / "first-network").write_text("ready", encoding="utf-8") + first_entered.set() + assert release.wait(15) + total = first_tokens if first else 4 + return ({"role": "assistant", "content": "first answer" if first else "second answer"}, + {"prompt_tokens": total - 2, "completion_tokens": 2, "total_tokens": total}) + + environment = LocalEnvironment(str(tmp_path)) + with _serve_model(reply) as (url, requests): + parent, tool, runtime = _manager(environment, budget=20_000) + aggregate_exhausted, cancel = asyncio.Event(), asyncio.Event() + parent.runner._team_budget_exhausted = aggregate_exhausted.is_set + parent.runner.default_cancel_event = cancel + tool._config.base_url = parent.agent.base_url = url + tool._config.api_key = parent.agent.api_key = "controlled-test" # pragma: allowlist secret + tool._config.request_pool = parent.agent.request_pool = str(pool) + tool._config.request_capacity = parent.agent.request_capacity = 1 + parent.agent.max_tokens_per_step = 128 + try: + launched = json.loads(await tool.execute_with_runtime( + {"action": "launch", "source": "authored.py", "python": sys.executable, + "budget": 20_000, "concurrency": 2, "args": {"escape": child_escape}}, runtime, + )) + child = tool._children[launched["id"]] + if child_escape: + assert await asyncio.to_thread(first_entered.wait, 15) + else: + await asyncio.wait_for(queued.wait(), 15) + assert len(requests) == 1 + if parent_stop == "team": + aggregate_exhausted.set() + elif parent_stop == "cancel": + cancel.set() + release.set() + await asyncio.wait_for(child.task, 15) + cancelled = parent_stop == "cancel" and child.result["status"] == "stopped" + if cancelled: + assert child.result["reason"] == "cancelled" + assert child.result["output"] is None + assert len(requests) == 1 + else: + assert child.result["status"] == "completed", child.result + assert child.result["agent_failures"] == [] + if first_tokens >= 20_000 or parent_stop is not None: + assert len(requests) == 1 + assert child.result["output"]["answers"] == ["first answer", ""] + reason = {"team": "team budget exceeded", "cancel": "interrupted by user"}.get( + parent_stop, "budget exhausted before model call", + ) + assert reason in child.result["output"]["reason"] + else: + assert len(requests) == 2 + assert child.result["output"]["answers"] == ["first answer", "second answer"] + assert child.bridge.tokens == child.tokens == parent.used_tokens == first_tokens + 4 * (len(requests) - 1) + finally: + release.set() + await parent.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize("first_tokens, parent_budget", [(5_000, 20_000), (20_210, 20_000), (20_210, None)]) +async def test_parallel_authored_children_share_fresh_parent_budget( + tmp_path, monkeypatch, first_tokens, parent_budget, +): + monkeypatch.setenv("PYTHONPATH", str(REPO_ROOT)) + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + (tmp_path / "authored.py").write_text( + 'async def main(ctx, args):\n return await ctx.agent(args["prompt"], tools=[], budget=10000)\n', + encoding="utf-8", + ) + pool = tmp_path / "pool" + pool.mkdir() + first_entered, release = threading.Event(), threading.Event() + queued, released_slot = asyncio.Event(), asyncio.Event() + native_slot = transport.request_slot + entries = 0 + + @asynccontextmanager + async def ordered_slot(*args): + nonlocal entries + entries += 1 + index = entries + if index > 1: + queued.set() + await released_slot.wait() + async with native_slot(*args): + yield + if index == 1: + released_slot.set() + + monkeypatch.setattr(transport, "request_slot", ordered_slot) + + def reply(body): + first = body["messages"][-1]["content"].startswith("first") + if first: + first_entered.set() + assert release.wait(15) + total = first_tokens if first else 4 + return ({"role": "assistant", "content": "answer"}, + {"prompt_tokens": total - 2, "completion_tokens": 2, "total_tokens": total}) + + environment = LocalEnvironment(str(tmp_path)) + with _serve_model(reply) as (url, requests): + parent, tool, runtime = _manager(environment, budget=parent_budget) + tool._config.base_url = parent.agent.base_url = url + tool._config.api_key = parent.agent.api_key = "controlled-test" # pragma: allowlist secret + tool._config.request_pool = parent.agent.request_pool = str(pool) + tool._config.request_capacity = parent.agent.request_capacity = 1 + parent.agent.max_tokens_per_step = 128 + children = [] + try: + for prompt in ("first", "second"): + launched = json.loads(await tool.execute_with_runtime( + {"action": "launch", "source": "authored.py", "python": sys.executable, + "budget": 10_000, "args": {"prompt": prompt}}, runtime, + )) + children.append(tool._children[launched["id"]]) + if prompt == "first": + assert await asyncio.to_thread(first_entered.wait, 15) + await asyncio.wait_for(queued.wait(), 15) + assert len(requests) == 1 + release.set() + await asyncio.wait_for(asyncio.gather(*(child.task for child in children)), 20) + admitted = parent_budget is None or first_tokens < parent_budget + assert len(requests) == (2 if admitted else 1) + assert all(child.result["status"] == "completed" for child in children) + assert children[1].result["output"] == ("answer" if admitted else "") + assert children[0].tokens == children[0].bridge.tokens == first_tokens + assert children[1].tokens == children[1].bridge.tokens == (4 if admitted else 0) + assert parent.used_tokens == first_tokens + (4 if admitted else 0) + finally: + release.set() + await parent.aclose() + await environment.cleanup() diff --git a/tests/runtime/test_authored_request_budget_http.py b/tests/runtime/test_authored_request_budget_http.py new file mode 100644 index 00000000..eaa35023 --- /dev/null +++ b/tests/runtime/test_authored_request_budget_http.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager + +import pytest + +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm import client as transport +from opencollab.adapters.llm.client import LLMClient +from opencollab.adapters.llm.request_admission import request_slot +from opencollab.adapters.session_environments import SessionEnvironmentPool +from opencollab.application._session_run_shared import _TokenBudgetStop +from opencollab.bootstrap.authored_workflow_bridge import AuthoredWorkflowBridge +from tests.runtime.test_llm_chat_error_usage_http import ReportedUsageStream +from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport + + +@pytest.fixture(params=[ + ({"provider": "openai", "wire_protocol": "chat_completions", "model": "gpt-4o"}, "max_tokens"), + ({"provider": "openai", "wire_protocol": "responses", "model": "gpt-4o"}, "max_output_tokens"), + ({"provider": "anthropic", "model": "claude-sonnet-4-6"}, "max_tokens"), +], ids=["chat", "responses", "anthropic"]) +def configuration(request): + return request.param + + +@pytest.mark.parametrize("outcome", ["success", "ledger_error", "ledger_cancellation"]) +async def test_native_usage_is_reported_once_inside_the_slot( + tmp_path, monkeypatch, configuration, outcome, +): + config, _field = configuration + usages = [] + install_sdk_transport(monkeypatch, config["provider"], completion_http_response) + native_slot = transport.request_slot + + @asynccontextmanager + async def observed_slot(*args): + async with native_slot(*args): + try: + yield + finally: + assert [usage.total_tokens for usage in usages] == [4] + + async def record(**kwargs): + if outcome == "ledger_error": + raise ValueError("controlled ledger failure") + if outcome == "ledger_cancellation": + raise asyncio.CancelledError + + monkeypatch.setattr(transport, "request_slot", observed_slot) + monkeypatch.setattr(transport, "_record_api_usage_async", record) + async with LLMClient( + **config, api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", request_pool=str(tmp_path), request_capacity=1, + ) as client: + operation = client.complete([{"role": "user", "content": "finish"}], on_usage=usages.append) + if outcome == "success": + assert (await operation).content == "ok" + else: + with pytest.raises(ValueError if outcome == "ledger_error" else asyncio.CancelledError) as caught: + await operation + assert caught.value.usage.total_tokens == 4 + async with request_slot(str(tmp_path), 1): + assert len(usages) == 1 + + +@pytest.mark.parametrize("change", ["remaining", "reduced", "exhausted", "unbounded", "slow_ledger"]) +async def test_host_bridge_rechecks_queued_budget_before_native_dispatch( + tmp_path, monkeypatch, configuration, change, +): + config, field = configuration + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + entered, release, queued = asyncio.Event(), asyncio.Event(), asyncio.Event() + requests = [] + first_tokens = 5_000 if change == "remaining" else 20_210 + + async def handler(request): + body = json.loads(request.content) + requests.append(body) + if len(requests) == 1: + entered.set() + await release.wait() + total = first_tokens if len(requests) == 1 else 4 + response = completion_http_response(request) + if request.url.path.endswith("/responses"): + return http_response( + request, 200, headers={"content-type": "text/event-stream"}, + text=response.text.replace('"input_tokens": 2', f'"input_tokens": {total - 2}').replace( + '"total_tokens": 4', f'"total_tokens": {total}', + ), + ) + payload = response.json() + if config["provider"] == "anthropic": + payload["usage"]["input_tokens"] = total - 2 + else: + payload["usage"].update(prompt_tokens=total - 2, total_tokens=total) + return http_response(request, 200, json=payload) + + install_sdk_transport(monkeypatch, config["provider"], handler) + record = transport._record_api_usage_async + + async def delayed_record(**kwargs): + if change == "slow_ledger": + await asyncio.sleep(0.12) + await record(**kwargs) + + monkeypatch.setattr(transport, "_record_api_usage_async", delayed_record) + pool = tmp_path / "pool" + pool.mkdir() + artifacts = tmp_path / "artifacts" + (artifacts / "bridge").mkdir(parents=True) + environment = LocalEnvironment(str(tmp_path)) + config = {**config, "api_key": "controlled-test", # pragma: allowlist secret + "base_url": "https://controlled.invalid/v1", + "request_pool": str(pool), "request_capacity": 1, "max_output_tokens": 128} + bridge = AuthoredWorkflowBridge( + environment, str(artifacts), SessionEnvironmentPool(str(tmp_path), isolation="shared", + base_environment=environment), + config, None, allow_unisolated_shell=True, budget=None if change == "unbounded" else 20_000, + ) + second_messages = [{"role": "user", "content": "second"}] + if change == "reduced": + first_tokens = 20_000 - bridge._model().estimate_request_tokens(second_messages) - 50 + native_slot = transport.request_slot + entries = 0 + + @asynccontextmanager + async def observed_slot(*args): + nonlocal entries + entries += 1 + if entries == 2: + queued.set() + async with native_slot(*args): + yield + + monkeypatch.setattr(transport, "request_slot", observed_slot) + + async def complete(identity, messages): + return await bridge._dispatch(identity, {"action": "complete", "arguments": { + "messages": messages, "max_output_tokens": 128, + }}) + + first = asyncio.create_task(complete("1", [{"role": "user", "content": "first"}])) + second = None + try: + await asyncio.wait_for(entered.wait(), 5) + second = asyncio.create_task(complete("2", second_messages)) + await asyncio.wait_for(queued.wait(), 5) + assert len(requests) == 1 + release.set() + await asyncio.wait_for(first, 5) + if change in {"exhausted", "slow_ledger"}: + with pytest.raises(_TokenBudgetStop): + await asyncio.wait_for(second, 5) + assert len(requests) == 1 + assert bridge.tokens == first_tokens + else: + assert (await asyncio.wait_for(second, 5))["content"] == "ok" + assert len(requests) == 2 + assert requests[1][field] == (50 if change == "reduced" else 128) + assert bridge.tokens == first_tokens + 4 + finally: + release.set() + for task in (first, second): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather(*(task for task in (first, second) if task is not None), return_exceptions=True) + await bridge.aclose() + await environment.cleanup() + + +async def test_host_bridge_charges_reported_error_usage_once_before_rechecking(tmp_path, monkeypatch): + requests = [] + + async def handler(request): + requests.append(json.loads(request.content)) + return http_response( + request, 200, headers={"content-type": "text/event-stream"}, + stream=ReportedUsageStream({"prompt_tokens": 20_200, "completion_tokens": 10, "total_tokens": 20_210}, + disconnect=True), + ) + + install_sdk_transport(monkeypatch, "openai", handler) + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + environment = LocalEnvironment(str(tmp_path)) + artifacts = tmp_path / "artifacts" + (artifacts / "bridge").mkdir(parents=True) + bridge = AuthoredWorkflowBridge( + environment, str(artifacts), SessionEnvironmentPool(str(tmp_path), isolation="shared", + base_environment=environment), + {"model": "gpt-4o", "provider": "openai", + "api_key": "controlled-test", # pragma: allowlist secret + "base_url": "https://controlled.invalid/v1", "request_pool": str(tmp_path), "request_capacity": 1, + "llm_stream_chat": True, "llm_max_retries": 0}, + None, allow_unisolated_shell=True, budget=20_000, + ) + try: + with pytest.raises(Exception) as caught: + await bridge._dispatch("1", {"action": "complete", "arguments": { + "messages": [{"role": "user", "content": "first"}], "max_output_tokens": 128, + }}) + assert caught.value.usage.total_tokens == bridge.tokens == 20_210 + with pytest.raises(_TokenBudgetStop): + await bridge._dispatch("2", {"action": "complete", "arguments": { + "messages": [{"role": "user", "content": "second"}], "max_output_tokens": 128, + }}) + assert len(requests) == 1 and bridge.tokens == 20_210 + finally: + await bridge.aclose() + await environment.cleanup() From 881cf97262820b9240679a0874c8bcaaec1d89a2 Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 12:01:31 +0800 Subject: [PATCH 06/82] fix: restore candidate ancestor permissions after publication errors --- opencollab/adapters/_candidate_adoption.py | 19 ++++-- .../test_candidate_adoption_recovery.py | 68 +++++++++++++++++++ 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index 2dd7aeca..44eb6b1b 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -74,19 +74,30 @@ async def _adoption_transaction( restores = [] recreate_directories = [] restore_directory_modes = [] + restore_parents = {posixpath.dirname(path) for path in roots} # Git prunes empty ancestors after deleting their last leaf. Recreate # those directories with write access until their saved leaves return. for index, (path, mode) in enumerate(directory_modes): source = shlex.quote(posixpath.join(workspace, path)) - marker = shlex.quote(posixpath.join(backup, f"directory-{index}")) - copies.append(f"printf '%s\\0%s\\0' {source} {mode} > {marker}.mode || exit") + metadata_file = shlex.quote(posixpath.join(backup, f"directory-{index}.mode")) + copies.append(f"printf '%s\\0%s\\0' {source} {mode} > {metadata_file} || exit") recreate_directories.append( f"if [ ! -e {source} ] && [ ! -L {source} ]; then " - f"if mkdir -m 700 -- {source} && : > {marker}; then :; else failed=1; fi; " + f"mkdir -m 700 -- {source} || failed=1; " f"elif [ ! -d {source} ] || [ -L {source} ]; then failed=1; fi" ) + if path in restore_parents: + recreate_directories.append( + f"if [ -d {source} ] && [ ! -L {source} ] && " + f"[ ! -w {source} ] && [ -O {source} ]; then " + f"chmod -- u+w {source} || failed=1; fi" + ) restore_directory_modes.append( - f"if [ -f {marker} ]; then chmod -- {mode} {source} || failed=1; fi" + f"if [ -d {source} ] && [ ! -L {source} ] && " + f"{{ current=$(stat -c %a -- {source} 2>/dev/null) || " + f"current=$(stat -f %Lp -- {source}); }}; then " + f'[ "$current" = {mode} ] || chmod -- {mode} {source} || failed=1; ' + "else failed=1; fi" ) for index, path in enumerate(roots): source = shlex.quote(posixpath.join(workspace, path)) diff --git a/tests/workflows/test_candidate_adoption_recovery.py b/tests/workflows/test_candidate_adoption_recovery.py index 8242f8cd..e77eb675 100644 --- a/tests/workflows/test_candidate_adoption_recovery.py +++ b/tests/workflows/test_candidate_adoption_recovery.py @@ -3,6 +3,7 @@ import json import os import shlex +import shutil import stat import subprocess import sys @@ -165,6 +166,73 @@ async def flow(ctx, inputs): await environment.cleanup() +@pytest.mark.parametrize("physical", [False, True]) +@pytest.mark.parametrize("change,sibling", [("rename", False), ("rename", True), ("readonly", True)]) +async def test_public_sdk_restores_changed_ancestor_permissions(tmp_path, monkeypatch, physical, change, sibling): + repo = _repository(tmp_path, physical=physical, cache=True) + (repo / "old/nested").mkdir(parents=True) + helper = repo / "old/nested/helper.py" + helper.write_text("VALUE = 1\n") + _git(repo, "add", "old/nested/helper.py") + (repo / "old").chmod(0o750) + (repo / "old/nested").chmod(0o500 if change == "readonly" else 0o710) + if sibling: + (repo / "old/sibling.txt").write_text("shared contents\n") + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + commands = tmp_path / "commands" + commands.mkdir() + log = tmp_path / "chmod.log" + chmod = shutil.which("chmod") + assert chmod is not None + (commands / "chmod").write_text( + f'#!/bin/sh\nprintf \'%s\\0\' "$@" >> {shlex.quote(str(log))}\n' + f'exec {shlex.quote(chmod)} "$@"\n' + ) + (commands / "chmod").chmod(0o755) + execute = environment.exec_cmd + + async def recorded(command, timeout=120.0, **kwargs): + if command.startswith("failed=0;"): + command = f"PATH={shlex.quote(str(commands) + os.pathsep + os.environ['PATH'])}; " + command + return await execute(command, timeout, **kwargs) + + monkeypatch.setattr(environment, "exec_cmd", recorded) + + async def edit(child, inputs): + folder = Path(child.workspace_root) + if change == "rename": + (folder / "old/nested/helper.py").rename(folder / "old/nested/renamed.py") + else: + (folder / "old/nested/helper.py").write_text("VALUE = 42\n") + _flatten(folder) + assert _run(folder) == "42\n" + return "tested" + + async def flow(ctx, inputs): + candidate = await ctx.candidate_workflow(edit, {}, label="ancestor-permissions") + if change == "rename": + assert "rename from old/nested/helper.py" in candidate.effective_diff + with pytest.raises(RuntimeError, match="Directory not empty|Permission denied|Is a directory"): + await ctx.adopt_candidate(candidate) + assert _contents(repo) == before + assert not environment.revoked + return "recovered" + + try: + result = await OpenCollab(repo, environment=environment).workflow(flow, trace=False) + assert result.ok, result.reason + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").glob("candidate-recovery-*")) + chmod_paths = [Path(value.decode()) for value in log.read_bytes().split(b"\0")[2::3] if value] + assert repo / "old/nested" in chmod_paths + assert (repo / "old" in chmod_paths) is not sibling + finally: + (repo / "old/nested").chmod(0o700) + await environment.cleanup() + + async def test_git_write_failure_after_successful_check_restores_partial_application(tmp_path): repo = _repository(tmp_path) (repo / "z").mkdir(mode=0o555) From 070b77293b57faf4f635ad6750bc54858c8c8f67 Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 12:01:31 +0800 Subject: [PATCH 07/82] fix: serialize candidate publication and recovery --- opencollab/adapters/_candidate_increment.py | 7 +- .../test_candidate_adoption_concurrency.py | 148 ++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 tests/workflows/test_candidate_adoption_concurrency.py diff --git a/opencollab/adapters/_candidate_increment.py b/opencollab/adapters/_candidate_increment.py index 51ebe361..296b1d2f 100644 --- a/opencollab/adapters/_candidate_increment.py +++ b/opencollab/adapters/_candidate_increment.py @@ -12,6 +12,7 @@ from opencollab.adapters._candidate_physical import _physical_git from opencollab.adapters._git_patch_bytes import _complete, _patch_temporary from opencollab.adapters.git_patch import _git_command_and_config_guard +from opencollab.adapters.tools.base import host_write_lock from opencollab.patches import patch_paths @@ -32,7 +33,11 @@ async def _apply_increment( if not isinstance(patch, str) or not patch.strip(): raise ValueError("candidate patch must be non-empty") preserved = tuple(_safe_path(path) for path in preserve_paths) - async with _patch_temporary( + lock_root = ( + workspace if getattr(environment, "local_filesystem", False) + else getattr(environment, "workspace", workspace) + ) + async with host_write_lock(posixpath.join(lock_root, ".candidate-adoption"), environment), _patch_temporary( environment, workspace, patch, prefix=".candidate-adopt-", suffix=".patch", cleanup_best_effort=True, repository=repository, ) as candidate_file: diff --git a/tests/workflows/test_candidate_adoption_concurrency.py b/tests/workflows/test_candidate_adoption_concurrency.py new file mode 100644 index 00000000..eb98e7e2 --- /dev/null +++ b/tests/workflows/test_candidate_adoption_concurrency.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from contextvars import ContextVar +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters._env_local import LocalEnvironment +from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace +from tests.workflows.test_candidate_adoption_recovery import _repository, _run + +_ADOPTER = ContextVar("adopter", default="setup") + + +class _OrderedEnvironment(LocalEnvironment): + def __init__(self, root, *, remote): + super().__init__(str(root)) + self.local_filesystem = not remote + self.first_entered = asyncio.Event() + self.release_first = asyncio.Event() + self.publications = [] + self.checks = [] + self.remote_lock = asyncio.Lock() + self.remote_lock_held = False + self.lock_paths = [] + + @asynccontextmanager + async def file_write_lock(self, path): + assert Path(path).parent == Path(self.workspace) + assert not Path(path).exists() + self.lock_paths.append(path) + async with self.remote_lock: + self.remote_lock_held = True + try: + yield path + finally: + self.remote_lock_held = False + + async def exec_cmd(self, command, timeout=120.0, **kwargs): + if " apply --check --binary " in command or " apply --binary " in command: + if not self.local_filesystem: + assert self.remote_lock_held + label = _ADOPTER.get() + if " apply --check --binary " in command: + self.checks.append(label) + else: + self.publications.append(label) + if label == "A": + self.first_entered.set() + await self.release_first.wait() + return await super().exec_cmd(command, timeout, **kwargs) + + +class _QueuedCandidates(EnvCandidateWorkspace): + def __init__(self, environment): + super().__init__(environment) + self.second_requested = asyncio.Event() + + async def adopt_run(self, candidate, preserve_paths=()): + if candidate.label == "B": + self.second_requested.set() + await super().adopt_run(candidate, preserve_paths) + + +@pytest.mark.parametrize("remote", [False, True]) +@pytest.mark.parametrize("entry,cancel,overlap", [ + ("sdk", False, False), ("sdk", False, True), + ("direct", True, False), ("sdk", True, False), +]) +async def test_candidate_publication_serializes_snapshots_and_queued_cancellation( + tmp_path, remote, entry, cancel, overlap, +): + repo = _repository(tmp_path) + index = (repo / ".git/index").read_bytes() + environment = _OrderedEnvironment(repo, remote=remote) + backend = _QueuedCandidates(environment) + + async def edit(child, inputs): + (Path(child.workspace_root) / inputs["path"]).write_text(inputs["contents"]) + return "prepared" + + async def flow(ctx, inputs): + first = await ctx.candidate_workflow(edit, {"path": "app.py", "contents": "print(42)\n"}, label="A") + second_path = "app.py" if overlap else "other.py" + second = await ctx.candidate_workflow( + edit, {"path": second_path, "contents": "print(99)\n"}, label="B", + ) + + async def adopt(candidate, selected_entry): + token = _ADOPTER.set(candidate.label) + try: + if selected_entry == "direct": + await backend.adopt_run(candidate) + else: + await ctx.adopt_candidate(candidate) + finally: + _ADOPTER.reset(token) + + first_task = asyncio.create_task(adopt(first, "sdk")) + second_task = None + try: + await asyncio.wait_for(environment.first_entered.wait(), 10) + second_task = asyncio.create_task(adopt(second, entry)) + await asyncio.wait_for(backend.second_requested.wait(), 10) + await asyncio.sleep(0.05) + assert not second_task.done() + assert environment.checks == environment.publications == ["A"] + if cancel: + second_task.cancel() + await asyncio.sleep(0) + second_task.cancel() + environment.release_first.set() + await first_task + assert _run(repo) == "42\n" + if cancel: + with pytest.raises(asyncio.CancelledError): + await second_task + elif overlap: + with pytest.raises(RuntimeError, match="patch does not apply"): + await second_task + else: + await second_task + assert _run(repo) == "42\n" + expected_second = "VALUE = 1\n" if overlap or cancel and entry == "direct" else "print(99)\n" + assert (repo / "other.py").read_text() == expected_second + assert environment.publications == (["A"] if overlap or cancel and entry == "direct" else ["A", "B"]) + return "serialized" + finally: + environment.release_first.set() + await asyncio.gather(first_task, *(task for task in (second_task,) if task is not None), + return_exceptions=True) + + try: + result = await OpenCollab(repo, environment=environment).workflow( + flow, candidate_workspace=backend, trace=False, timeout=30, + ) + assert result.ok, result.reason + assert result.output == "serialized" + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not environment.revoked + if remote: + assert environment.lock_paths + finally: + await environment.cleanup() From add00cf9d9aa0e15701dba7e170bf407680254e4 Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 12:08:09 +0800 Subject: [PATCH 08/82] fix: recover candidate files in read-only workspace roots --- opencollab/adapters/_candidate_adoption.py | 4 ++-- tests/workflows/test_candidate_adoption_recovery.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index 44eb6b1b..81047c42 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -38,7 +38,7 @@ async def _adoption_transaction( for path in sorted(set(inspected.split("\0")) - {""}): if not any(path.startswith(root + "/") for root in roots): roots.append(path) - ancestors = set() + ancestors = {"."} for path in roots: parent = posixpath.dirname(path) while parent: @@ -74,7 +74,7 @@ async def _adoption_transaction( restores = [] recreate_directories = [] restore_directory_modes = [] - restore_parents = {posixpath.dirname(path) for path in roots} + restore_parents = {posixpath.dirname(path) or "." for path in roots} # Git prunes empty ancestors after deleting their last leaf. Recreate # those directories with write access until their saved leaves return. for index, (path, mode) in enumerate(directory_modes): diff --git a/tests/workflows/test_candidate_adoption_recovery.py b/tests/workflows/test_candidate_adoption_recovery.py index e77eb675..eb69bc03 100644 --- a/tests/workflows/test_candidate_adoption_recovery.py +++ b/tests/workflows/test_candidate_adoption_recovery.py @@ -167,7 +167,9 @@ async def flow(ctx, inputs): @pytest.mark.parametrize("physical", [False, True]) -@pytest.mark.parametrize("change,sibling", [("rename", False), ("rename", True), ("readonly", True)]) +@pytest.mark.parametrize("change,sibling", [ + ("rename", False), ("rename", True), ("readonly", True), ("readonly-root", True), +]) async def test_public_sdk_restores_changed_ancestor_permissions(tmp_path, monkeypatch, physical, change, sibling): repo = _repository(tmp_path, physical=physical, cache=True) (repo / "old/nested").mkdir(parents=True) @@ -178,6 +180,9 @@ async def test_public_sdk_restores_changed_ancestor_permissions(tmp_path, monkey (repo / "old/nested").chmod(0o500 if change == "readonly" else 0o710) if sibling: (repo / "old/sibling.txt").write_text("shared contents\n") + if change == "readonly-root": + repo.chmod(0o500) + root_mode = stat.S_IMODE(repo.stat().st_mode) before = _contents(repo) index = (repo / ".git/index").read_bytes() environment = LocalEnvironment(str(repo)) @@ -226,9 +231,12 @@ async def flow(ctx, inputs): assert (repo / ".git/index").read_bytes() == index assert not list((repo / ".git").glob("candidate-recovery-*")) chmod_paths = [Path(value.decode()) for value in log.read_bytes().split(b"\0")[2::3] if value] - assert repo / "old/nested" in chmod_paths + assert stat.S_IMODE(repo.stat().st_mode) == root_mode + assert (repo / "old/nested" in chmod_paths) is (change != "readonly-root") assert (repo / "old" in chmod_paths) is not sibling + assert (repo in chmod_paths) is (change == "readonly-root") finally: + repo.chmod(0o700) (repo / "old/nested").chmod(0o700) await environment.cleanup() From 760eb42fe3cb19e8b60ef62c924819929344756e Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 12:09:04 +0800 Subject: [PATCH 09/82] fix(workflow): charge authored usage before team admission --- opencollab/bootstrap/authored_workflow.py | 15 ++-- .../bootstrap/authored_workflow_bridge.py | 8 +- ..._member_authored_workflow_queued_budget.py | 78 ++++++++++++++++--- 3 files changed, 83 insertions(+), 18 deletions(-) diff --git a/opencollab/bootstrap/authored_workflow.py b/opencollab/bootstrap/authored_workflow.py index d5d55cf2..5a6941e3 100644 --- a/opencollab/bootstrap/authored_workflow.py +++ b/opencollab/bootstrap/authored_workflow.py @@ -55,6 +55,7 @@ class _Child: stop_requested: bool = False status: str = "running" tokens: int = 0 + reported_tokens: int = 0 result: dict[str, Any] | None = None cleanup_error: Exception | None = None bridge: AuthoredWorkflowBridge | None = None @@ -302,6 +303,7 @@ async def _launch(self, params: dict[str, Any], runtime: ToolRuntime) -> _Child: environment, artifacts, pool, config, runtime, budget=budget, remaining_budget=lambda: self._remaining_child_budget(child), + on_usage=lambda tokens: self._account_usage(child, tokens, reported=False), team_budget_exhausted=parent.runner._team_budget_exhausted, cancel_requested=lambda: child.stop_requested or parent.runner._run_cancellation_requested(), allow_unisolated_shell=self._allow_unisolated_shell @@ -329,15 +331,18 @@ async def _refresh_usage(self, child: _Child) -> None: return self._account_usage(child, usage["tokens"]) - def _account_usage(self, child: _Child, tokens: Any) -> None: - if isinstance(tokens, bool) or not isinstance(tokens, int) or tokens < child.tokens: + def _account_usage(self, child: _Child, tokens: Any, *, reported: bool = True) -> None: + minimum = child.reported_tokens if reported else child.tokens + if isinstance(tokens, bool) or not isinstance(tokens, int) or tokens < minimum: raise ValueError("invalid child usage total") - delta = tokens - child.tokens + if reported: + child.reported_tokens = tokens + delta = max(child.tokens, tokens) - child.tokens if delta: parent = self._get_session() parent.state.add_used_tokens(delta) parent.enqueue_auto_save() - child.tokens = tokens + child.tokens += delta async def _execute(self, child: _Child, command: str, request_bytes: bytes) -> None: execution = None @@ -390,7 +395,7 @@ async def _execute(self, child: _Child, command: str, request_bytes: bytes) -> N assert child.bridge is not None await await_owned_operation(child.bridge.aclose(), propagate_cancellation=False) await self._refresh_usage(child) - self._account_usage(child, max(child.tokens, child.bridge.tokens)) + self._account_usage(child, max(child.tokens, child.bridge.tokens), reported=False) except Exception as exc: status = "failed" child.cleanup_error = exc diff --git a/opencollab/bootstrap/authored_workflow_bridge.py b/opencollab/bootstrap/authored_workflow_bridge.py index a35934ee..5e4de1be 100644 --- a/opencollab/bootstrap/authored_workflow_bridge.py +++ b/opencollab/bootstrap/authored_workflow_bridge.py @@ -114,6 +114,7 @@ class AuthoredWorkflowBridge: def __init__(self, environment: Any, artifacts: str, pool: Any, llm: Any, runtime: Any, *, allow_unisolated_shell: bool, budget: int | None = None, remaining_budget: Callable[[], int | None] | None = None, + on_usage: Callable[[int], None] | None = None, team_budget_exhausted: Callable[[], bool] | None = None, cancel_requested: Callable[[], bool] | None = None) -> None: relative = posixpath.relpath(artifacts, environment.workspace) @@ -129,6 +130,7 @@ def __init__(self, environment: Any, artifacts: str, pool: Any, llm: Any, runtim self.tokens = 0 self._budget = budget self._remaining_budget = remaining_budget + self._on_usage = on_usage self._team_budget_exhausted = team_budget_exhausted self._cancel_requested = cancel_requested self._tasks: dict[str, asyncio.Task] = {} @@ -191,6 +193,8 @@ def _request_output_limit(self, arguments: dict[str, Any]) -> int: def _account_native_usage(self, usage: Any) -> None: self.tokens += _normalize_completion_usage(usage)[1] + if self._on_usage is not None: + self._on_usage(self.tokens) async def poll(self) -> None: try: @@ -299,10 +303,10 @@ def admitted() -> None: response = await self._model().complete(**arguments) except BaseException as exc: if not native_accounting and getattr(exc, "usage", None) is not None: - self.tokens += _normalize_completion_usage(exc.usage)[1] + self._account_native_usage(exc.usage) raise if not native_accounting: - self.tokens += _normalize_completion_usage(response.usage)[1] + self._account_native_usage(response.usage) return to_plain_data(response) finally: if notifications: diff --git a/tests/interfaces/test_member_authored_workflow_queued_budget.py b/tests/interfaces/test_member_authored_workflow_queued_budget.py index 21cda938..df936bc1 100644 --- a/tests/interfaces/test_member_authored_workflow_queued_budget.py +++ b/tests/interfaces/test_member_authored_workflow_queued_budget.py @@ -10,6 +10,10 @@ from opencollab.adapters.env import LocalEnvironment from opencollab.adapters.llm import client as transport +from opencollab.application.scheduler import Scheduler +from opencollab.bootstrap.authored_workflow import _Child +from opencollab.bootstrap.session_factory import _team_budget_guard +from opencollab.domain.scheduler import SessionControlBlock from tests.interfaces.test_member_authored_workflow import _manager, _serve_model from tests.support.paths import REPO_ROOT @@ -119,8 +123,9 @@ def reply(body): @pytest.mark.parametrize("first_tokens, parent_budget", [(5_000, 20_000), (20_210, 20_000), (20_210, None)]) +@pytest.mark.parametrize("separate_parents", [False, True], ids=["one-role", "two-team-roles"]) async def test_parallel_authored_children_share_fresh_parent_budget( - tmp_path, monkeypatch, first_tokens, parent_budget, + tmp_path, monkeypatch, first_tokens, parent_budget, separate_parents, ): monkeypatch.setenv("PYTHONPATH", str(REPO_ROOT)) monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) @@ -150,8 +155,11 @@ async def ordered_slot(*args): monkeypatch.setattr(transport, "request_slot", ordered_slot) + observed_spend = [] + def reply(body): first = body["messages"][-1]["content"].startswith("first") + observed_spend.append(sum(parent.used_tokens for parent, _tool, _runtime in managers)) if first: first_entered.set() assert release.wait(15) @@ -160,16 +168,29 @@ def reply(body): {"prompt_tokens": total - 2, "completion_tokens": 2, "total_tokens": total}) environment = LocalEnvironment(str(tmp_path)) + managers = [] + scheduler = Scheduler(session_factory=None, worktree_pool=None, event_sink=None, max_budget_tokens=20_000) with _serve_model(reply) as (url, requests): - parent, tool, runtime = _manager(environment, budget=parent_budget) - tool._config.base_url = parent.agent.base_url = url - tool._config.api_key = parent.agent.api_key = "controlled-test" # pragma: allowlist secret - tool._config.request_pool = parent.agent.request_pool = str(pool) - tool._config.request_capacity = parent.agent.request_capacity = 1 - parent.agent.max_tokens_per_step = 128 + allowance = 10_000 if separate_parents and parent_budget is not None else parent_budget + for _ in range(2 if separate_parents else 1): + parent, tool, runtime = _manager(environment, budget=allowance) + if separate_parents: + parent.runner._team_budget_exhausted = _team_budget_guard(scheduler) + aid = scheduler.table.allocate_aid() + scheduler.table.add(SessionControlBlock( + aid=aid, parent_aid=None, agent=parent.agent, state=parent.state, + )) + scheduler._sessions[aid] = parent + tool._config.base_url = parent.agent.base_url = url + tool._config.api_key = parent.agent.api_key = "controlled-test" # pragma: allowlist secret + tool._config.request_pool = parent.agent.request_pool = str(pool) + tool._config.request_capacity = parent.agent.request_capacity = 1 + parent.agent.max_tokens_per_step = 128 + managers.append((parent, tool, runtime)) children = [] try: - for prompt in ("first", "second"): + for index, prompt in enumerate(("first", "second")): + parent, tool, runtime = managers[index if separate_parents else 0] launched = json.loads(await tool.execute_with_runtime( {"action": "launch", "source": "authored.py", "python": sys.executable, "budget": 10_000, "args": {"prompt": prompt}}, runtime, @@ -181,14 +202,49 @@ def reply(body): assert len(requests) == 1 release.set() await asyncio.wait_for(asyncio.gather(*(child.task for child in children)), 20) - admitted = parent_budget is None or first_tokens < parent_budget + admitted = (parent_budget is None and not separate_parents) or first_tokens < 20_000 assert len(requests) == (2 if admitted else 1) assert all(child.result["status"] == "completed" for child in children) assert children[1].result["output"] == ("answer" if admitted else "") assert children[0].tokens == children[0].bridge.tokens == first_tokens assert children[1].tokens == children[1].bridge.tokens == (4 if admitted else 0) - assert parent.used_tokens == first_tokens + (4 if admitted else 0) + total = first_tokens + (4 if admitted else 0) + assert sum(parent.used_tokens for parent, _tool, _runtime in managers) == total + if admitted: + assert observed_spend[1] == first_tokens + if separate_parents: + assert scheduler.used_tokens == first_tokens + (4 if admitted else 0) finally: release.set() - await parent.aclose() + for parent, _tool, _runtime in managers: + await parent.aclose() await environment.cleanup() + + +async def test_host_usage_and_stale_worker_reports_are_deduplicated(tmp_path): + environment = LocalEnvironment(str(tmp_path)) + artifacts = tmp_path / "child" + artifacts.mkdir() + parent, tool, _runtime = _manager(environment, budget=1_000) + child = _Child("child", environment, str(artifacts), 400) + try: + tool._account_usage(child, 90, reported=False) + assert parent.used_tokens == child.tokens == 90 + assert child.reported_tokens == 0 + for reported in (0, 30, 30, 60, 90): + (artifacts / "usage.json").write_text(json.dumps({"tokens": reported}), encoding="utf-8") + await tool._refresh_usage(child) + assert parent.used_tokens == child.tokens == 90 + assert child.reported_tokens == reported + tool._account_usage(child, 100, reported=False) + tool._account_usage(child, 90) + assert child.reported_tokens == 90 + tool._account_usage(child, 100) + assert parent.used_tokens == child.tokens == child.reported_tokens == 100 + for invalid in (99, -1, True, 100.0, "100"): + with pytest.raises(ValueError, match="invalid child usage total"): + tool._account_usage(child, invalid) + assert parent.used_tokens == child.tokens == child.reported_tokens == 100 + finally: + await parent.aclose() + await environment.cleanup() From b91634c1685b7eb53b9691b4af1f6c2f54d16f93 Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 12:42:39 +0800 Subject: [PATCH 10/82] fix: preserve provider sentinels and POSIX test environments --- opencollab/adapters/llm/client.py | 6 +++--- tests/workflows/test_candidate_symlink_delivery.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index 92ed8972..c73a6652 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -390,7 +390,7 @@ async def complete( first_event_timeout=self.first_event_timeout, stream_idle_timeout=self.stream_idle_timeout, ) - settled_usage = response.usage + settled_usage = getattr(response, "usage", None) transport_timing = first_token.snapshot() try: response.transport_timing = transport_timing @@ -411,9 +411,9 @@ async def complete( ) return response except BaseException as exc: - settled_usage = response.usage if response is not None else getattr(exc, "usage", None) + settled_usage = getattr(response if response is not None else exc, "usage", None) if response is not None: - exc.usage = response.usage + exc.usage = settled_usage else: try: await _record_api_usage_async( diff --git a/tests/workflows/test_candidate_symlink_delivery.py b/tests/workflows/test_candidate_symlink_delivery.py index f7de2725..d60ec425 100644 --- a/tests/workflows/test_candidate_symlink_delivery.py +++ b/tests/workflows/test_candidate_symlink_delivery.py @@ -181,7 +181,7 @@ async def test_link_probe_without_bash_keeps_executable_physical_delivery(tmp_pa binaries = tmp_path / "commands" binaries.mkdir() for name in ("git", "mktemp", "rm", "cat", "grep", "mkdir", "sh", "sed", "awk", "basename", - "dirname", "uname", "expr", "tr", "wc", "cut", "pwd", "sort", "cp"): + "dirname", "uname", "expr", "tr", "wc", "cut", "pwd", "sort", "cp", "stat", "mv", "chmod"): command = shutil.which(name) assert command is not None (binaries / name).symlink_to(command) From c671a3be8690e0ddb03d76504a0753f960f1676e Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 13:48:09 +0800 Subject: [PATCH 11/82] fix(scheduler): wait for cross-agent budget cleanup before stalling --- opencollab/application/_scheduler_run.py | 51 +++- ...st_scheduler_cross_agent_budget_cleanup.py | 223 ++++++++++++++++++ 2 files changed, 266 insertions(+), 8 deletions(-) create mode 100644 tests/workflows/test_scheduler_cross_agent_budget_cleanup.py diff --git a/opencollab/application/_scheduler_run.py b/opencollab/application/_scheduler_run.py index 8d838fa7..b69e6231 100644 --- a/opencollab/application/_scheduler_run.py +++ b/opencollab/application/_scheduler_run.py @@ -381,6 +381,19 @@ async def _settle_cancelled_suspended_turn(self, aid: int) -> None: ) self._write_manifest() + def _message_budget_cleanup_tasks(self) -> set[asyncio.Task]: + if not any( + inbox and aid not in self._cancelled_turn_inbox_holds + for aid, inbox in self._message_inbox.items() + ): + return set() + # Any member's deferred grant can hold the shared budget an inbox needs. + return { + task + for _aid, _grant, _baseline, tasks, _finished in self._pending_provider_budget_leases.values() + for task in tasks + } + def _active_scheduler_tasks(self) -> set[asyncio.Task]: """Return live scheduler-owned producers, excluding the current waiter.""" current = asyncio.current_task() @@ -401,20 +414,42 @@ def _active_scheduler_tasks(self) -> set[asyncio.Task]: *self._startup_tasks.values(), *self._message_delivery_tasks.values(), *message_cleanup, + *self._message_budget_cleanup_tasks(), ) if not task.done() and task is not current } async def _wait_for_scheduler_progress(self, aid: int) -> None: """Block for a producer completion or report a state with no producer.""" - await self._drain_ready_message_inboxes() - scb = self.table.get(aid) - if scb is not None and scb.state.phase.is_terminal() and self._quiescent(): - return - producers = self._active_scheduler_tasks() - if not producers: - raise SchedulerStalledError(aid) - await asyncio.wait(producers, return_when=asyncio.FIRST_COMPLETED) + settled_budget_owners: set[asyncio.Task] = set() + while True: + budget_owners = self._message_budget_cleanup_tasks() + await self._drain_ready_message_inboxes() + scb = self.table.get(aid) + if scb is not None and scb.state.phase.is_terminal() and self._quiescent(): + return + producers = self._active_scheduler_tasks() + if producers: + await asyncio.wait(producers, return_when=asyncio.FIRST_COMPLETED) + return + budget_owners.update(self._message_budget_cleanup_tasks()) + callbacks = budget_owners - settled_budget_owners + if not callbacks: + raise SchedulerStalledError(aid) + # A done task can still hold its grant until the registered usage + # and reservation callbacks run. Follow them with an owned callback. + callback_tail = asyncio.get_running_loop().create_future() + remaining = set(callbacks) + + def after_budget_callback(task: asyncio.Task) -> None: + remaining.discard(task) + if not remaining and not callback_tail.done(): + callback_tail.set_result(None) + + for task in callbacks: + task.add_done_callback(after_budget_callback) + await callback_tail + settled_budget_owners.update(callbacks) def _quiescent(self) -> bool: """True when no session is mid-flight: none is awaiting events, none has diff --git a/tests/workflows/test_scheduler_cross_agent_budget_cleanup.py b/tests/workflows/test_scheduler_cross_agent_budget_cleanup.py new file mode 100644 index 00000000..881f152d --- /dev/null +++ b/tests/workflows/test_scheduler_cross_agent_budget_cleanup.py @@ -0,0 +1,223 @@ +"""Accepted messages share progress with another agent's deferred budget owner.""" + +from __future__ import annotations + +import asyncio +import json +from types import SimpleNamespace + +import pytest + +from opencollab.adapters.llm.client import LLMClient +from opencollab.adapters.worktree_pool import WorktreePool +from opencollab.application.event_bus import EventBus +from opencollab.application.scheduler import Scheduler, SchedulerTurnError +from opencollab.bootstrap import build_session +from opencollab.domain.agent import Agent +from opencollab.domain.scheduler import SessionControlBlock +from opencollab.domain.session import SessionPhase, SessionState +from tests.support.provider_sdk_http import completion_http_response, install_sdk_transport + + +class _Factory: + def __init__(self, client, timeout): + self.client = client + self.timeout = timeout + + def build_spawn_session(self, *, role, budget, aid, task, **kwargs): + return build_session( + agent=Agent(name=role, system_prompt=role), llm=self.client, + env=kwargs["env"], aid=aid, max_budget_tokens=budget, llm_timeout=self.timeout, + seed_user_messages=[{"role": "user", "content": task}], + ) + + +@pytest.mark.parametrize("owner_aid", [0, 2]) +@pytest.mark.parametrize("ending", ["success", "cancelled", "late_success", "late_error"]) +async def test_accepted_inbox_waits_for_another_agents_budget_owner( + monkeypatch, tmp_path, owner_aid, ending, +): + started = asyncio.Event() + cancelled = asyncio.Event() + release = asyncio.Event() + waiting_for_budget_owner = asyncio.Event() + requests = [] + + async def handler(request): + body = json.loads(request.content) + requests.append(body) + latest_user = next(m["content"] for m in reversed(body["messages"]) if m["role"] == "user") + if latest_user == "owner operation": + started.set() + try: + await release.wait() + except asyncio.CancelledError: + cancelled.set() + if ending == "cancelled": + raise + await release.wait() + if ending == "late_error": + raise RuntimeError("late provider failure") + return completion_http_response(request, text="owner answer") + if latest_user == "lead warmup": + return completion_http_response(request, output_tokens=19_998, text="lead ready") + if latest_user == "owner warmup": + return completion_http_response(request, output_tokens=9_998, text="owner ready") + return completion_http_response(request, text="recipient processed") + + install_sdk_transport(monkeypatch, "openai", handler) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", "/dev/null") + async with LLMClient( + model="gpt-4o", provider="openai", api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + ) as client: + total = 400_000 if owner_aid == 0 else 40_000 + timeout = 1.0 if ending == "success" else 0.05 + scheduler = Scheduler( + session_factory=_Factory(client, 1.0), + worktree_pool=WorktreePool(str(tmp_path), use_worktrees=False), + event_sink=EventBus(), max_budget_tokens=total, + ) + lead = build_session( + agent=Agent(name="lead", system_prompt="lead"), llm=client, + max_budget_tokens=total, llm_timeout=timeout, + ) + scheduler.register_lead(lead) + if owner_aid: + assert await scheduler.run("lead warmup") == "lead ready" + recipient = await scheduler.spawn(0, "recipient", "recipient warmup") + await scheduler._tasks[recipient] + recipient_session = scheduler._sessions[recipient] + assert recipient_session.phase is SessionPhase.DONE, recipient_session.state.terminal_reason + if owner_aid: + assert await scheduler.spawn(0, "owner", "owner warmup") == owner_aid + await scheduler._tasks[owner_aid] + owner = scheduler._sessions[owner_aid] + owner.runner._per_call_timeout = timeout + original_active = scheduler._active_scheduler_tasks + + def observe_progress(): + pending = original_active() + if ( + scheduler._pending_provider_budget_leases + and not any(not task.done() for task in scheduler._tasks.values()) + and any(task in pending for task in owner.runner.pending_cleanup_tasks) + ): + waiting_for_budget_owner.set() + return pending + + monkeypatch.setattr(scheduler, "_active_scheduler_tasks", observe_progress) + active = asyncio.create_task(scheduler.run_turn(owner_aid, "owner operation")) + try: + await asyncio.wait_for(started.wait(), 2) + assert scheduler.allocated_tokens == total + assert await scheduler.send_message(0, recipient, "work", "accepted follow-up") == ( + "Message queued to aid 1." + ) + assert len(scheduler._message_inbox[recipient]) == 1 + if ending == "success": + release.set() + assert await asyncio.wait_for(active, 2) == "owner answer" + else: + await asyncio.wait_for(cancelled.wait(), 2) + if ending != "cancelled": + await asyncio.wait_for(waiting_for_budget_owner.wait(), 2) + assert not active.done() + assert scheduler.allocated_tokens == total + assert len(scheduler._message_inbox[recipient]) == 1 + release.set() + with pytest.raises(SchedulerTurnError) as failure: + await asyncio.wait_for(active, 2) + assert failure.value.aid == owner_aid + assert failure.value.phase is SessionPhase.ERROR + assert "GenerationTimeoutError" in failure.value.terminal_reason + + recipient_requests = [r for r in requests if r["messages"][0]["content"] == "recipient"] + assert len(requests) == (3 if owner_aid == 0 else 5) + assert len(recipient_requests) == 2 + assert "accepted follow-up" in str(recipient_requests[-1]) + assert not scheduler._message_inbox[recipient] + assert scheduler._sessions[recipient].phase is SessionPhase.DONE + assert not owner.runner.pending_cleanup_tasks + assert scheduler._pending_provider_budget_leases == {} + assert scheduler.allocated_tokens == scheduler.used_tokens + assert scheduler.used_tokens == ( + (0 if owner_aid == 0 else 30_000) + + 8 + (4 if ending in {"success", "late_success"} else 0) + ) + assert owner.runner.late_provider_usage == ((4,) if ending == "late_success" else ()) + if ending.startswith("late"): + assert "owner answer" not in str(owner.messages) + finally: + release.set() + await scheduler.cleanup() + await asyncio.gather(active, return_exceptions=True) + + +def _budget_callback_scheduler(tmp_path, owner): + state = SessionState(messages=[]) + state.set_phase(SessionPhase.DONE) + lead = SimpleNamespace( + agent=Agent(name="lead", system_prompt="lead"), state=state, + runner=SimpleNamespace(pending_cleanup_tasks=(owner,)), used_tokens=0, + ) + scheduler = Scheduler( + session_factory=_Factory(None, 1.0), + worktree_pool=WorktreePool(str(tmp_path), use_worktrees=False), + event_sink=EventBus(), max_budget_tokens=100_000, + ) + scheduler.register_lead(lead) + scheduler._reserve_turn_lease(0) + recipient_state = SessionState(messages=[]) + recipient_state.set_phase(SessionPhase.DONE) + scheduler.table.add(SessionControlBlock( + aid=1, parent_aid=0, agent=Agent(name="recipient", system_prompt="recipient"), + state=recipient_state, + )) + return scheduler, state + + +async def test_done_provider_callback_releases_budget_before_the_next_inbox_drain(monkeypatch, tmp_path): + async def completed_owner(): + return None + + owner = asyncio.create_task(completed_owner()) + await owner + scheduler, state = _budget_callback_scheduler(tmp_path, owner) + owner.add_done_callback(lambda _task: state.add_used_tokens(17)) + assert scheduler._defer_budget_lease_for_provider_cleanup(0) + scheduler._message_inbox[1] = [object()] + observations = [] + + async def drain_after_budget_release(): + observations.append(scheduler.allocated_tokens) + if scheduler.allocated_tokens < scheduler._max_budget_tokens: + scheduler._message_inbox[1].clear() + + monkeypatch.setattr(scheduler, "_drain_ready_message_inboxes", drain_after_budget_release) + assert not scheduler._active_scheduler_tasks() + assert scheduler._pending_provider_budget_leases + await scheduler._wait_for_scheduler_progress(0) + assert observations == [100_000, 17] + assert not scheduler._message_inbox[1] + assert not scheduler._pending_provider_budget_leases + assert scheduler.allocated_tokens == scheduler.used_tokens == 17 + + +async def test_deferred_budget_owner_counts_as_progress_only_for_an_unheld_inbox(tmp_path): + release = asyncio.Event() + owner = asyncio.create_task(release.wait()) + scheduler, _state = _budget_callback_scheduler(tmp_path, owner) + assert scheduler._defer_budget_lease_for_provider_cleanup(0) + try: + assert not scheduler._active_scheduler_tasks() + scheduler._message_inbox[1] = [object()] + scheduler._cancelled_turn_inbox_holds.add(1) + assert not scheduler._active_scheduler_tasks() + scheduler._cancelled_turn_inbox_holds.clear() + assert scheduler._active_scheduler_tasks() == {owner} + finally: + release.set() + await owner From f41ed4d608ddeb041624a3763204ce9e522634cc Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 13:50:44 +0800 Subject: [PATCH 12/82] fix(workflow): share candidate and structured edit ownership Hold candidate publication and recovery under the same target and ancestor locks used by structured file edits. Group bridge ownership in one native task and retain one workspace lock for VM edits. Cover SDK shared agents, native and bridge publication, directory rollback, cancellation, Docker execution wrappers, and VM workspace locking with executable regressions. --- opencollab/adapters/_candidate_increment.py | 72 +++---- opencollab/adapters/_env_vm.py | 1 + opencollab/adapters/tools/base.py | 49 ++++- .../bootstrap/authored_workflow_bridge.py | 32 ++-- tests/runtime/test_vm_mutation_locks.py | 88 +++++++++ .../test_candidate_adoption_concurrency.py | 2 + .../test_candidate_mutation_backends.py | 176 ++++++++++++++++++ .../test_candidate_shared_mutations.py | 139 ++++++++++++++ 8 files changed, 511 insertions(+), 48 deletions(-) create mode 100644 tests/runtime/test_vm_mutation_locks.py create mode 100644 tests/workflows/test_candidate_mutation_backends.py create mode 100644 tests/workflows/test_candidate_shared_mutations.py diff --git a/opencollab/adapters/_candidate_increment.py b/opencollab/adapters/_candidate_increment.py index 296b1d2f..244276bc 100644 --- a/opencollab/adapters/_candidate_increment.py +++ b/opencollab/adapters/_candidate_increment.py @@ -12,7 +12,7 @@ from opencollab.adapters._candidate_physical import _physical_git from opencollab.adapters._git_patch_bytes import _complete, _patch_temporary from opencollab.adapters.git_patch import _git_command_and_config_guard -from opencollab.adapters.tools.base import host_write_lock +from opencollab.adapters.tools.base import host_write_locks from opencollab.patches import patch_paths @@ -37,7 +37,7 @@ async def _apply_increment( workspace if getattr(environment, "local_filesystem", False) else getattr(environment, "workspace", workspace) ) - async with host_write_lock(posixpath.join(lock_root, ".candidate-adoption"), environment), _patch_temporary( + async with _patch_temporary( environment, workspace, patch, prefix=".candidate-adopt-", suffix=".patch", cleanup_best_effort=True, repository=repository, ) as candidate_file: @@ -54,35 +54,39 @@ async def _apply_increment( for protected in preserved for path in candidate_paths ): raise ValueError("candidate patch overlaps a preserved path") - git = "git -c core.filemode=true" - guard = "" - if strict and repository: - git, guard = _git_command_and_config_guard() - async with AsyncExitStack() as resources: - command = f"cd -- {shlex.quote(workspace)} && {guard}{git}" - operation = "candidate adoption" - if physical_paths and repository: - if guard: - _complete(await environment.exec_cmd( - f"cd -- {shlex.quote(workspace)} && {guard}:", timeout=120, - ), "physical candidate configuration inspection") - command, _ = await resources.enter_async_context(_physical_git( - environment, workspace, physical_paths, strict=strict, - )) - operation = "physical candidate adoption" - # Let Git reject unsafe path aliases before a recursive backup can - # encounter repository metadata. Publication can still fail later. - _complete(await environment.exec_cmd( - f"{command} apply --check --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}", - timeout=120, - ), operation) - async with _adoption_transaction( - environment, workspace, sorted(candidate_paths), repository=repository, - ): - try: - _complete(await environment.exec_cmd( - f"{command} apply --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}", - timeout=120, - ), operation) - finally: - await resources.aclose() + async with host_write_locks( + (posixpath.join(lock_root, ".candidate-adoption"), + *(posixpath.join(workspace, path) for path in sorted(candidate_paths))), environment, + ): + git = "git -c core.filemode=true" + guard = "" + if strict and repository: + git, guard = _git_command_and_config_guard() + async with AsyncExitStack() as resources: + command = f"cd -- {shlex.quote(workspace)} && {guard}{git}" + operation = "candidate adoption" + if physical_paths and repository: + if guard: + _complete(await environment.exec_cmd( + f"cd -- {shlex.quote(workspace)} && {guard}:", timeout=120, + ), "physical candidate configuration inspection") + command, _ = await resources.enter_async_context(_physical_git( + environment, workspace, physical_paths, strict=strict, + )) + operation = "physical candidate adoption" + # Let Git reject unsafe path aliases before a recursive backup can + # encounter repository metadata. Publication can still fail later. + _complete(await environment.exec_cmd( + f"{command} apply --check --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}", + timeout=120, + ), operation) + async with _adoption_transaction( + environment, workspace, sorted(candidate_paths), repository=repository, + ): + try: + _complete(await environment.exec_cmd( + f"{command} apply --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}", + timeout=120, + ), operation) + finally: + await resources.aclose() diff --git a/opencollab/adapters/_env_vm.py b/opencollab/adapters/_env_vm.py index c83a5da3..2ddbec44 100644 --- a/opencollab/adapters/_env_vm.py +++ b/opencollab/adapters/_env_vm.py @@ -57,6 +57,7 @@ class VMEnvironment(Environment): """ process_isolated = True + file_write_lock_scope = "workspace" def __init__(self, connection: VMConnection, workspace: str, *, state_directory: str, run_id: str | None = None): super().__init__() diff --git a/opencollab/adapters/tools/base.py b/opencollab/adapters/tools/base.py index 59c5c33f..f9a04301 100644 --- a/opencollab/adapters/tools/base.py +++ b/opencollab/adapters/tools/base.py @@ -6,8 +6,9 @@ import contextlib import hashlib import os +import posixpath import tempfile -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Sequence from typing import Any from filelock import FileLock, Timeout @@ -29,9 +30,53 @@ def _host_lock_path(path: str, env: Any) -> str: digest = hashlib.sha256(identity.encode("utf-8", errors="surrogatepass")).hexdigest() return os.path.join(_host_lock_root(), f"{digest}.lock") +def _write_lock_paths(paths: Sequence[str], env: Any) -> list[str]: + local = bool(getattr(env, "local_filesystem", False)) + path_module = os.path if local else posixpath + workspace = path_module.abspath(getattr(env, "workspace", "") or ".") + if local: + workspace = os.path.realpath(workspace) + targets = set() + for path in paths: + target = path_module.normpath(path_module.join(workspace, path)) + if local: + target = os.path.join(os.path.realpath(os.path.dirname(target)), os.path.basename(target)) + targets.add(target) + parent = path_module.dirname(target) + while parent != workspace and parent.startswith(workspace.rstrip("/") + "/"): + targets.add(parent) + parent = path_module.dirname(parent) + return sorted(targets) + + +@contextlib.asynccontextmanager +async def host_write_locks(paths: Sequence[str], env: Any) -> AsyncIterator[Any]: + """Own edited paths and their ancestors through publication or recovery.""" + if env is not None and not getattr(env, "local_filesystem", False): + grouped = getattr(env, "file_write_locks", None) + if callable(grouped): + async with grouped(paths) as target: + yield target + return + if getattr(env, "file_write_lock_scope", None) == "workspace": + async with _target_write_lock(paths[0], env) as target: + yield target + return + async with contextlib.AsyncExitStack() as locks: + target = None + for path in _write_lock_paths(paths, env): + target = await locks.enter_async_context(_target_write_lock(path, env)) + yield target + + @contextlib.asynccontextmanager async def host_write_lock(path: str, env: Any) -> AsyncIterator[Any]: - """Hold the environment's file lock across the complete native edit.""" + async with host_write_locks((path,), env) as target: + yield target + + +@contextlib.asynccontextmanager +async def _target_write_lock(path: str, env: Any) -> AsyncIterator[Any]: if env is not None and not getattr(env, "local_filesystem", False): environment_lock = getattr(env, "file_write_lock", None) if callable(environment_lock): diff --git a/opencollab/bootstrap/authored_workflow_bridge.py b/opencollab/bootstrap/authored_workflow_bridge.py index 5e4de1be..e6d560f2 100644 --- a/opencollab/bootstrap/authored_workflow_bridge.py +++ b/opencollab/bootstrap/authored_workflow_bridge.py @@ -7,7 +7,7 @@ import math import posixpath import time -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Sequence from contextlib import asynccontextmanager from contextvars import ContextVar from dataclasses import asdict @@ -18,7 +18,7 @@ from opencollab.adapters.llm import errors as llm_errors from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.llm.types import LLMResponse, Usage, to_plain_data -from opencollab.adapters.tools.base import host_write_lock +from opencollab.adapters.tools.base import host_write_locks from opencollab.application._session_run_shared import _TeamBudgetStop, _TokenBudgetStop from opencollab.application._session_run_usage import _normalize_completion_usage from opencollab.application.async_timeout import await_owned_operation, consume_task_result @@ -59,16 +59,16 @@ def _restore_error(value: dict[str, Any]) -> Exception: class _FileEdit: """Run native operations in the task holding their environment's edit lock.""" - def __init__(self, environment: Any, path: str) -> None: + def __init__(self, environment: Any, paths: Sequence[str]) -> None: self.environment = environment self.ready = asyncio.get_running_loop().create_future() self.ready.add_done_callback(consume_task_result) self.calls: asyncio.Queue[tuple[str, dict[str, Any], asyncio.Future[Any]] | None] = asyncio.Queue() - self.task = asyncio.create_task(self._run(path)) + self.task = asyncio.create_task(self._run(paths)) - async def _run(self, path: str) -> None: + async def _run(self, paths: Sequence[str]) -> None: try: - async with host_write_lock(path, self.environment): + async with host_write_locks(paths, self.environment): self.ready.set_result(None) while (call := await self.calls.get()) is not None: action, arguments, reply = call @@ -255,10 +255,13 @@ async def _dispatch(self, identity: str, request: dict[str, Any]) -> Any: if isinstance(handle, bool) or not isinstance(handle, int) or handle not in self._environments: raise ValueError("unknown environment for this authored run") environment = self._environments[handle] - if action == "file_write_lock": - if set(arguments) != {"path"}: - raise ValueError("file edit lock needs a path") - edit = _FileEdit(environment, arguments["path"]) + if action in {"file_write_lock", "file_write_locks"}: + if set(arguments) != ({"path"} if action == "file_write_lock" else {"paths"}): + raise ValueError("file edit lock needs paths") + paths = [arguments["path"]] if action == "file_write_lock" else arguments["paths"] + if not isinstance(paths, list) or not paths or any(not isinstance(path, str) for path in paths): + raise ValueError("file edit lock needs non-empty text paths") + edit = _FileEdit(environment, paths) self._edits[identity] = edit try: await asyncio.shield(edit.ready) @@ -437,7 +440,7 @@ async def acknowledged() -> Any: usage = getattr(exc, "usage", None) else: usage = Usage(**value["usage"]) if action == "complete" and isinstance(value, dict) else None - if action == "file_write_lock": + if action in {"file_write_lock", "file_write_locks"}: try: await await_owned_operation(self.call("file_write_unlock", {"id": value}, environment)) except Exception as exc: @@ -477,7 +480,12 @@ async def _call(self, action: str, arguments: dict[str, Any]) -> Any: @asynccontextmanager async def file_write_lock(self, path: str) -> AsyncIterator[None]: - identity = await self.client.call("file_write_lock", {"path": path}, self.handle) + async with self.file_write_locks((path,)): + yield None + + @asynccontextmanager + async def file_write_locks(self, paths: Sequence[str]) -> AsyncIterator[None]: + identity = await self.client.call("file_write_locks", {"paths": list(paths)}, self.handle) token = self._edit.set(identity) try: yield None diff --git a/tests/runtime/test_vm_mutation_locks.py b/tests/runtime/test_vm_mutation_locks.py new file mode 100644 index 00000000..7042bcb9 --- /dev/null +++ b/tests/runtime/test_vm_mutation_locks.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from opencollab.adapters._env_local import LocalEnvironment +from opencollab.adapters.tools.base import host_write_locks +from opencollab.adapters.tools.fs import FileWriteTool +from opencollab.application.tool_execution import ToolRuntime +from tests.runtime.test_vm_environment import _env + + +@pytest.mark.parametrize("cancel", [False, True]) +async def test_vm_grouped_mutation_uses_one_native_workspace_lock(tmp_path, monkeypatch, cancel): + (tmp_path / "nested").mkdir() + (tmp_path / "nested/value.py").write_text("VALUE = 1\n") + native = LocalEnvironment(str(tmp_path)) + environment = _env() + environment.workspace = str(tmp_path) + environment._identity = {"workspace": str(tmp_path)} + lock = asyncio.Lock() + owners = set() + acquisitions = [] + + async def rpc(payload): + operation = payload["operation_id"] + if payload["method"] == "cancel": + if operation in owners: + owners.remove(operation) + lock.release() + return {"status": "cancelled"} + assert payload["method"] == "start" + action = payload["request"]["action"] + arguments = payload["request"]["args"] + if action == "lock": + await lock.acquire() + owners.add(operation) + acquisitions.append(operation) + return {"status": "running", "lock_acquired": True} + assert lock.locked() + if action == "read": + value = await native.read_file(arguments["path"]) + else: + assert action == "write" + value = await native.write_file_with_change(arguments["path"], arguments["content"]) + return {"status": "completed", "value": value} + + monkeypatch.setattr(environment, "_rpc", rpc) + entered, release = asyncio.Event(), asyncio.Event() + + async def publish(): + async with host_write_locks(( + str(tmp_path / ".candidate-adoption"), str(tmp_path / "nested/value.py"), + str(tmp_path / "unrelated.py"), + ), environment): + await environment.write_file("nested/value.py", "VALUE = 42\n") + entered.set() + await release.wait() + + publication = asyncio.create_task(publish()) + writer = None + try: + await asyncio.wait_for(entered.wait(), 2) + assert len(acquisitions) == 1 + writer = asyncio.create_task(FileWriteTool().execute_with_runtime({ + "path": "nested/value.py", "mode": "create", "content": "VALUE = 100\n", + }, ToolRuntime(environment, None, None))) + await asyncio.sleep(0.05) + assert not writer.done() + if cancel: + publication.cancel() + with pytest.raises(asyncio.CancelledError): + await publication + else: + release.set() + await publication + assert "Created/wrote" in await asyncio.wait_for(writer, 2) + assert (tmp_path / "nested/value.py").read_text() == "VALUE = 100\n" + assert len(acquisitions) == 2 + assert not owners + assert not environment.pending_operations + assert not environment.revoked + finally: + release.set() + await asyncio.gather(publication, *([writer] if writer is not None else []), return_exceptions=True) + await environment.cleanup() + await native.cleanup() diff --git a/tests/workflows/test_candidate_adoption_concurrency.py b/tests/workflows/test_candidate_adoption_concurrency.py index eb98e7e2..5999d23b 100644 --- a/tests/workflows/test_candidate_adoption_concurrency.py +++ b/tests/workflows/test_candidate_adoption_concurrency.py @@ -16,6 +16,8 @@ class _OrderedEnvironment(LocalEnvironment): + file_write_lock_scope = "workspace" + def __init__(self, root, *, remote): super().__init__(str(root)) self.local_filesystem = not remote diff --git a/tests/workflows/test_candidate_mutation_backends.py b/tests/workflows/test_candidate_mutation_backends.py new file mode 100644 index 00000000..25b4acf3 --- /dev/null +++ b/tests/workflows/test_candidate_mutation_backends.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from opencollab.adapters._candidate_increment import _apply_increment +from opencollab.adapters._env_local import LocalEnvironment +from opencollab.adapters.session_environments import SessionEnvironmentPool +from opencollab.adapters.tools.base import host_write_locks +from opencollab.adapters.tools.fs import FileWriteTool +from opencollab.application.tool_execution import ToolRuntime +from opencollab.bootstrap.authored_workflow_bridge import AuthoredWorkflowBridge, BridgeClient, BridgeEnvironment +from tests.support.docker_native_edits_support import LocalDockerTransport, docker_environment +from tests.workflows.test_candidate_adoption_recovery import _flatten, _repository, _run +from tests.workflows.test_workflow_candidate_workspace import _git + + +@pytest.mark.parametrize("backend", ["local", "docker"]) +@pytest.mark.parametrize("bridge_entry", [False, True]) +@pytest.mark.parametrize("failure", [False, True]) +async def test_candidate_and_bridge_share_native_mutation_ownership( + tmp_path, monkeypatch, backend, bridge_entry, failure, +): + monkeypatch.chdir(tmp_path) + repo = _repository(tmp_path) + _git(repo, "add", "-A") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare native publication") + if failure: + _flatten(repo) + else: + (repo / "settings/defaults.py").write_text("VALUE = 42\n") + _git(repo, "add", "-A") + patch = _git(repo, "diff", "--cached", "--binary") + _git(repo, "reset", "--hard", "-q") + if failure: + assert _run(repo, cache=True) == "1\n" + index = (repo / ".git/index").read_bytes() + native = (LocalEnvironment(str(repo)) if backend == "local" else + docker_environment(repo, LocalDockerTransport(repo, pair_reads=False))) + entered, release, stop = asyncio.Event(), asyncio.Event(), asyncio.Event() + execute = native.exec_cmd + + async def publication_barrier(cmd, timeout=120, **kwargs): + command = cmd + if " apply --binary " in command and ".candidate-adopt-" in command: + entered.set() + await release.wait() + return await execute(command, timeout, **kwargs) + + monkeypatch.setattr(native, "exec_cmd", publication_barrier) + mailbox_root = tmp_path / "mailbox" + artifacts = mailbox_root / "artifacts" + artifacts.mkdir(parents=True) + mailbox = LocalEnvironment(str(mailbox_root)) + bridge = AuthoredWorkflowBridge( + mailbox, str(artifacts), + SessionEnvironmentPool(str(mailbox_root), isolation="shared", base_environment=mailbox), + {}, ToolRuntime(mailbox, None, None), allow_unisolated_shell=True, + ) + bridge._environments[1] = native + proxy = BridgeEnvironment(BridgeClient(artifacts, bridge.options()), { + "handle": 1, "workspace": native.workspace, "process_isolated": False, + }) + + async def serve(): + while not stop.is_set(): + await bridge.poll() + await asyncio.sleep(0.005) + + server = asyncio.create_task(serve()) + publication = asyncio.create_task(_apply_increment(proxy if bridge_entry else native, str(repo), patch)) + writer = None + try: + entered_task = asyncio.create_task(entered.wait()) + try: + await asyncio.wait((entered_task, publication), timeout=10, return_when=asyncio.FIRST_COMPLETED) + if publication.done(): + await publication + await asyncio.wait_for(entered_task, 1) + finally: + entered_task.cancel() + await asyncio.gather(entered_task, return_exceptions=True) + writer = asyncio.create_task(FileWriteTool().execute_with_runtime({ + "path": "settings/defaults.py", "mode": "create", "content": "VALUE = 100\n", + }, ToolRuntime(native if bridge_entry else proxy, None, None))) + await asyncio.sleep(0.05) + assert not writer.done() + assert (repo / "settings/defaults.py").read_text() == "VALUE = 1\n" + release.set() + if failure: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await asyncio.wait_for(publication, 10) + else: + await asyncio.wait_for(publication, 10) + assert "Created/wrote" in await asyncio.wait_for(writer, 10) + assert _run(repo) == "100\n" + assert (repo / ".git/index").read_bytes() == index + assert not native.revoked + assert not bridge._edits + finally: + release.set() + await asyncio.gather(publication, *([writer] if writer is not None else []), return_exceptions=True) + stop.set() + await server + await bridge.aclose() + await native.cleanup() + await mailbox.cleanup() + + +async def test_cancelled_publication_recovers_before_releasing_shared_writer(tmp_path, monkeypatch): + repo = _repository(tmp_path) + target = repo / "settings/defaults.py" + target.write_text("VALUE = 42\n") + patch = _git(repo, "diff", "--binary", "--", "settings/defaults.py") + target.write_text("VALUE = 1\n") + environment = LocalEnvironment(str(repo)) + entered = asyncio.Event() + execute = environment.exec_cmd + + async def pause_publication(command, timeout=120, **kwargs): + if " apply --binary " in command and ".candidate-adopt-" in command: + # A real completed Git publication is cancelled before its caller + # receives the result, exercising recovery of changed bytes. + result = await execute(command, timeout, **kwargs) + assert result.returncode == 0 + entered.set() + await asyncio.Event().wait() + return await execute(command, timeout, **kwargs) + + monkeypatch.setattr(environment, "exec_cmd", pause_publication) + publication = asyncio.create_task(_apply_increment(environment, str(repo), patch)) + writer = None + try: + await asyncio.wait_for(entered.wait(), 10) + writer = asyncio.create_task(FileWriteTool().execute_with_runtime({ + "path": str(target), "mode": "create", "content": "VALUE = 100\n", + }, ToolRuntime(environment, None, None))) + await asyncio.sleep(0.05) + assert not writer.done() + publication.cancel() + with pytest.raises(asyncio.CancelledError): + await publication + assert "Created/wrote" in await asyncio.wait_for(writer, 10) + assert _run(repo) == "100\n" + assert not list((repo / ".git").glob("candidate-recovery-*")) + finally: + if not publication.done(): + publication.cancel() + await asyncio.gather(publication, *([writer] if writer is not None else []), return_exceptions=True) + await environment.cleanup() + + +async def test_local_mutation_lock_normalizes_workspace_parent_aliases(tmp_path): + workspace = tmp_path / "workspace" + target = workspace / "nested/value.py" + target.parent.mkdir(parents=True) + target.write_text("VALUE = 1\n") + alias = tmp_path / "alias" + alias.symlink_to(workspace, target_is_directory=True) + environment = LocalEnvironment(str(workspace)) + writer = None + try: + async with host_write_locks((str(alias / "nested/value.py"),), environment): + writer = asyncio.create_task(FileWriteTool().execute_with_runtime({ + "path": str(target), "mode": "create", "content": "VALUE = 100\n", + }, ToolRuntime(environment, None, None))) + await asyncio.sleep(0.05) + assert not writer.done() + assert target.read_text() == "VALUE = 1\n" + assert "Created/wrote" in await asyncio.wait_for(writer, 2) + assert target.read_text() == "VALUE = 100\n" + finally: + if writer is not None: + await asyncio.gather(writer, return_exceptions=True) + await environment.cleanup() diff --git a/tests/workflows/test_candidate_shared_mutations.py b/tests/workflows/test_candidate_shared_mutations.py new file mode 100644 index 00000000..5a92ea26 --- /dev/null +++ b/tests/workflows/test_candidate_shared_mutations.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters._env_local import LocalEnvironment +from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.tools import builtin_tools +from tests.workflows.test_candidate_adoption_recovery import _flatten, _repository, _run +from tests.workflows.test_workflow_candidate_workspace import _git + + +class _PublicationBarrier(LocalEnvironment): + def __init__(self, workspace): + super().__init__(str(workspace)) + self.entered = asyncio.Event() + self.release = asyncio.Event() + self.publications = [] + + async def exec_cmd(self, command, timeout=120, **kwargs): + publication = " apply --binary " in command and ".candidate-adopt-" in command + if publication: + self.entered.set() + await self.release.wait() + result = await super().exec_cmd(command, timeout, **kwargs) + if publication: + self.publications.append(result.returncode) + return result + + +class _WriterModel: + def __init__(self, tool, path): + self.tool, self.path, self.calls = tool, path, 0 + self.requested = asyncio.Event() + + async def complete(self, messages, tools=None, **kwargs): + self.calls += 1 + if self.calls == 1: + self.requested.set() + arguments = {"path": self.path} + if self.tool == "file_write": + arguments.update(mode="create", content="VALUE = 100\n") + else: + arguments.update(mode="line_replace", start_line=1, end_line=1, new_str="VALUE = 100") + return LLMResponse(tool_calls=[{ + "id": "shared-edit", "type": "function", "function": { + "name": self.tool, "arguments": json.dumps(arguments), + }, + }], usage=Usage(10, 5), finish_reason="tool_calls") + assert any("100" in str(message) for message in messages) + return LLMResponse(content="Shared edit complete", usage=Usage(10, 5), finish_reason="stop") + + +@pytest.mark.parametrize("tool", ["file_write", "apply_patch"]) +@pytest.mark.parametrize("relation,failure", [ + ("target", False), ("target", True), ("descendant", True), + ("unrelated", False), ("unrelated", True), +]) +async def test_sdk_shared_agent_owns_mutations_through_candidate_publication( + tmp_path, tool, relation, failure, +): + repo = _repository(tmp_path, cache=relation == "descendant") + if failure and relation != "descendant": + (repo / "blocked").mkdir() + (repo / "blocked/entry.py").write_text("VALUE = 1\n") + _git(repo, "add", "blocked/entry.py") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare directory replacement") + (repo / "blocked/__pycache__").mkdir() + (repo / "blocked/__pycache__/retained.pyc").write_bytes(b"retained ignored bytes") + environment = _PublicationBarrier(repo) + path = "other.py" if relation == "unrelated" else "settings/defaults.py" + model = _WriterModel(tool, path) + index = (repo / ".git/index").read_bytes() + + async def edit(child, _): + folder = Path(child.workspace_root) + if relation == "descendant": + _flatten(folder) + else: + (folder / "settings/defaults.py").write_text("VALUE = 42\n") + if failure: + (folder / "blocked/entry.py").unlink() + (folder / "blocked").rmdir() + (folder / "blocked").write_text("replacement\n") + return "prepared" + + async def flow(ctx, _): + candidate = await ctx.candidate_workflow(edit, {}, label="forty-two") + publication = asyncio.create_task(ctx.adopt_candidate(candidate)) + writer = None + try: + await asyncio.wait_for(environment.entered.wait(), 10) + writer = asyncio.create_task(ctx.agent_run( + "Set the shared value to 100", tools=list(builtin_tools(tool, headless=False)), + isolation="shared", budget=2000, system_prompt="Use the requested file tool", + )) + await asyncio.wait_for(model.requested.wait(), 10) + if relation == "unrelated": + result = await asyncio.wait_for(asyncio.shield(writer), 10) + assert result.status == "completed" + assert (repo / path).read_text() == "VALUE = 100\n" + else: + await asyncio.sleep(0.05) + assert not writer.done() + assert (repo / path).read_text() == "VALUE = 1\n" + environment.release.set() + if failure: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await publication + else: + await publication + result = await writer + assert result.status == "completed" + assert (repo / path).read_text() == "VALUE = 100\n" + assert len(environment.publications) == 1 + assert (environment.publications[0] != 0) is failure + return "retained" + finally: + environment.release.set() + await asyncio.gather(publication, *([writer] if writer is not None else []), return_exceptions=True) + + try: + result = await OpenCollab(repo, environment=environment, model="local-scripted").workflow( + flow, llm=model, trace=False, timeout=40, budget=10000, concurrency=2, + limit_mode="explicit", agent_profile="single2", + ) + assert result.ok, result.reason + assert result.output == "retained" + assert _run(repo) == ("100\n" if relation != "unrelated" else "1\n" if failure else "42\n") + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not environment.revoked + finally: + environment.release.set() + await environment.cleanup() From 32f5be3c46c190f0d816e62da5db5b1eed79ecfc Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 13:51:26 +0800 Subject: [PATCH 13/82] fix(llm): recheck token budgets before provider retries Charge known failed-attempt usage before retry backoff and revalidate each physical request against current session and authored workflow limits. Preserve aggregate response, event, and ledger accounting while deducting the usage already charged. Add loopback TCP regressions for truncated Chat streams, Responses failures, Anthropic retries, shared workflow and team spend, callback failures, and authored subprocess accounting. --- opencollab/adapters/llm/anthropic_provider.py | 53 ++- opencollab/adapters/llm/client.py | 40 +- opencollab/adapters/llm/openai_provider.py | 45 ++- opencollab/adapters/llm/responses_provider.py | 21 +- opencollab/adapters/llm/retry.py | 13 +- .../application/_session_run_completion.py | 22 +- opencollab/application/_session_run_usage.py | 17 +- opencollab/application/session_run.py | 3 +- .../bootstrap/authored_workflow_bridge.py | 18 +- .../runtime/test_llm_chat_error_usage_http.py | 1 + .../test_provider_attempt_budget_http.py | 359 ++++++++++++++++++ .../test_responses_error_usage_http.py | 1 + 12 files changed, 539 insertions(+), 54 deletions(-) create mode 100644 tests/runtime/test_provider_attempt_budget_http.py diff --git a/opencollab/adapters/llm/anthropic_provider.py b/opencollab/adapters/llm/anthropic_provider.py index 6a1fc61e..9de632d1 100644 --- a/opencollab/adapters/llm/anthropic_provider.py +++ b/opencollab/adapters/llm/anthropic_provider.py @@ -10,8 +10,9 @@ import inspect import json import re -from typing import Any +from typing import Any, Callable +from opencollab.adapters.llm._attempt_usage import _combine_attempt_usage from opencollab.adapters.llm.first_token import NOT_STREAMED, begin_attempt from opencollab.adapters.llm.retry import RetryTimeBudget, with_retry from opencollab.adapters.llm.tool_contracts import ( @@ -385,6 +386,8 @@ async def complete_anthropic( max_output_tokens: int | None = None, reasoning_effort: str | None = None, provider_error_time_budget: RetryTimeBudget | None = None, + before_attempt: Callable[[], int | None] | None = None, + on_attempt_usage: Callable[[Usage], None] | None = None, ) -> LLMResponse: """Single-shot completion against the Anthropic API.""" kwargs = _build_request_kwargs( @@ -400,18 +403,44 @@ async def complete_anthropic( reasoning_effort, ) - async def request_once() -> Any: - # Record the non-streamed attempt's start and unavailable first-token timing. - begin_attempt(streamed=False, unavailable_reason=NOT_STREAMED) - create = client.messages.create - return await create(**_sdk_sampling_kwargs(create, kwargs)) + failed_usages: list[Usage | None] = [] - resp = await with_retry( - request_once, - max_retries=max_retries, - retry_time_budget=provider_error_time_budget, - ) - return _parse_response(resp) + def prepare_attempt() -> None: + if before_attempt is not None and (limit := before_attempt()) is not None: + kwargs["max_tokens"] = limit + + def report_failure(_exc: BaseException) -> None: + if failed_usages and (usage := failed_usages[-1]) is not None and on_attempt_usage is not None: + on_attempt_usage(usage) + + async def request_once() -> LLMResponse: + resp = None + try: + begin_attempt(streamed=False, unavailable_reason=NOT_STREAMED) + create = client.messages.create + resp = await create(**_sdk_sampling_kwargs(create, kwargs)) + return _parse_response(resp) + except BaseException as exc: + usage = _parse_usage(resp.usage) if resp is not None else getattr(exc, "usage", None) + failed_usages.append(usage) + raise + + try: + response = await with_retry( + request_once, + max_retries=max_retries, + retry_time_budget=provider_error_time_budget, + before_attempt=prepare_attempt, + on_failure=report_failure, + ) + except BaseException as exc: + usage = _combine_attempt_usage(failed_usages) + if usage is not None: + exc.usage = usage + raise + if failed_usages: + response.usage = _combine_attempt_usage([*failed_usages, response.usage]) + return response def _sdk_sampling_kwargs(create: Any, kwargs: dict[str, Any]) -> dict[str, Any]: """Preserve supported sampling on SDKs that moved it to extra_body.""" diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index c73a6652..dd660e75 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -124,6 +124,7 @@ def __init__( self.request_capacity = request_capacity self.supports_request_admission = request_pool is not None self.supports_request_revalidation = True + self.supports_attempt_usage = True self.model = model self.provider = provider self.max_retries = max(0, max_retries) @@ -294,6 +295,7 @@ async def complete( on_admitted: Callable[[], None] | None = None, before_request: Callable[[], int] | None = None, on_usage: Callable[[Any], None] | None = None, + on_attempt_usage: Callable[[Any], None] | None = None, ) -> LLMResponse: """Single-shot completion. Returns full response. @@ -308,10 +310,28 @@ async def complete( ``top_p`` is ``None`` by default (provider default); when set it is sent as the nucleus-sampling knob. Unset == byte-identical request as today. + + ``before_request`` runs before each physical attempt. Failed attempts + report their known usage through ``on_attempt_usage`` before retry + backoff; ``on_usage`` receives the final aggregate once. """ start = time.monotonic() response = None settled_usage = None + admitted = False + + def prepare_attempt() -> int | None: + nonlocal admitted + limit = max_output_tokens + if before_request is not None: + current_limit = before_request() + limit = current_limit if limit is None else min(limit, current_limit) + if not admitted: + admitted = True + if on_admitted is not None: + on_admitted() + return limit + retry_budget = ( self.provider_retry_budget if self._retry_budget_explicit @@ -322,14 +342,6 @@ async def complete( async with AsyncExitStack() as request_owner: try: await request_owner.enter_async_context(request_slot(self.request_pool, self.request_capacity)) - if before_request is not None: - current_limit = before_request() - max_output_tokens = ( - current_limit if max_output_tokens is None - else min(max_output_tokens, current_limit) - ) - if on_admitted is not None: - on_admitted() if self._anthropic: response = await complete_anthropic( self._anthropic, @@ -345,6 +357,8 @@ async def complete( max_output_tokens=max_output_tokens, reasoning_effort=reasoning_effort, provider_error_time_budget=retry_budget, + before_attempt=prepare_attempt, + on_attempt_usage=on_attempt_usage, ) elif self.wire_protocol == RESPONSES: response = await complete_responses( @@ -370,6 +384,8 @@ async def complete( stream_idle_timeout=self.stream_idle_timeout, round_timeout=self.request_timeout, provider_error_time_budget=retry_budget, + before_attempt=prepare_attempt, + on_attempt_usage=on_attempt_usage, ) else: response = await complete_openai( @@ -386,6 +402,8 @@ async def complete( max_output_tokens=max_output_tokens, reasoning_effort=reasoning_effort, provider_error_time_budget=retry_budget, + before_attempt=prepare_attempt, + on_attempt_usage=on_attempt_usage, stream=self.stream_chat, first_event_timeout=self.first_event_timeout, stream_idle_timeout=self.stream_idle_timeout, @@ -435,4 +453,8 @@ async def complete( raise finally: if on_usage is not None and settled_usage is not None: - on_usage(settled_usage) + try: + on_usage(settled_usage) + except BaseException as exc: + exc.usage = settled_usage + raise diff --git a/opencollab/adapters/llm/openai_provider.py b/opencollab/adapters/llm/openai_provider.py index 5bb015f9..51aef665 100644 --- a/opencollab/adapters/llm/openai_provider.py +++ b/opencollab/adapters/llm/openai_provider.py @@ -7,7 +7,7 @@ from __future__ import annotations import re -from typing import Any +from typing import Any, Callable from opencollab.adapters.llm._attempt_usage import _combine_attempt_usage from opencollab.adapters.llm._chat_response import _parse_response, _reported_chat_usage @@ -229,6 +229,8 @@ async def complete_openai( stream: bool = False, first_event_timeout: float | None = 180.0, stream_idle_timeout: float | None = 180.0, + before_attempt: Callable[[], int | None] | None = None, + on_attempt_usage: Callable[[Usage], None] | None = None, ) -> LLMResponse: """Single-shot completion against an OpenAI-compatible endpoint. @@ -248,36 +250,41 @@ async def complete_openai( max_output_tokens, reasoning_effort, ) - if not stream: + if stream: + kwargs.update(_STREAM_REQUEST_FIELDS) + failed_usages: list[Usage | None] = [] - async def unstreamed_once() -> Any: - # Record the non-streamed attempt's start and unavailable first-token timing. - begin_attempt(streamed=False, unavailable_reason=NOT_STREAMED) - return await client.chat.completions.create(**kwargs) + def prepare_attempt() -> None: + if before_attempt is not None and (limit := before_attempt()) is not None: + field = "max_completion_tokens" if _uses_reasoning_request_fields(model) else "max_tokens" + kwargs[field] = limit - resp = await with_retry( - unstreamed_once, - max_retries=max_retries, - retry_time_budget=provider_error_time_budget, - ) - return _parse_response(resp, kwargs["messages"], kwargs.get("tools")) - - stream_kwargs = {**kwargs, **_STREAM_REQUEST_FIELDS} - failed_usages: list[Usage | None] = [] + def report_failure(_exc: BaseException) -> None: + if failed_usages and (usage := failed_usages[-1]) is not None and on_attempt_usage is not None: + on_attempt_usage(usage) async def request_once() -> LLMResponse: # Retry stream creation and draining together. Creation returns at headers, # so its success alone cannot cover a later stream failure. state = None + resp = None try: + if not stream: + begin_attempt(streamed=False, unavailable_reason=NOT_STREAMED) + resp = await client.chat.completions.create(**kwargs) + return _parse_response(resp, kwargs["messages"], kwargs.get("tools")) state = await _create_and_consume_chat_stream( - client, stream_kwargs, first_event_timeout, stream_idle_timeout + client, kwargs, first_event_timeout, stream_idle_timeout ) return _stream_state_to_response( - state, stream_kwargs["messages"], stream_kwargs.get("tools") + state, kwargs["messages"], kwargs.get("tools") ) except BaseException as exc: - usage = _reported_chat_usage(state.usage_object) if state is not None else getattr(exc, "usage", None) + usage = ( + _reported_chat_usage(state.usage_object) if state is not None + else _reported_chat_usage(resp.usage) if resp is not None + else getattr(exc, "usage", None) + ) failed_usages.append(usage) raise @@ -286,6 +293,8 @@ async def request_once() -> LLMResponse: request_once, max_retries=max_retries, retry_time_budget=provider_error_time_budget, + before_attempt=prepare_attempt, + on_failure=report_failure, ) except BaseException as exc: usage = _combine_attempt_usage(failed_usages) diff --git a/opencollab/adapters/llm/responses_provider.py b/opencollab/adapters/llm/responses_provider.py index acc11ed7..b95b01e2 100644 --- a/opencollab/adapters/llm/responses_provider.py +++ b/opencollab/adapters/llm/responses_provider.py @@ -7,7 +7,7 @@ import logging import os from dataclasses import dataclass, field -from typing import Any +from typing import Any, Callable from opencollab.adapters.llm._responses_instructions import _check_instructions_echo from opencollab.adapters.llm._responses_output import ( @@ -685,6 +685,8 @@ async def complete_responses( round_timeout: float | None = None, provider_error_time_budget: RetryTimeBudget | None = None, stream: bool = True, + before_attempt: Callable[[], int | None] | None = None, + on_attempt_usage: Callable[[Usage], None] | None = None, ) -> LLMResponse: """Run one locally replayable Responses request and require typed completion.""" external_isolation = os.environ.get("OPENCOLLAB_EXTERNAL_PROVIDER_ISOLATION") == "1" @@ -711,6 +713,14 @@ async def complete_responses( stream = stream and capabilities.supports_responses_streaming failed_usages: list[Usage | None] = [] + def prepare_attempt() -> None: + if before_attempt is not None and (limit := before_attempt()) is not None: + kwargs["max_output_tokens"] = limit + + def report_failure(_exc: BaseException) -> None: + if failed_usages and (usage := failed_usages[-1]) is not None and on_attempt_usage is not None: + on_attempt_usage(usage) + async def request_once() -> LLMResponse: terminal = None try: @@ -740,7 +750,8 @@ async def request_once() -> LLMResponse: converted_tools, ) except BaseException as exc: - failed_usages.append(_reported_responses_usage(terminal) or getattr(exc, "usage", None)) + usage = _reported_responses_usage(terminal) or getattr(exc, "usage", None) + failed_usages.append(usage) raise if retry_budget is not None: @@ -757,8 +768,12 @@ async def run() -> LLMResponse: if retry_budget is not None: return await with_retry( bounded_request_once, max_retries=retry_limit, retry_time_budget=retry_budget, + before_attempt=prepare_attempt, + on_failure=report_failure, ) - return await with_retry(request_once, max_retries=retry_limit) + return await with_retry( + request_once, max_retries=retry_limit, before_attempt=prepare_attempt, on_failure=report_failure, + ) try: try: diff --git a/opencollab/adapters/llm/retry.py b/opencollab/adapters/llm/retry.py index c6bf204e..a72f7e01 100644 --- a/opencollab/adapters/llm/retry.py +++ b/opencollab/adapters/llm/retry.py @@ -139,6 +139,8 @@ async def with_retry( max_retries: int | None, *, retry_time_budget: RetryTimeBudget | None = None, + before_attempt: Callable[[], None] | None = None, + on_failure: Callable[[BaseException], None] | None = None, ) -> Any: """Retry transient provider errors with exponential backoff. @@ -146,14 +148,19 @@ async def with_retry( """ attempt = 0 while True: + if before_attempt is not None: + before_attempt() started = time.monotonic() try: return await call_factory() - except Exception as e: - if not is_retryable_error(e): + except BaseException as e: + elapsed = time.monotonic() - started + if on_failure is not None: + on_failure(e) + if not isinstance(e, Exception) or not is_retryable_error(e): raise retry_time_available = retry_time_budget is None or retry_time_budget.consume( - time.monotonic() - started + elapsed ) if max_retries is not None and attempt >= max_retries: raise diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index 22400efd..c6e2764e 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -28,7 +28,11 @@ _TokenBudgetStop, ) from opencollab.application._session_run_trace import _SessionRunTraceMixin -from opencollab.application._session_run_usage import _complete_with_error_usage, _normalize_completion_usage +from opencollab.application._session_run_usage import ( + _AttemptUsageAccounting, + _complete_with_error_usage, + _normalize_completion_usage, +) from opencollab.application._tool_loop_execution import _apply_completed_prefix_progress from opencollab.application.async_timeout import CallerTimeoutError, abandon_on_timeout from opencollab.application.ports import ( @@ -805,6 +809,9 @@ async def _invoke_summary( kwargs["max_output_tokens"] = max_output_tokens start = time.monotonic() protected_call = self.state.wind_down_done + attempt_usage = _AttemptUsageAccounting(self, protected_call=protected_call) + if getattr(self.llm, "supports_attempt_usage", False): + kwargs["on_attempt_usage"] = attempt_usage.record admitted = asyncio.Event() if getattr(self.llm, "supports_request_admission", False) else None if admitted is not None: kwargs["on_admitted"] = admitted.set @@ -825,7 +832,7 @@ def revalidate() -> int: def account_response(response: CompletionResponse) -> None: input_tokens, total_tokens = _normalize_completion_usage(response.usage) - self.state.add_used_tokens(total_tokens) + self.state.add_used_tokens(total_tokens - attempt_usage.tokens) self._record_usage_event(response.usage, total_tokens, purpose="summary", late=abandoned) self._mark_budget_reserve_consumed(protected_call=protected_call) self.state.add_markup_recovered(getattr(response.usage, "markup_recovered", 0)) @@ -842,7 +849,7 @@ async def complete_owned() -> CompletionResponse: self._raise_if_run_cancelled() return await _complete_with_error_usage( self, complete, messages, protected_call=protected_call, - usage_purpose="summary", on_response=account_response, **kwargs, + usage_purpose="summary", on_response=account_response, attempt_usage=attempt_usage, **kwargs, ) finally: self._draining_provider_tasks.discard(asyncio.current_task()) @@ -883,6 +890,9 @@ async def _invoke_llm(self, **kwargs: Any) -> CompletionResponse: if self._run_control is not None: self._raise_if_run_cancelled() protected_call = self.state.wind_down_done + attempt_usage = _AttemptUsageAccounting(self, protected_call=protected_call) + if getattr(self.llm, "supports_attempt_usage", False): + kwargs["on_attempt_usage"] = attempt_usage.record admitted = asyncio.Event() if getattr(self.llm, "supports_request_admission", False) else None if admitted is not None: kwargs["on_admitted"] = admitted.set @@ -898,9 +908,13 @@ async def complete_owned() -> CompletionResponse: try: response = await _complete_with_error_usage( self, self.llm.complete, protected_call=protected_call, **kwargs, + attempt_usage=attempt_usage, ) late_accounted = asyncio.current_task() in self._draining_provider_tasks - self._account_completion_usage(response, protected_call=protected_call, late=late_accounted) + self._account_completion_usage( + response, protected_call=protected_call, late=late_accounted, + accounted_tokens=attempt_usage.tokens, + ) return response finally: self._draining_provider_tasks.discard(asyncio.current_task()) diff --git a/opencollab/application/_session_run_usage.py b/opencollab/application/_session_run_usage.py index 49477bec..3a4cd1db 100644 --- a/opencollab/application/_session_run_usage.py +++ b/opencollab/application/_session_run_usage.py @@ -33,12 +33,27 @@ def _normalize_completion_usage(usage: Any) -> tuple[int, int]: context_tokens = _nonnegative_usage_int(context_tokens, "context_tokens") return input_tokens if context_tokens is None else context_tokens, max(reported_total, input_tokens + output_tokens) + +class _AttemptUsageAccounting: + def __init__(self, runner: Any, *, protected_call: bool) -> None: + self.runner = runner + self.protected_call = protected_call + self.tokens = 0 + + def record(self, usage: Any) -> None: + total = _normalize_completion_usage(usage)[1] + self.runner.state.add_used_tokens(total) + self.tokens += total + self.runner._mark_budget_reserve_consumed(protected_call=self.protected_call) + + async def _complete_with_error_usage( runner: Any, complete: Callable[..., Awaitable[Any]], *args: Any, protected_call: bool, usage_purpose: str = "completion", + attempt_usage: _AttemptUsageAccounting | None = None, **kwargs: Any, ) -> Any: """Charge reported error usage inside the provider owner exactly once.""" @@ -49,7 +64,7 @@ async def _complete_with_error_usage( usage = getattr(exc, "usage", None) if usage is not None: _input_tokens, total_tokens = _normalize_completion_usage(usage) - runner.state.add_used_tokens(total_tokens) + runner.state.add_used_tokens(total_tokens - (attempt_usage.tokens if attempt_usage is not None else 0)) late = asyncio.current_task() in runner._draining_provider_tasks record_usage = getattr(runner, "_record_usage_event", None) if callable(record_usage): diff --git a/opencollab/application/session_run.py b/opencollab/application/session_run.py index 7af543a0..e71e0cfa 100644 --- a/opencollab/application/session_run.py +++ b/opencollab/application/session_run.py @@ -266,10 +266,11 @@ def _record_late_provider_result( def _account_completion_usage( self, response: Any, *, protected_call: bool, late: bool, usage_accounted: bool = False, + accounted_tokens: int = 0, ) -> None: _input_tokens, total_tokens = _normalize_completion_usage(response.usage) if not usage_accounted: - self.state.add_used_tokens(total_tokens) + self.state.add_used_tokens(total_tokens - accounted_tokens) self._record_usage_event(response.usage, total_tokens, purpose="completion", late=late) self._mark_budget_reserve_consumed(protected_call=protected_call) if late: diff --git a/opencollab/bootstrap/authored_workflow_bridge.py b/opencollab/bootstrap/authored_workflow_bridge.py index e6d560f2..2066ffc0 100644 --- a/opencollab/bootstrap/authored_workflow_bridge.py +++ b/opencollab/bootstrap/authored_workflow_bridge.py @@ -191,8 +191,8 @@ def _request_output_limit(self, arguments: dict[str, Any]) -> int: ) return min(configured, remaining - reserved) - def _account_native_usage(self, usage: Any) -> None: - self.tokens += _normalize_completion_usage(usage)[1] + def _account_native_usage(self, usage: Any, *, accounted_tokens: int = 0) -> None: + self.tokens += _normalize_completion_usage(usage)[1] - accounted_tokens if self._on_usage is not None: self._on_usage(self.tokens) @@ -293,8 +293,20 @@ async def _dispatch(self, identity: str, request: dict[str, Any]) -> Any: notifications = [] native_accounting = getattr(self._model(), "supports_request_revalidation", False) if native_accounting: + accounted_tokens = 0 + + def account_attempt(usage: Any) -> None: + nonlocal accounted_tokens + total = _normalize_completion_usage(usage)[1] + accounted_tokens += total + self._account_native_usage(usage) + arguments["before_request"] = lambda: self._request_output_limit(arguments) - arguments["on_usage"] = self._account_native_usage + arguments["on_usage"] = lambda usage: self._account_native_usage( + usage, accounted_tokens=accounted_tokens, + ) + if getattr(self._model(), "supports_attempt_usage", False): + arguments["on_attempt_usage"] = account_attempt if getattr(self._model(), "supports_request_admission", False): def admitted() -> None: notifications.append(asyncio.create_task(self._environment.write_file( diff --git a/tests/runtime/test_llm_chat_error_usage_http.py b/tests/runtime/test_llm_chat_error_usage_http.py index 59b310ea..3035d372 100644 --- a/tests/runtime/test_llm_chat_error_usage_http.py +++ b/tests/runtime/test_llm_chat_error_usage_http.py @@ -174,6 +174,7 @@ async def wait_for_cancellation(_seconds): await session.add_user_message("Reply ok.") run = asyncio.create_task(session.run_loop()) await asyncio.wait_for(entered_backoff.wait(), 1) + assert session.used_tokens == 27 run.cancel() with pytest.raises(asyncio.CancelledError): await run diff --git a/tests/runtime/test_provider_attempt_budget_http.py b/tests/runtime/test_provider_attempt_budget_http.py new file mode 100644 index 00000000..22390cf3 --- /dev/null +++ b/tests/runtime/test_provider_attempt_budget_http.py @@ -0,0 +1,359 @@ +from __future__ import annotations + +import asyncio +import json +import socket +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from opencollab import OpenCollab, RunControl +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm import retry +from opencollab.adapters.llm.client import LLMClient +from opencollab.application.event_bus import EventBus +from opencollab.application.scheduler import Scheduler +from opencollab.bootstrap import build_session +from opencollab.domain.agent import Agent +from opencollab.domain.scheduler import SessionControlBlock +from opencollab.domain.session import SessionPhase, SessionState +from tests.interfaces.test_member_authored_workflow import _manager +from tests.support.environment_test_support import local_environment_factory as local_environment_factory +from tests.support.paths import REPO_ROOT + + +def _event(value): + return f"data: {json.dumps(value)}\n\n".encode() + + +@pytest.fixture +def attempt_server(monkeypatch, tmp_path): + servers = [] + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + monkeypatch.delenv("OPENCOLLAB_EXTERNAL_PROVIDER_ISOLATION", raising=False) + monkeypatch.setattr(retry, "extract_retry_after_seconds", lambda _error: 0.0) + monkeypatch.setattr(retry.random, "uniform", lambda _lo, _hi: 0.0) + + def start(protocol, observe=lambda: None): + requests = [] + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + requests.append({"body": body, "observed": observe()}) + first = len(requests) == 1 + total = 1500 if first else 4 + status = 200 + content_type = "text/event-stream" + if protocol == "chat": + common = {"id": "chat_attempt", "object": "chat.completion.chunk", + "created": 1, "model": body["model"]} + payload = b"".join([ + _event({**common, "choices": [{"index": 0, "delta": { + "role": "assistant", "content": "ok"}, "finish_reason": None}]}), + _event({**common, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}), + _event({**common, "choices": [], "usage": { + "prompt_tokens": total - 2, "completion_tokens": 2, "total_tokens": total}}), + ]) + if not first: + payload += b"data: [DONE]\n\n" + elif protocol == "responses": + item = {"id": "msg_attempt", "type": "message", "role": "assistant", + "status": "completed", "content": [{"type": "output_text", "text": "ok", + "annotations": []}]} + response = {"id": "resp_attempt", "object": "response", "model": body["model"], + "status": "failed" if first else "completed", "output": [] if first else [item], + "error": {"code": "server_error", "message": "Temporarily unavailable"} + if first else None, "incomplete_details": None, + "usage": {"input_tokens": total - 2, "output_tokens": 2, "total_tokens": total}} + payload = _event({"type": "response.failed", "response": response}) if first else b"".join([ + _event({"type": "response.output_item.done", "output_index": 0, "item": item}), + _event({"type": "response.completed", "response": response}), + ]) + else: + content_type = "application/json" + status = 503 if first else 200 + payload = json.dumps({"type": "error", "error": { + "type": "overloaded_error", "message": "Temporarily unavailable"}} + if first else {"id": "msg_attempt", "type": "message", "role": "assistant", + "model": body["model"], "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", "stop_sequence": None, + "usage": {"input_tokens": 2, "output_tokens": 2}}).encode() + self.send_response(status) + self.send_header("Content-Type", content_type) + if protocol == "chat" and first: + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + self.wfile.write(f"{len(payload):x}\r\n".encode() + payload + b"\r\n") + self.wfile.flush() + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + self.close_connection = True + else: + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + self.wfile.flush() + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + servers.append((server, thread)) + return f"http://127.0.0.1:{server.server_port}/v1", requests + + yield start + for server, thread in servers: + server.shutdown() + server.server_close() + thread.join() + + +def _client(protocol, url, *, retries=1): + return LLMClient( + provider="anthropic" if protocol == "anthropic" else "openai", + model="claude-sonnet-4-6" if protocol == "anthropic" else "gpt-4o", + wire_protocol="responses" if protocol == "responses" else "chat_completions", + api_key="controlled-test", base_url=url, # pragma: allowlist secret + stream_chat=protocol == "chat", max_retries=retries, + ) + + +def _records(tmp_path): + return [json.loads(line) for line in (tmp_path / "usage.jsonl").read_text().splitlines()] + + +@pytest.mark.parametrize("protocol", ["chat", "responses"]) +@pytest.mark.parametrize("budget, retries, count", [(2000, 1, 1), (20000, 1, 2), (None, 1, 2), (2000, 0, 1)]) +async def test_public_agent_rechecks_reported_retry_spend(tmp_path, attempt_server, protocol, budget, retries, count): + url, requests = attempt_server(protocol) + events = [] + result = await OpenCollab( + tmp_path, model="routed-worker", model_routes={"routed-worker": { + "model": "gpt-4o", "provider": "openai", "api_key": "controlled-test", # pragma: allowlist secret + "base_url": url, "wire_protocol": "responses" if protocol == "responses" else "chat_completions", + }}, config={"max_output_tokens": 128, "llm_stream_chat": protocol == "chat", "llm_max_retries": retries}, + ).agent("Reply ok.", tools=[], budget=budget, trace=False, run_control=RunControl( + on_event=lambda event: events.append(event.data) if event.type == "usage" else None, + )) + assert len(requests) == count + assert result.tokens == 1500 + 4 * (count - 1) + assert result.status == ("failed" if retries == 0 else "stopped" if count == 1 else "completed") + if count == 1 and retries: + assert "budget exhausted before model call" in result.reason + records = _records(tmp_path) + assert len(records) == len(events) == 1 + assert records[0]["usage"]["total_tokens"] == events[0]["total_tokens"] == result.tokens + + +@pytest.mark.parametrize("protocol, field", [("chat", "max_tokens"), ("responses", "max_output_tokens"), + ("anthropic", "max_tokens")]) +@pytest.mark.parametrize("change", ["reduced", "exhausted", "team", "cancel"]) +@pytest.mark.parametrize("summary", [False, True]) +async def test_physical_retry_observes_current_session_state( + tmp_path, monkeypatch, local_environment_factory, attempt_server, protocol, field, change, summary, +): + sessions = [] + url, requests = attempt_server(protocol, lambda: sessions[0].used_tokens) + team_stop, cancel = asyncio.Event(), asyncio.Event() + known = 0 if protocol == "anthropic" else 1500 + async with _client(protocol, url) as client: + session = build_session( + env=local_environment_factory(), llm=client, max_budget_tokens=20000, + agent=Agent(name="worker", model=client.model, provider=client.provider, + system_prompt="Finish the task.", max_tokens_per_step=128), + team_budget_exhausted=team_stop.is_set, default_cancel_event=cancel, + ) + sessions.append(session) + await session.add_user_message("Reply ok.") + messages = list(session.messages) + reserved = client.estimate_request_tokens(messages) + + def before_backoff(_error): + assert session.used_tokens == known + if change in {"reduced", "exhausted"}: + session.max_budget_tokens = known + reserved + (37 if change == "reduced" else 0) + elif change == "team": + team_stop.set() + else: + cancel.set() + return 0.0 + + monkeypatch.setattr(retry, "extract_retry_after_seconds", before_backoff) + + async def complete_summary(messages, *, on_response, **kwargs): + response = await client.complete(messages, **kwargs) + on_response(response) + return response + + if summary: + operation = session.runner._invoke_summary(complete_summary, messages) + if change == "reduced": + assert (await operation).content == "ok" + else: + with pytest.raises(Exception) as error: + await operation + assert type(error.value).__name__ == { + "exhausted": "_TokenBudgetStop", "team": "_TeamBudgetStop", "cancel": "_RunCancelled", + }[change] + if known: + assert error.value.usage.total_tokens == known + else: + assert await session.run_loop() == ("ok" if change == "reduced" else "") + assert session.phase is (SessionPhase.DONE if change == "reduced" else SessionPhase.STOPPED) + assert len(requests) == (2 if change == "reduced" else 1) + if change == "reduced": + assert requests[1]["body"][field] == 37 + assert requests[1]["observed"] == known + assert session.used_tokens == known + (4 if change == "reduced" else 0) + records = _records(tmp_path) + assert len(records) == 1 + assert records[0]["usage"]["total_tokens"] == session.used_tokens + + +@pytest.mark.parametrize("protocol", ["chat", "responses"]) +@pytest.mark.parametrize("callback", ["attempt", "final", "revalidate"]) +async def test_callback_errors_stop_retry_and_retain_usage(tmp_path, attempt_server, protocol, callback): + url, requests = attempt_server(protocol) + attempts, final, admissions, checks = [], [], [], [] + + def failed_callback(): + raise RuntimeError("timeout in caller callback") + + def on_attempt(usage): + attempts.append(usage.total_tokens) + if callback == "attempt": + failed_callback() + + def on_usage(usage): + final.append(usage.total_tokens) + if callback == "final": + failed_callback() + + def before_request(): + checks.append(True) + if len(checks) == 2 and callback == "revalidate": + failed_callback() + return 128 + + async with _client(protocol, url, retries=3) as client: + with pytest.raises(RuntimeError, match="timeout in caller callback") as error: + await client.complete( + [{"role": "user", "content": "Reply ok."}], before_request=before_request, + on_attempt_usage=on_attempt, on_usage=on_usage, on_admitted=lambda: admissions.append(True), + ) + expected = 1504 if callback == "final" else 1500 + assert len(requests) == (2 if callback == "final" else 1) + assert attempts == [1500] + assert final == [expected] + assert admissions == [True] + assert error.value.usage.total_tokens == expected + records = _records(tmp_path) + assert len(records) == 1 + assert records[0]["usage"]["total_tokens"] == expected + + +@pytest.mark.parametrize("protocol", ["chat", "responses"]) +async def test_workflow_observes_failure_spend_during_retry(tmp_path, monkeypatch, attempt_server, protocol): + contexts, seen = [], [] + url, requests = attempt_server(protocol, lambda: contexts[0].tokens_spent()) + + def backoff(_error): + seen.append(contexts[0].tokens_spent()) + return 0.0 + + monkeypatch.setattr(retry, "extract_retry_after_seconds", backoff) + async with _client(protocol, url) as client: + async def flow(ctx, _args): + contexts.append(ctx) + return await ctx.agent("Reply ok.", tools=[], budget=10000) + + result = await OpenCollab(tmp_path, config={"max_output_tokens": 128}).workflow( + flow, llm=client, budget=20000, trace=False, limit_mode="explicit", + ) + assert result.status == "completed" + assert result.tokens == 1504 + assert seen == [1500] + assert [request["observed"] for request in requests] == [0, 1500] + + +@pytest.mark.parametrize("budget, retries, count", [(2000, 1, 1), (20000, 1, 2), (None, 1, 2), (2000, 0, 1)]) +async def test_authored_subprocess_accounts_failure_before_retry( + tmp_path, monkeypatch, attempt_server, budget, retries, count, +): + monkeypatch.setenv("PYTHONPATH", str(REPO_ROOT)) + hosts = [] + url, requests = attempt_server("chat", lambda: (hosts[0].used_tokens, hosts[1].bridge.tokens)) + (tmp_path / "authored.py").write_text( + 'async def main(ctx, args):\n' + f' return await ctx.agent({("context " * 300)!r}, tools=[], budget={budget!r})\n', + encoding="utf-8", + ) + environment = LocalEnvironment(str(tmp_path)) + parent, tool, runtime = _manager(environment, budget=budget) + for target in (tool._config, parent.agent): + target.base_url = url + target.api_key = "controlled-test" # pragma: allowlist secret + target.llm_stream_chat = True + target.llm_max_retries = retries + parent.agent.max_tokens_per_step = 128 + hosts.append(parent) + try: + launched = json.loads(await tool.execute_with_runtime( + {"action": "launch", "source": "authored.py", "python": sys.executable, "budget": budget}, runtime, + )) + child = tool._children[launched["id"]] + hosts.append(child) + await asyncio.wait_for(child.task, 20) + assert child.result["status"] == "completed", child.result + assert len(requests) == count + assert child.tokens == child.bridge.tokens == parent.used_tokens == 1500 + 4 * (count - 1) + if count == 2: + assert requests[1]["observed"] == (1500, 1500) + records = _records(tmp_path) + assert len(records) == 1 + assert records[0]["usage"]["total_tokens"] == parent.used_tokens + finally: + await parent.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize("protocol", ["chat", "responses"]) +async def test_team_observes_failure_spend_before_retry( + tmp_path, monkeypatch, local_environment_factory, attempt_server, protocol, +): + scheduler = Scheduler(session_factory=None, worktree_pool=None, event_sink=EventBus(), max_budget_tokens=2000) + sibling = SessionState(messages=[], used_tokens=750) + scheduler.table.add(SessionControlBlock( + aid=1, parent_aid=0, agent=Agent(name="sibling", system_prompt="Finish."), state=sibling, + )) + url, requests = attempt_server(protocol) + + def backoff(_error): + assert scheduler.used_tokens == 2250 + assert scheduler.budget_exhausted + return 0.0 + + monkeypatch.setattr(retry, "extract_retry_after_seconds", backoff) + async with _client(protocol, url) as client: + session = build_session( + env=local_environment_factory(), llm=client, max_budget_tokens=20000, + agent=Agent(name="worker", model="gpt-4o", system_prompt="Finish.", max_tokens_per_step=128), + team_budget_exhausted=lambda: scheduler.budget_exhausted, + ) + scheduler.table.add(SessionControlBlock(aid=0, parent_aid=None, agent=session.agent, state=session.state)) + await session.add_user_message("Reply ok.") + assert await session.run_loop() == "" + assert len(requests) == 1 + assert session.used_tokens == 1500 + assert session.phase is SessionPhase.STOPPED + assert "team budget exceeded" in session.state.terminal_reason + assert _records(tmp_path)[0]["usage"]["total_tokens"] == 1500 diff --git a/tests/runtime/test_responses_error_usage_http.py b/tests/runtime/test_responses_error_usage_http.py index 79c448cb..308609c1 100644 --- a/tests/runtime/test_responses_error_usage_http.py +++ b/tests/runtime/test_responses_error_usage_http.py @@ -529,6 +529,7 @@ def retry_after(_error): await session.add_user_message("Give a short answer") turn = asyncio.create_task(session.run_loop()) await asyncio.wait_for(backoff_started.wait(), 1) + assert session.used_tokens == 37 turn.cancel() with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(turn, 1) From e0c179e77110dfbfd286ccc8965bd15246c9b408 Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 14:02:32 +0800 Subject: [PATCH 14/82] fix(workflow): run large candidate recovery commands from scripts --- opencollab/adapters/_candidate_adoption.py | 95 ++++++++++++------- .../test_candidate_adoption_command_size.py | 74 +++++++++++++++ 2 files changed, 137 insertions(+), 32 deletions(-) create mode 100644 tests/workflows/test_candidate_adoption_command_size.py diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index 81047c42..d27b93ea 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -5,14 +5,35 @@ import posixpath import shlex from collections.abc import AsyncIterator, Sequence -from contextlib import asynccontextmanager +from contextlib import AsyncExitStack, asynccontextmanager from typing import Any -from opencollab.adapters._git_patch_bytes import _complete +from opencollab.adapters._git_patch_bytes import _complete, _patch_temporary from opencollab.application.async_timeout import await_owned_operation from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError +@asynccontextmanager +async def _adoption_command( + environment: Any, workspace: str, command: str, *, repository: bool, +) -> AsyncIterator[str]: + if len(command.encode("utf-8", errors="surrogateescape")) <= 32_768: + yield command + return + resources = AsyncExitStack() + try: + script = await resources.enter_async_context(_patch_temporary( + environment, workspace, command + "\n", prefix=".candidate-command-", suffix=".sh", + cleanup_best_effort=True, repository=repository, + )) + yield f"sh {shlex.quote(script)}" + finally: + try: + await await_owned_operation(resources.aclose()) + except Exception: + pass + + @asynccontextmanager async def _adoption_transaction( environment: Any, workspace: str, paths: Sequence[str], *, repository: bool, @@ -26,14 +47,15 @@ async def _adoption_transaction( raise RuntimeError(f"invalid path in candidate patch: {path!r}") # Own a missing or non-directory ancestor too, so rollback also reverses # directory creation and file/directory replacements without following links. - inspected = _complete(await environment.exec_cmd( + inspection = ( f"cd -- {shlex.quote(workspace)} && " f"for path in {shlex.join(paths)}; do root=$path; parent=${{path%/*}}; " 'while [ "$parent" != "$path" ]; do ' 'if [ ! -d "$parent" ] || [ -L "$parent" ]; then root=$parent; fi; ' - 'path=$parent; parent=${path%/*}; done; printf "%s\\0" "$root"; done', - timeout=120, - ), "candidate adoption path inspection") + 'path=$parent; parent=${path%/*}; done; printf "%s\\0" "$root"; done' + ) + async with _adoption_command(environment, workspace, inspection, repository=repository) as command: + inspected = _complete(await environment.exec_cmd(command, timeout=120), "candidate adoption path inspection") roots: list[str] = [] for path in sorted(set(inspected.split("\0")) - {""}): if not any(path.startswith(root + "/") for root in roots): @@ -46,15 +68,18 @@ async def _adoption_transaction( parent = posixpath.dirname(parent) directory_modes: list[tuple[str, str]] = [] if ancestors: - metadata = _complete(await environment.exec_cmd( + inspection = ( f"cd -- {shlex.quote(workspace)} && " f"for path in {shlex.join(sorted(ancestors, key=lambda path: (path.count('/'), path)))}; do " 'if [ -d "$path" ] && [ ! -L "$path" ]; then ' 'mode=$(stat -c %a -- "$path" 2>/dev/null) || ' 'mode=$(stat -f %Lp -- "$path") || exit; ' - 'printf "%s\\0%s\\0" "$path" "$mode"; fi; done', - timeout=120, - ), "candidate ancestor directory inspection").split("\0") + 'printf "%s\\0%s\\0" "$path" "$mode"; fi; done' + ) + async with _adoption_command(environment, workspace, inspection, repository=repository) as command: + metadata = _complete(await environment.exec_cmd( + command, timeout=120, + ), "candidate ancestor directory inspection").split("\0") for path, mode in zip(metadata[::2], metadata[1::2]): if not mode or len(mode) > 4 or any(digit not in "01234567" for digit in mode): raise RuntimeError("candidate ancestor directory mode is invalid") @@ -114,29 +139,35 @@ async def _adoption_transaction( f"elif rm -rf -- {source} && [ ! -e {source} ] && [ ! -L {source} ]; " "then :; else failed=1; fi" ) - _complete(await environment.exec_cmd( - "; ".join(copies), timeout=900, - ), "candidate recovery snapshot") - try: - yield - except BaseException as failure: + recovery_script = "failed=0; " + "; ".join(( + *recreate_directories, *restores, *reversed(restore_directory_modes), + )) + '; [ "$failed" -eq 0 ]' + async with AsyncExitStack() as commands: + snapshot_command = await commands.enter_async_context(_adoption_command( + environment, workspace, "; ".join(copies), repository=repository, + )) + # Store long recovery commands before publishing any source changes. + recovery_command = await commands.enter_async_context(_adoption_command( + environment, workspace, recovery_script, repository=repository, + )) + _complete(await environment.exec_cmd(snapshot_command, timeout=900), "candidate recovery snapshot") try: - _complete(await await_owned_operation(environment.exec_cmd( - "failed=0; " + "; ".join(( - *recreate_directories, *restores, *reversed(restore_directory_modes), - )) + '; [ "$failed" -eq 0 ]', - timeout=900, - )), "candidate adoption recovery") - except BaseException as recovery: - keep_backup = True - revoke = getattr(environment, "revoke", None) - if callable(revoke): - revoke() - raise CandidateAdoptionRecoveryError( - f"candidate adoption failed ({failure}); recovery failed ({recovery}); " - f"original files retained in {backup}" - ) from recovery - raise + yield + except BaseException as failure: + try: + _complete(await await_owned_operation(environment.exec_cmd( + recovery_command, timeout=900, + )), "candidate adoption recovery") + except BaseException as recovery: + keep_backup = True + revoke = getattr(environment, "revoke", None) + if callable(revoke): + revoke() + raise CandidateAdoptionRecoveryError( + f"candidate adoption failed ({failure}); recovery failed ({recovery}); " + f"original files retained in {backup}" + ) from recovery + raise finally: if not keep_backup: try: diff --git a/tests/workflows/test_candidate_adoption_command_size.py b/tests/workflows/test_candidate_adoption_command_size.py new file mode 100644 index 00000000..c9e6f9b1 --- /dev/null +++ b/tests/workflows/test_candidate_adoption_command_size.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import errno +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters._env_local import LocalEnvironment +from tests.workflows.test_workflow_candidate_workspace import _git + + +class _ArgumentBoundEnvironment(LocalEnvironment): + async def exec_cmd(self, command, timeout=120.0, **kwargs): + if len(command.encode()) > 65_536: + raise OSError(errno.E2BIG, "transport command argument is too long") + return await super().exec_cmd(command, timeout, **kwargs) + + +@pytest.mark.parametrize("blocked", [False, True]) +async def test_bulk_candidate_adoption_keeps_recovery_below_argument_limits(tmp_path, blocked): + repo = tmp_path / "repo" + (repo / "src").mkdir(parents=True) + (repo / "settings").mkdir() + names = [f"{number:04d}_" + "module_" * 10 + ".py" for number in range(900)] + for name in names: + (repo / "src" / name).write_text("VALUE = 1\n") + (repo / "settings/value.txt").write_text("original settings\n") + (repo / ".gitignore").write_text("*.cache\n") + _git(repo, "init", "-q") + _git(repo, "config", "user.name", "Test") + _git(repo, "config", "user.email", "test@example.invalid") + _git(repo, "add", ".") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare a bulk candidate") + if blocked: + (repo / "settings/session.cache").write_text("runtime cache\n") + index = (repo / ".git/index").read_bytes() + environment = _ArgumentBoundEnvironment(str(repo)) + + async def edit(child, inputs): + folder = Path(child.workspace_root) + for name in names: + (folder / "src" / name).write_text("VALUE = 2\n") + (folder / "settings/value.txt").unlink() + (folder / "settings").rmdir() + (folder / "settings").write_text("new settings\n") + return "updated" + + async def flow(ctx, inputs): + candidate = await ctx.candidate_workflow(edit, {}, label="bulk-rewrite") + assert len(candidate.effective_diff.encode()) < 1_000_000 + if blocked: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await ctx.adopt_candidate(candidate) + else: + await ctx.adopt_candidate(candidate) + return "finished" + + try: + result = await OpenCollab(repo, environment=environment).workflow(flow, {}, trace=False) + assert result.ok, result.reason + assert not environment.revoked + expected = "VALUE = 1\n" if blocked else "VALUE = 2\n" + assert all((repo / "src" / name).read_text() == expected for name in names) + if blocked: + assert (repo / "settings/value.txt").read_text() == "original settings\n" + assert (repo / "settings/session.cache").read_text() == "runtime cache\n" + else: + assert (repo / "settings").read_text() == "new settings\n" + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not list((repo / ".git").rglob(".candidate-command-*.sh")) + finally: + await environment.cleanup() From 3f3f8fd517c8fd749bd229a396f4618021a10a89 Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 14:05:52 +0800 Subject: [PATCH 15/82] fix(llm): preserve caller timeout errors in Responses callbacks Keep callback TimeoutError instances distinct from the Responses round deadline, including unlimited request timeouts. Retain known attempt usage when a callback stops the call. Cover each callback position with finite and unlimited request timeouts, with provider retry time budgets enabled and disabled. --- opencollab/adapters/llm/responses_provider.py | 21 ++++++-- .../test_provider_attempt_budget_http.py | 51 ++++++++++++++++++- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/opencollab/adapters/llm/responses_provider.py b/opencollab/adapters/llm/responses_provider.py index b95b01e2..f125edb6 100644 --- a/opencollab/adapters/llm/responses_provider.py +++ b/opencollab/adapters/llm/responses_provider.py @@ -712,14 +712,25 @@ async def complete_responses( capabilities = model_capabilities(model) stream = stream and capabilities.supports_responses_streaming failed_usages: list[Usage | None] = [] + callback_timeout: asyncio.TimeoutError | None = None def prepare_attempt() -> None: - if before_attempt is not None and (limit := before_attempt()) is not None: - kwargs["max_output_tokens"] = limit + nonlocal callback_timeout + try: + if before_attempt is not None and (limit := before_attempt()) is not None: + kwargs["max_output_tokens"] = limit + except asyncio.TimeoutError as exc: + callback_timeout = exc + raise def report_failure(_exc: BaseException) -> None: - if failed_usages and (usage := failed_usages[-1]) is not None and on_attempt_usage is not None: - on_attempt_usage(usage) + nonlocal callback_timeout + try: + if failed_usages and (usage := failed_usages[-1]) is not None and on_attempt_usage is not None: + on_attempt_usage(usage) + except asyncio.TimeoutError as exc: + callback_timeout = exc + raise async def request_once() -> LLMResponse: terminal = None @@ -782,6 +793,8 @@ async def run() -> LLMResponse: else: response = await asyncio.wait_for(run(), timeout=round_timeout) except asyncio.TimeoutError as exc: + if exc is callback_timeout or round_timeout is None: + raise raise ResponsesProtocolError(f"Responses round deadline exceeded after {round_timeout:g}s") from exc except BaseException as exc: usage = _combine_responses_usage(failed_usages) diff --git a/tests/runtime/test_provider_attempt_budget_http.py b/tests/runtime/test_provider_attempt_budget_http.py index 22390cf3..7a629929 100644 --- a/tests/runtime/test_provider_attempt_budget_http.py +++ b/tests/runtime/test_provider_attempt_budget_http.py @@ -117,13 +117,14 @@ def do_POST(self): thread.join() -def _client(protocol, url, *, retries=1): +def _client(protocol, url, *, retries=1, **kwargs): return LLMClient( provider="anthropic" if protocol == "anthropic" else "openai", model="claude-sonnet-4-6" if protocol == "anthropic" else "gpt-4o", wire_protocol="responses" if protocol == "responses" else "chat_completions", api_key="controlled-test", base_url=url, # pragma: allowlist secret stream_chat=protocol == "chat", max_retries=retries, + **kwargs, ) @@ -261,6 +262,54 @@ def before_request(): assert records[0]["usage"]["total_tokens"] == expected +@pytest.mark.parametrize("callback", ["before_first", "admitted", "before_retry", "attempt", "final"]) +@pytest.mark.parametrize("timeout", [None, 600.0]) +@pytest.mark.parametrize("retry_budget", [0, 30]) +async def test_responses_preserves_callback_timeout_identity( + tmp_path, attempt_server, callback, timeout, retry_budget, +): + url, requests = attempt_server("responses") + original = TimeoutError("owned caller deadline") + checks, attempts, finals = [], [], [] + + def before_request(): + checks.append(True) + if callback == "before_first" or callback == "before_retry" and len(checks) == 2: + raise original + return 128 + + def on_admitted(): + if callback == "admitted": + raise original + + def on_attempt(usage): + attempts.append(usage.total_tokens) + if callback == "attempt": + raise original + + def on_usage(usage): + finals.append(usage.total_tokens) + if callback == "final": + raise original + + async with _client("responses", url, request_timeout=timeout, provider_error_time_budget=retry_budget) as client: + with pytest.raises(TimeoutError) as error: + await client.complete( + [{"role": "user", "content": "Reply ok."}], before_request=before_request, + on_admitted=on_admitted, on_attempt_usage=on_attempt, on_usage=on_usage, + ) + assert error.value is original + count = 0 if callback in {"before_first", "admitted"} else 2 if callback == "final" else 1 + expected = 0 if not count else 1500 + 4 * (count - 1) + assert len(requests) == count + assert attempts == ([1500] if count else []) + assert finals == ([expected] if count else []) + assert getattr(getattr(error.value, "usage", None), "total_tokens", 0) == expected + records = _records(tmp_path) + assert len(records) == 1 + assert records[0]["usage"]["total_tokens"] == expected + + @pytest.mark.parametrize("protocol", ["chat", "responses"]) async def test_workflow_observes_failure_spend_during_retry(tmp_path, monkeypatch, attempt_server, protocol): contexts, seen = [], [] From f2ca14c9864437a5bd820eb5aeae2d9a79426b5b Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 14:11:29 +0800 Subject: [PATCH 16/82] fix(docker): lock physical ancestors for aliased edits Resolve original edit paths and alias ancestors before acquiring the complete ordered native lock group. Include physical ancestors while preserving alias nodes, relative execution directories, and symlink parent traversal. Add SDK publication and cancellation controls with distinct guest and host namespaces, plus native lock ordering and cancelled waiter coverage. --- opencollab/adapters/_env_docker.py | 48 ++++- opencollab/adapters/tools/base.py | 6 +- .../test_candidate_docker_alias_mutations.py | 193 ++++++++++++++++++ 3 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 tests/workflows/test_candidate_docker_alias_mutations.py diff --git a/opencollab/adapters/_env_docker.py b/opencollab/adapters/_env_docker.py index 4996aa14..d90e059a 100644 --- a/opencollab/adapters/_env_docker.py +++ b/opencollab/adapters/_env_docker.py @@ -13,7 +13,7 @@ import stat import threading import uuid -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncIterator, Callable, Sequence from dataclasses import dataclass, field from typing import NoReturn @@ -30,6 +30,7 @@ timed_out_result, ) from opencollab.adapters._safe_file_common import _normalize_newlines, _split_lines +from opencollab.adapters.tools.base import _write_lock_paths from opencollab.application.async_timeout import await_owned_operation from opencollab.application.exception_notes import add_exception_note from opencollab.domain.docker import DockerOptions @@ -691,6 +692,41 @@ async def write_file(self, path: str, content: str) -> None: async def write_file_with_change(self, path: str, content: str) -> bool | None: return await self._write_file(path, content, observe_change=True) + @contextlib.asynccontextmanager + async def file_write_locks(self, paths: Sequence[str]) -> AsyncIterator[None]: + self._ensure_active() + await self._bind_attached() + if self._container_id is None: + raise RuntimeError("Container not started. Call setup() first.") + targets = { + await self._resolve_write_target(self._validate_container_path(path)) + for path in paths + } + physical_workspace = self._resolved_workspace or self.workspace + ancestors = set() + for path in paths: + parent = posixpath.dirname(path) + while parent not in ("", ".", "/", self.workspace, physical_workspace): + if posixpath.isabs(parent) and not any( + _within_directory(root, posixpath.normpath(parent)) + for root in (self.workspace, physical_workspace) + ): + break + if posixpath.basename(parent) not in (".", ".."): + ancestors.add(parent) + parent = posixpath.dirname(parent) + targets.update({ + await self._resolve_write_target(self._validate_container_path(path)) + for path in ancestors + }) + # Preserve alias nodes and add physical ancestors before taking any + # locks, so directory recovery and alias edits share one lock order. + ordered = _write_lock_paths(targets, self, workspace=physical_workspace) + async with contextlib.AsyncExitStack() as locks: + for target in ordered: + await locks.enter_async_context(self._resolved_file_write_lock(target)) + yield None + @contextlib.asynccontextmanager async def file_write_lock( self, @@ -701,12 +737,16 @@ async def file_write_lock( """Hold the target lock across a native edit, including its final write.""" self._ensure_active() await self._bind_attached() - container_id = self._container_id - if container_id is None: + if self._container_id is None: raise RuntimeError("Container not started. Call setup() first.") target = self._validate_container_path(path) target = await self._resolve_write_target(target, containment_root=containment_root) - key = f"{container_id}\0{target}" + async with self._resolved_file_write_lock(target): + yield target + + @contextlib.asynccontextmanager + async def _resolved_file_write_lock(self, target: str) -> AsyncIterator[str]: + key = f"{self._container_id}\0{target}" task = asyncio.current_task() assert task is not None # Native tools already hold this lock when they call write_file. Only diff --git a/opencollab/adapters/tools/base.py b/opencollab/adapters/tools/base.py index f9a04301..a4d2651e 100644 --- a/opencollab/adapters/tools/base.py +++ b/opencollab/adapters/tools/base.py @@ -30,10 +30,12 @@ def _host_lock_path(path: str, env: Any) -> str: digest = hashlib.sha256(identity.encode("utf-8", errors="surrogatepass")).hexdigest() return os.path.join(_host_lock_root(), f"{digest}.lock") -def _write_lock_paths(paths: Sequence[str], env: Any) -> list[str]: +def _write_lock_paths( + paths: Sequence[str], env: Any, *, workspace: str | None = None, +) -> list[str]: local = bool(getattr(env, "local_filesystem", False)) path_module = os.path if local else posixpath - workspace = path_module.abspath(getattr(env, "workspace", "") or ".") + workspace = path_module.abspath(workspace or getattr(env, "workspace", "") or ".") if local: workspace = os.path.realpath(workspace) targets = set() diff --git a/tests/workflows/test_candidate_docker_alias_mutations.py b/tests/workflows/test_candidate_docker_alias_mutations.py new file mode 100644 index 00000000..11c0af4e --- /dev/null +++ b/tests/workflows/test_candidate_docker_alias_mutations.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters._env_docker import DockerEnvironment +from opencollab.adapters._env_process import ProcessResult +from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace +from opencollab.adapters.tools.base import host_write_locks +from opencollab.adapters.tools.fs import FileWriteTool +from opencollab.application.tool_execution import ToolRuntime +from opencollab.tools import builtin_tools +from tests.support.docker_native_edits_support import LocalDockerTransport +from tests.workflows.test_candidate_adoption_recovery import _flatten, _repository, _run +from tests.workflows.test_candidate_shared_mutations import _WriterModel +from tests.workflows.test_workflow_candidate_workspace import _git + + +class _RemoteNamespaceTransport(LocalDockerTransport): + remote = "/oc-native-alias-workspace" + + async def __call__(self, *argv, **kwargs): + actual = str(self.workspace) + mapped = tuple(arg.replace(self.remote, actual) if isinstance(arg, str) else arg for arg in argv) + result = await super().__call__(*mapped, **kwargs) + return ProcessResult(result.returncode, result.stdout.replace(actual.encode(), self.remote.encode()), + result.stderr.replace(actual.encode(), self.remote.encode())) + + +class _RemoteCandidates(EnvCandidateWorkspace): + def __init__(self, environment, transport): + super().__init__(environment) + self.transport = transport + + async def _candidate_environment(self, path): + environment = await super()._candidate_environment(path) + self.transport.attach(environment) + return environment + + +def _remote_environment(repo): + transport = _RemoteNamespaceTransport(repo, pair_reads=False) + environment = DockerEnvironment( + workspace=transport.remote, container_id="a" * 64, exec_workdir=transport.remote, + command_prefix="export OC_NATIVE_EDIT_PROBE=wrapped", + ) + transport.attach(environment) + return environment, transport + + +@pytest.mark.parametrize("tool", ["file_write", "apply_patch"]) +@pytest.mark.parametrize("path", ["alias/runtime.cache", "settings/runtime.cache", "unrelated.cache"]) +@pytest.mark.parametrize("cancel", [False, True]) +async def test_sdk_remote_alias_mutations_survive_directory_recovery(tmp_path, monkeypatch, tool, path, cancel): + repo = _repository(tmp_path) + (repo / ".gitignore").write_text("__pycache__/\n*.cache\n") + (repo / "alias").symlink_to("settings", target_is_directory=True) + _git(repo, "add", ".gitignore", "alias") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: retain workspace directory alias") + for cached in ("settings/runtime.cache", "unrelated.cache"): + (repo / cached).write_text("VALUE = 1\n") + index = (repo / ".git/index").read_bytes() + environment, transport = _remote_environment(repo) + entered, release = asyncio.Event(), asyncio.Event() + execute = environment.exec_cmd + publications = [] + + async def publication_barrier(cmd, timeout=120, **kwargs): + publication = " apply --binary " in cmd and ".candidate-adopt-" in cmd + if publication: + entered.set() + await release.wait() + result = await execute(cmd, timeout, **kwargs) + if publication: + publications.append(result.returncode) + return result + + monkeypatch.setattr(environment, "exec_cmd", publication_barrier) + model = _WriterModel(tool, path) + + async def edit(child, _): + _flatten(Path(child.workspace_root)) + return "prepared" + + async def flow(ctx, _): + candidate = await ctx.candidate_workflow(edit, {}, label="flatten-directory") + publication = asyncio.create_task(ctx.adopt_candidate(candidate)) + writer = None + try: + await asyncio.wait_for(entered.wait(), 15) + writer = asyncio.create_task(ctx.agent_run( + "Set the runtime cache value to 100", tools=list(builtin_tools(tool, headless=False)), + isolation="shared", budget=2000, system_prompt="Use the requested file tool", + )) + await asyncio.wait_for(model.requested.wait(), 10) + if path == "unrelated.cache": + assert (await asyncio.wait_for(asyncio.shield(writer), 10)).status == "completed" + else: + await asyncio.sleep(0.1) + assert not writer.done() + assert (repo / path).read_text() == "VALUE = 1\n" + if cancel: + publication.cancel() + await asyncio.sleep(0) + publication.cancel() + assert not publication.done() + release.set() + if cancel: + with pytest.raises(asyncio.CancelledError): + await publication + else: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await publication + assert (await asyncio.wait_for(writer, 10)).status == "completed" + assert (repo / path).read_text() == "VALUE = 100\n" + assert len(publications) == 1 and publications[0] != 0 + return "retained" + finally: + release.set() + await asyncio.gather(publication, *([writer] if writer is not None else []), return_exceptions=True) + + try: + result = await OpenCollab(repo, environment=environment, model="local-scripted").workflow( + flow, llm=model, candidate_workspace=_RemoteCandidates(environment, transport), trace=False, + timeout=60, budget=10000, concurrency=2, limit_mode="explicit", agent_profile="single2", + ) + assert result.ok, result.reason + assert result.output == "retained" + assert _run(repo) == "1\n" + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not environment._temporary_files + assert not environment.revoked + finally: + release.set() + await environment.cleanup() + + +async def test_remote_alias_group_prepares_locks_before_acquisition_and_releases_cancelled_waiter( + tmp_path, monkeypatch, +): + (tmp_path / "settings").mkdir() + (tmp_path / "settings/value.py").write_text("VALUE = 1\n") + (tmp_path / "alias").symlink_to("settings", target_is_directory=True) + environment, _ = _remote_environment(tmp_path) + from opencollab.adapters import _env_docker as docker_module + resolve = environment._resolve_write_target + resolving_task = asyncio.current_task() + resolutions = [] + + async def resolve_before_locks(target, **kwargs): + assert not any(owner is resolving_task for owner in docker_module._WRITE_LOCK_OWNERS.values()) + result = await resolve(target, **kwargs) + resolutions.append(result) + return result + + monkeypatch.setattr(environment, "_resolve_write_target", resolve_before_locks) + waiting = None + try: + async with host_write_locks((environment.workspace + "/alias/value.py",), environment): + owned = [key.split("\0", 1)[1] for key, owner in docker_module._WRITE_LOCK_OWNERS.items() + if owner is resolving_task] + assert owned == sorted(owned) + assert owned == [environment.workspace + suffix for suffix in ( + "/alias", "/settings", "/settings/value.py", + )] + monkeypatch.setattr(environment, "_resolve_write_target", resolve) + async with host_write_locks((environment.workspace + "/settings",), environment): + waiting = asyncio.create_task(FileWriteTool().execute_with_runtime({ + "path": "alias/value.py", "mode": "create", "content": "VALUE = 100\n", + }, ToolRuntime(environment, None, None))) + await asyncio.sleep(0.1) + assert not waiting.done() + alias_key = environment._container_id + "\0" + environment.workspace + "/alias" + assert alias_key in docker_module._WRITE_LOCK_OWNERS + waiting.cancel() + with pytest.raises(asyncio.CancelledError): + await waiting + assert alias_key not in docker_module._WRITE_LOCK_OWNERS + assert resolutions + output = await asyncio.wait_for(FileWriteTool().execute_with_runtime({ + "path": "alias/value.py", "mode": "create", "content": "VALUE = 100\n", + }, ToolRuntime(environment, None, None)), 5) + assert "Created/wrote" in output + assert (tmp_path / "settings/value.py").read_text() == "VALUE = 100\n" + assert not docker_module._WRITE_LOCK_OWNERS + finally: + if waiting is not None: + await asyncio.gather(waiting, return_exceptions=True) + await environment.cleanup() From de5b3cec25de54523cce24d8b4e196cdb1b3835d Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 15:07:17 +0800 Subject: [PATCH 17/82] fix(llm): prepare first request at client admission Restore initial request validation and admission notification immediately after acquiring the request slot. Cache the initial output limit for the first provider hook and revalidate subsequent attempts without repeating the first callback. Exercise provider replacements and native hooks across all three protocols. Keep answer-retention tests focused on admitted retries whose reported input exceeds the estimate and exhausts the budget after returning useful text. --- opencollab/adapters/llm/client.py | 26 +++++---- .../interfaces/test_sdk_budget_stop_answer.py | 14 +++-- tests/runtime/test_model_request_admission.py | 55 +++++++++++++++++++ 3 files changed, 81 insertions(+), 14 deletions(-) diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index dd660e75..7e8579cf 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -318,20 +318,23 @@ async def complete( start = time.monotonic() response = None settled_usage = None - admitted = False + first_attempt = True + initial_limit: int | None = None - def prepare_attempt() -> int | None: - nonlocal admitted + def request_limit() -> int | None: limit = max_output_tokens if before_request is not None: current_limit = before_request() limit = current_limit if limit is None else min(limit, current_limit) - if not admitted: - admitted = True - if on_admitted is not None: - on_admitted() return limit + def prepare_attempt() -> int | None: + nonlocal first_attempt + if first_attempt: + first_attempt = False + return initial_limit + return request_limit() + retry_budget = ( self.provider_retry_budget if self._retry_budget_explicit @@ -342,6 +345,9 @@ def prepare_attempt() -> int | None: async with AsyncExitStack() as request_owner: try: await request_owner.enter_async_context(request_slot(self.request_pool, self.request_capacity)) + initial_limit = request_limit() + if on_admitted is not None: + on_admitted() if self._anthropic: response = await complete_anthropic( self._anthropic, @@ -354,7 +360,7 @@ def prepare_attempt() -> int | None: thinking_params=thinking_params, tool_choice=tool_choice, top_p=top_p, - max_output_tokens=max_output_tokens, + max_output_tokens=initial_limit, reasoning_effort=reasoning_effort, provider_error_time_budget=retry_budget, before_attempt=prepare_attempt, @@ -372,7 +378,7 @@ def prepare_attempt() -> int | None: thinking_params=thinking_params, tool_choice=tool_choice, top_p=top_p, - max_output_tokens=max_output_tokens, + max_output_tokens=initial_limit, reasoning_effort=reasoning_effort, prompt_cache_namespace=self._responses_prompt_cache_namespace, response_session_id=response_session_id or uuid.uuid4().hex, @@ -399,7 +405,7 @@ def prepare_attempt() -> int | None: thinking_params=thinking_params, tool_choice=tool_choice, top_p=top_p, - max_output_tokens=max_output_tokens, + max_output_tokens=initial_limit, reasoning_effort=reasoning_effort, provider_error_time_budget=retry_budget, before_attempt=prepare_attempt, diff --git a/tests/interfaces/test_sdk_budget_stop_answer.py b/tests/interfaces/test_sdk_budget_stop_answer.py index 42ba5113..dac26d17 100644 --- a/tests/interfaces/test_sdk_budget_stop_answer.py +++ b/tests/interfaces/test_sdk_budget_stop_answer.py @@ -8,12 +8,15 @@ import pytest from opencollab import BudgetDecision, OpenCollab, RunControl +from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.tools.base import Tool from opencollab.bootstrap import agent_runtime from tests.support.provider_sdk_http import http_response, install_sdk_transport from tests.support.responses_provider_test_support import function_item, message_item ANSWER = "The measured total is 42. The calculation is complete." +REQUEST_INPUT_ESTIMATE = 500 +INITIAL_BUDGET = 1075 PROMPT = "Read these notes and return the measured total.\n" + ( "The measured count was recorded as 42 and checked by the operator.\n" * 30 ) @@ -41,7 +44,7 @@ async def test_budget_stop_preserves_answer_after_native_retry(tmp_path, monkeyp retry = case not in {"single", "truncated"} with_tool = case.endswith("_tool") has_text = case not in {"soft_tool", "hard_tool"} - hard_budget = 1000 if case.startswith("hard_") else 3000 + hard_budget = INITIAL_BUDGET if case.startswith("hard_") else 3000 expected_tokens = 1115 if retry else 565 stopped_for_budget = case.startswith(("soft_", "hard_")) original = agent_runtime.build_session @@ -93,6 +96,7 @@ def handler(request): monkeypatch.delenv("OPENCOLLAB_UNBOUNDED_LIMITS", raising=False) monkeypatch.delenv("OPENCOLLAB_EXTERNAL_PROVIDER_ISOLATION", raising=False) monkeypatch.setattr(agent_runtime, "build_session", observed_session) + monkeypatch.setattr(LLMClient, "estimate_request_tokens", lambda *_args, **_kwargs: REQUEST_INPUT_ESTIMATE) install_sdk_transport(monkeypatch, "openai", handler) artifacts = tmp_path / "artifacts" result = await OpenCollab( @@ -102,7 +106,7 @@ def handler(request): ).agent2( PROMPT, tools=(tool,) if with_tool else (), system_prompt="Answer the user using the supplied notes.", budget=hard_budget, max_steps=4, trace=trace, artifacts=artifacts, - run_control=RunControl(initial_soft_budget_tokens=1000, decide_budget=decide, on_event=events.append), + run_control=RunControl(initial_soft_budget_tokens=INITIAL_BUDGET, decide_budget=decide, on_event=events.append), ) saved = OpenCollab.read_session_snapshot(artifacts / "agent.json") messages = saved["messages"] @@ -127,7 +131,9 @@ def handler(request): assert all((0 if retry and index == 0 else 15) <= body["max_output_tokens"] for index, body in enumerate(requests)) request_snapshot = next(snapshot for snapshot in snapshots if snapshot.reason == "request") - assert request_snapshot.reserved_input_tokens >= 550 + assert request_snapshot.reserved_input_tokens == REQUEST_INPUT_ESTIMATE + if retry: + assert requests[1]["max_output_tokens"] == INITIAL_BUDGET - 550 - REQUEST_INPUT_ESTIMATE == 25 assert [event.data["total_tokens"] for event in events if event.type == "usage"] == [expected_tokens] assert len(ledger) == 1 and ledger[0]["usage"]["total_tokens"] == expected_tokens assert ledger[0]["usage"]["input_tokens"] == (1100 if retry else 550) @@ -141,7 +147,7 @@ def handler(request): assert all(not message.get("response_items") and not message.get("provider_state") for message in messages) if case.startswith("soft_"): assert snapshots[-1].reason == "after_response" - assert snapshots[-1].used_tokens == expected_tokens and snapshots[-1].soft_budget_tokens == 1000 + assert snapshots[-1].used_tokens == expected_tokens and snapshots[-1].soft_budget_tokens == INITIAL_BUDGET else: assert all(snapshot.used_tokens < hard_budget for snapshot in snapshots) elif case == "growth": diff --git a/tests/runtime/test_model_request_admission.py b/tests/runtime/test_model_request_admission.py index 108b3306..8cc116e1 100644 --- a/tests/runtime/test_model_request_admission.py +++ b/tests/runtime/test_model_request_admission.py @@ -4,6 +4,7 @@ import os import subprocess import sys +from contextlib import asynccontextmanager import pytest @@ -14,6 +15,60 @@ from tests.support.paths import PACKAGE_ROOT +@pytest.mark.parametrize("configuration, provider_function", [ + ({"model": "gpt-4o"}, "complete_openai"), + ({"model": "gpt-4o", "wire_protocol": "responses"}, "complete_responses"), + ({"model": "claude-sonnet-4-6", "provider": "anthropic"}, "complete_anthropic"), +], ids=["chat", "responses", "anthropic"]) +@pytest.mark.parametrize("consume_hooks", [False, True]) +@pytest.mark.parametrize("configured_limit", [None, 50, 128]) +async def test_slot_admission_prepares_first_request_before_provider_entry( + tmp_path, monkeypatch, configuration, provider_function, consume_hooks, configured_limit, +): + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + queued = asyncio.Event() + events = [] + limits = iter([80, 31]) + native_slot = transport.request_slot + first_limit = 80 if configured_limit is None else min(configured_limit, 80) + + @asynccontextmanager + async def observed_slot(*args): + queued.set() + async with native_slot(*args): + yield + + def revalidate(): + limit = next(limits) + events.append(f"limit_{limit}") + return limit + + async def complete(*_args, **kwargs): + events.append("provider") + assert events == ["limit_80", "admitted", "provider"] + assert kwargs["max_output_tokens"] == first_limit + if consume_hooks: + assert kwargs["before_attempt"]() == first_limit + assert events == ["limit_80", "admitted", "provider"] + assert kwargs["before_attempt"]() == 31 + return LLMResponse(content="answer") + + monkeypatch.setattr(transport, "request_slot", observed_slot) + monkeypatch.setattr(transport, provider_function, complete) + async with LLMClient( + **configuration, api_key="local-test", request_pool=str(tmp_path), request_capacity=1, + ) as client: + async with native_slot(str(tmp_path), 1): + owner = asyncio.create_task(client.complete( + [], max_output_tokens=configured_limit, before_request=revalidate, + on_admitted=lambda: events.append("admitted"), + )) + await asyncio.wait_for(queued.wait(), 1) + assert events == [] + assert (await asyncio.wait_for(owner, 1)).content == "answer" + assert events == ["limit_80", "admitted", "provider", *(["limit_31"] if consume_hooks else [])] + + async def test_cancellation_retains_slot_until_native_cleanup_finishes(tmp_path, monkeypatch): entered = asyncio.Event() cleaning = asyncio.Event() From 70be4fa786f3742b58dea4bf5ce821ec36c8fc2a Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 16:57:52 +0800 Subject: [PATCH 18/82] fix(anthropic): invalidate native history after transcript edits --- opencollab/application/_anthropic_history.py | 6 +- opencollab/application/session_snapshot.py | 14 ++- .../test_anthropic_preserved_history.py | 112 ++++++++++++++++++ 3 files changed, 129 insertions(+), 3 deletions(-) diff --git a/opencollab/application/_anthropic_history.py b/opencollab/application/_anthropic_history.py index 52e2cf51..d8fc4a61 100644 --- a/opencollab/application/_anthropic_history.py +++ b/opencollab/application/_anthropic_history.py @@ -35,10 +35,13 @@ def _edited_anthropic_history(before: list[dict], after: list[dict]) -> list[dic return after def _anthropic_history_input(state: SessionState) -> list[dict]: - """Append fresh transcript messages to the last native provider view.""" + """Reuse the native view until its original transcript prefix changes.""" history = state.anthropic_history if history is None: return state.messages + source = history.get("source_messages") + if source is not None and state.messages[:len(source)] != source: + return _edited_anthropic_history(source, state.messages) return copy.deepcopy([*history["messages"], *state.messages[history["source_count"]:]]) def _remember_anthropic_history(state: SessionState, messages: list[dict], tools: list[dict] | None) -> list[dict]: @@ -51,6 +54,7 @@ def _remember_anthropic_history(state: SessionState, messages: list[dict], tools messages = _strip_anthropic_thinking(messages) state.anthropic_history = copy.deepcopy({ "messages": messages, "source_count": len(state.messages), "tools": tools, + "source_messages": state.messages, }) return messages diff --git a/opencollab/application/session_snapshot.py b/opencollab/application/session_snapshot.py index 5b12ed94..c031540f 100644 --- a/opencollab/application/session_snapshot.py +++ b/opencollab/application/session_snapshot.py @@ -21,13 +21,23 @@ def _restore_anthropic_history(value: object, messages: list[dict]) -> dict | No count = value.get("source_count") view = value.get("messages") tools = value.get("tools") + source = value.get("source_messages") if ( - isinstance(count, bool) or not isinstance(count, int) or not 0 <= count <= len(messages) + isinstance(count, bool) or not isinstance(count, int) or count < 0 or not isinstance(view, list) or not all(isinstance(message, dict) for message in view) or (tools is not None and (not isinstance(tools, list) or not all(isinstance(tool, dict) for tool in tools))) ): return None - return copy.deepcopy({"messages": view, "source_count": count, "tools": tools}) + if source is None: + if count > len(messages): + return None + source = messages[:count] + elif ( + not isinstance(source, list) or len(source) != count + or not all(isinstance(message, dict) for message in source) + ): + return None + return copy.deepcopy({"messages": view, "source_count": count, "tools": tools, "source_messages": source}) def _restore_phase(raw: object) -> SessionPhase: value = str(raw) diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index c39d4771..0c55986e 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -11,10 +11,13 @@ from opencollab.adapters.env import LocalEnvironment from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.tools.fs import FileReadTool, FileWriteTool +from opencollab.application._anthropic_history import _anthropic_history_input, _remember_anthropic_history +from opencollab.application.session_snapshot import _restore_anthropic_history from opencollab.application.shaping import AutoCompactShaper, PerToolResultBudgetShaper, ToolOutputClearShaper from opencollab.application.steering import READS_NUDGE_HARD from opencollab.bootstrap import build_session, load_session from opencollab.domain.agent import Agent +from opencollab.domain.session import SessionState from tests.support.provider_sdk_http import http_response, install_sdk_transport _THINKING = {"thinking", "redacted_thinking"} @@ -333,3 +336,112 @@ async def test_prefix_constraint_uses_native_transport_and_documented_models(mod async with LLMClient(model=model, provider=provider, api_key="controlled-test", # pragma: allowlist secret base_url="https://controlled.invalid/v1") as client: assert client.requires_thinking_prefix is expected + + +@pytest.mark.parametrize("moment", ["live", "before-save", "after-load"]) +@pytest.mark.parametrize("edit", ["text", "nested", "replacement", "deletion", "tail-deletion", "system"]) +async def test_source_edits_reach_native_requests_and_retire_only_invalid_thinking( + monkeypatch, tmp_path, moment, edit, +): + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + peer = _BoundThinkingProvider(reads=0) + install_sdk_transport(monkeypatch, "anthropic", peer) + path = tmp_path / "session.json" + + def apply_edit(session): + if edit == "text": + session.messages[3]["content"] = "Choose green" + elif edit == "nested": + session.messages[3]["content"][0]["text"] = "Choose green" + elif edit == "replacement": + session.messages[3] = {"role": "user", "content": "Choose green"} + elif edit == "deletion": + del session.messages[3] + elif edit == "tail-deletion": + del session.messages[3:] + else: + session.messages[0]["content"] = "Follow the revised instructions" + + async with _client() as client: + session = build_session(agent=_agent(), llm=client, env=LocalEnvironment(str(tmp_path)), + max_budget_tokens=100_000, max_steps=20) + try: + await session.add_user_message("Answer the first question") + assert await session.run_loop() == "done" + await session.add_user_message("Choose red") + session.messages[-1]["content"] = [{"type": "text", "text": "Choose red"}] + assert await session.run_loop() == "done" + if moment != "after-load": + apply_edit(session) + if moment != "live": + session.save(str(path)) + await session.aclose() + await session.env.cleanup() + session = load_session(str(path), agent=_agent(), llm=client, + env=LocalEnvironment(str(tmp_path)), max_budget_tokens=100_000, max_steps=20) + if moment == "after-load": + apply_edit(session) + edited = copy.deepcopy(session.messages) + await session.add_user_message("Continue using the current instructions") + assert await session.run_loop() == "done" + assert session.messages[:len(edited)] == edited + await session.add_user_message("Continue once more") + assert await session.run_loop() == "done" + finally: + await session.aclose() + await session.env.cleanup() + + assert len(peer.requests) == 4 + body = peer.requests[2] + if edit == "system": + assert body["system"] == "Follow the revised instructions" + assert _signatures(body) == [] + else: + assert _signatures(body) == ["signed-1", "redacted-1"] + assert "Choose red" not in json.dumps(body) + if edit not in {"deletion", "tail-deletion"}: + assert "Choose green" in json.dumps(body) + assert _signatures(peer.requests[-1]) == [*_signatures(body), "signed-3", "redacted-3"] + + +@pytest.mark.parametrize("legacy", [False, True]) +def test_restored_unchanged_prefix_preserves_projection_and_tracks_future_nested_edits(legacy): + state = SessionState(messages=[ + {"role": "system", "content": "Identity"}, + {"role": "user", "content": [{"type": "text", "text": "Old requirement"}]}, + {"role": "assistant", "content": "Large old result"}, + ]) + view = [state.messages[0], {"role": "system", "compacted": True, "content": "Earlier context"}] + assert _remember_anthropic_history(state, view, None) == view + stored = copy.deepcopy(state.anthropic_history) + if legacy: + del stored["source_messages"] + state.anthropic_history = _restore_anthropic_history(stored, state.messages) + assert _anthropic_history_input(state) == view + state.append_message({"role": "user", "content": "New request"}) + assert _anthropic_history_input(state) == [*view, state.messages[-1]] + assert "Large old result" not in json.dumps(_anthropic_history_input(state)) + state.messages[1]["content"][0]["text"] = "Revised requirement" + assert _anthropic_history_input(state) == state.messages + assert state.anthropic_history["source_messages"][1]["content"][0]["text"] == "Old requirement" + + +@pytest.mark.parametrize("source", ["invalid", [], [{}, {}], [False]]) +def test_restore_rejects_malformed_optional_source_prefix(source): + message = {"role": "user", "content": "Request"} + assert _restore_anthropic_history({ + "messages": [message], "source_count": 1, "tools": None, "source_messages": source, + }, [message]) is None + + +@pytest.mark.parametrize("count", [0, 2]) +def test_legacy_native_history_restores_without_an_optional_source_prefix(count): + messages = [{"role": "user", "content": "First"}, {"role": "assistant", "content": "Answer"}] + view = [{"role": "system", "compacted": True, "content": "Earlier context"}] + restored = _restore_anthropic_history({"messages": view, "source_count": count, "tools": None}, messages) + assert restored == {"messages": view, "source_count": count, "tools": None, "source_messages": messages[:count]} + messages[0]["content"] = "Changed" + if count: + assert restored["source_messages"][0]["content"] == "First" From 87d8d4be486c61b1a3e1c78fbc348d7466b0a95b Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 16:57:53 +0800 Subject: [PATCH 19/82] fix(docker): retain temporary ownership before publication --- opencollab/adapters/_env_docker.py | 3 +- .../test_docker_owned_temp_cancellation.py | 59 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 tests/runtime/test_docker_owned_temp_cancellation.py diff --git a/opencollab/adapters/_env_docker.py b/opencollab/adapters/_env_docker.py index d90e059a..88d7c187 100644 --- a/opencollab/adapters/_env_docker.py +++ b/opencollab/adapters/_env_docker.py @@ -797,6 +797,8 @@ async def _write_file( f"docker file exceeds write limit of {ENV_FILE_WRITE_LIMIT_BYTES} bytes: {path}" ) async with self.file_write_lock(path, containment_root=containment_root) as target: + if containment_root == "/tmp": + self._temporary_files.add(path) return await self._write_file_atomic(target, payload, observe_change=observe_change) async def _resolve_write_target( @@ -1029,7 +1031,6 @@ async def write_temp_file( # Patch staging owns this path. It is outside the project on purpose; # workspace containment would reject it and drop the binary patch. await self._write_file(path, content, observe_change=False, containment_root="/tmp") - self._temporary_files.add(path) return path async def remove_file(self, path: str) -> None: diff --git a/tests/runtime/test_docker_owned_temp_cancellation.py b/tests/runtime/test_docker_owned_temp_cancellation.py new file mode 100644 index 00000000..b74b40b9 --- /dev/null +++ b/tests/runtime/test_docker_owned_temp_cancellation.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from tests.support.docker_native_edits_support import LocalDockerTransport, docker_environment + + +class _PublishedReceipt(LocalDockerTransport): + def __init__(self, workspace, *, delay): + super().__init__(workspace, pair_reads=False) + self.delay = delay + self.published = asyncio.Event() + + async def __call__(self, *argv, **kwargs): + result = await super().__call__(*argv, **kwargs) + if kwargs.get("input_bytes") is not None and "opencollab-write " in argv[-1]: + self.published.set() + if self.delay: + await asyncio.Event().wait() + return result + + +@pytest.mark.parametrize("cancel", [False, True], ids=["completed", "lost-receipt"]) +async def test_attached_cleanup_removes_published_temporary_after_lost_receipt(tmp_path, cancel): + transport = _PublishedReceipt(tmp_path, delay=cancel) + env = docker_environment(tmp_path, transport) + targets = [] + write_atomic = env._write_file_atomic + + async def observe_target(target, payload, **kwargs): + targets.append(Path(target)) + return await write_atomic(target, payload, **kwargs) + + env._write_file_atomic = observe_target + writer = asyncio.create_task(env.write_temp_file("retained candidate\n", prefix="oc-owned-temp-")) + try: + await asyncio.wait_for(transport.published.wait(), 5) + if cancel: + writer.cancel() + with pytest.raises(asyncio.CancelledError): + await writer + else: + await writer + assert len(targets) == 1 + assert targets[0].read_text() == "retained candidate\n" + assert {Path(path).resolve() for path in env._temporary_files} == {targets[0].resolve()} + await env.cleanup() + assert not targets[0].exists() + assert not env._temporary_files + finally: + if not writer.done(): + writer.cancel() + await asyncio.gather(writer, return_exceptions=True) + await env.cleanup() + for target in targets: + target.unlink(missing_ok=True) From 897da539e9bbd133b3ee7ac953d2a9391b64a2e6 Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 17:00:04 +0800 Subject: [PATCH 20/82] fix(workflow): retain loaded modules through owned cleanup --- .../bootstrap/_workflow_runtime_discovery.py | 47 +++- .../test_sdk_workflow_import_cleanup.py | 222 ++++++++++++++++++ .../test_workflow_discovery_module_names.py | 91 +++++++ 3 files changed, 359 insertions(+), 1 deletion(-) create mode 100644 tests/interfaces/test_sdk_workflow_import_cleanup.py diff --git a/opencollab/bootstrap/_workflow_runtime_discovery.py b/opencollab/bootstrap/_workflow_runtime_discovery.py index 2a4903bd..c53283f8 100644 --- a/opencollab/bootstrap/_workflow_runtime_discovery.py +++ b/opencollab/bootstrap/_workflow_runtime_discovery.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import importlib.abc import importlib.machinery import importlib.util @@ -14,6 +15,7 @@ from types import ModuleType from opencollab.adapters.safe_files import read_regular_bytes +from opencollab.application.workflow import WorkflowContext from opencollab.application.workflow_registry import Registry, WorkflowSpec from opencollab.bootstrap._workflow_runtime_state import ( MAX_WORKFLOW_DIRECTORY_ENTRIES, @@ -158,6 +160,49 @@ def release(self) -> None: self.capture_modules() _remove_workflow_package(self.package_name) + def release_after_cleanup(self, ctx: object) -> None: + if not isinstance(ctx, WorkflowContext): + self.release() + return + caller = asyncio.current_task() + + def pending_tasks(): + return {task for task in getattr(ctx, "pending_cleanup_tasks", ()) if task is not caller} + + loop = asyncio.get_running_loop() + watched = set() + scheduled = False + saw_empty = False + released = False + + def schedule(_task=None) -> None: + nonlocal scheduled + if not scheduled and not released: + scheduled = True + loop.call_soon(observe) + + def observe() -> None: + nonlocal scheduled, saw_empty, released + if released: + return + scheduled = False + pending = pending_tasks() + watched.intersection_update(pending) + if pending: + saw_empty = False + for task in pending - watched: + watched.add(task) + task.add_done_callback(schedule) + elif saw_empty: + released = True + self.release() + else: + # Finishing task callbacks may register another cleanup task. + saw_empty = True + schedule() + + observe() + def _bind_runtime_package(spec: WorkflowSpec, owner: _WorkflowModuleOwner) -> WorkflowSpec: original = spec.fn @@ -167,7 +212,7 @@ async def run(ctx, args): try: return await original(ctx, args) finally: - owner.release() + owner.release_after_cleanup(ctx) bound = WorkflowSpec( name=spec.name, diff --git a/tests/interfaces/test_sdk_workflow_import_cleanup.py b/tests/interfaces/test_sdk_workflow_import_cleanup.py new file mode 100644 index 00000000..d2cc7852 --- /dev/null +++ b/tests/interfaces/test_sdk_workflow_import_cleanup.py @@ -0,0 +1,222 @@ +"""Loaded workflows retain relative imports through owned tool cleanup.""" + +from __future__ import annotations + +import asyncio +import json +import subprocess +import sys + +import pytest + +from opencollab import OpenCollab, RunError +from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.bootstrap import _workflow_runtime_discovery as discovery + + +def _import_state(): + return ( + {name for name in sys.modules if name.startswith("_opencollab_workflow_")}, + dict(discovery._WORKFLOW_IMPORT_FINDER._package_roots), + ) + + +class ResourceModel: + def context_window(self): + return 200_000 + + async def complete(self, messages, **kwargs): + return LLMResponse(tool_calls=[{ + "id": "resource", "type": "function", "function": { + "name": "resource", "arguments": json.dumps({}), + }, + }], usage=Usage(input_tokens=5, output_tokens=5), finish_reason="tool_calls") + + +def _loaded_workflow(tmp_path): + directory = tmp_path / "workflows" + directory.mkdir() + (directory / "_cleanup.py").write_text( + "def close_resource(events):\n events['closed'].set()\n", + encoding="utf-8", + ) + (directory / "resource.py").write_text('''import asyncio +from opencollab import workflow + +class Resource: + name = "resource" + description = "Hold a resource until cancellation." + parameters = {"type": "object", "properties": {}} + + def __init__(self, events): + self.events = events + + def to_openai_schema(self): + return {"type": "function", "function": { + "name": self.name, "description": self.description, "parameters": self.parameters, + }} + + async def execute_with_runtime(self, args, runtime): + self.events["entered"].set() + try: + await asyncio.Event().wait() + finally: + self.events["cleanup_started"].set() + while not self.events["release"].is_set(): + try: + await self.events["release"].wait() + except asyncio.CancelledError: + self.events["cancelled_again"].set() + from ._cleanup import close_resource + close_resource(self.events) + +@workflow +async def resource_cleanup(ctx, args): + events = args["events"] + events["context"] = ctx + timeout = 0.05 if args["mode"] == "timeout" else None + operation = asyncio.create_task(ctx.agent("Hold the resource", tools=[Resource(events)], timeout=timeout)) + await events["entered"].wait() + if args["mode"] == "recover": + operation.cancel() + try: + await operation + except asyncio.CancelledError: + if args["mode"] != "recover": + raise + events["returned"].set() + return "recovered" +''', encoding="utf-8") + return discovery.discover_workflows(str(directory)).get("resource_cleanup") + + +def _events(): + return {name: asyncio.Event() for name in ( + "entered", "cleanup_started", "release", "closed", "returned", "cancelled_again", + )} + + +@pytest.mark.parametrize("mode", ["recover", "timeout", "caller", "repeated"]) +async def test_loaded_workflow_keeps_imports_until_cancelled_tool_closes(tmp_path, mode): + before = _import_state() + spec = _loaded_workflow(tmp_path) + events = _events() + running = asyncio.create_task(OpenCollab(tmp_path, model="offline").workflow( + spec, {"events": events, "mode": mode}, llm=ResourceModel(), trace=False, + cleanup_timeout=2, + )) + try: + await asyncio.wait_for(events["entered"].wait(), 5) + if mode in {"caller", "repeated"}: + running.cancel() + else: + await asyncio.wait_for(events["returned"].wait(), 5) + await asyncio.wait_for(events["cleanup_started"].wait(), 5) + if mode == "repeated": + running.cancel() + await asyncio.sleep(0) + running.cancel() + await asyncio.sleep(0) + assert _import_state() != before + assert not events["closed"].is_set() + events["release"].set() + if mode in {"caller", "repeated"}: + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(running, 5) + else: + result = await asyncio.wait_for(running, 5) + assert result.status == "completed", result.reason + assert result.output == "recovered" + assert events["closed"].is_set() + assert _import_state() == before + finally: + events["release"].set() + if not running.done(): + running.cancel() + await asyncio.gather(running, return_exceptions=True) + + +async def test_cleanup_deadline_keeps_imports_for_lingering_owned_work(tmp_path): + before = _import_state() + spec = _loaded_workflow(tmp_path) + events = _events() + running = asyncio.create_task(OpenCollab(tmp_path, model="offline").workflow( + spec, {"events": events, "mode": "recover"}, llm=ResourceModel(), trace=False, + cleanup_timeout=0.01, + )) + try: + await asyncio.wait_for(events["returned"].wait(), 5) + with pytest.raises(RunError, match="without finalized execution evidence"): + await asyncio.wait_for(asyncio.shield(running), 5) + assert _import_state() != before + assert not events["closed"].is_set() + events["release"].set() + await asyncio.wait_for(events["closed"].wait(), 5) + await asyncio.wait_for(events["context"].wait_for_pending_cleanup(), 5) + assert _import_state() == before + finally: + events["release"].set() + if not running.done(): + running.cancel() + await asyncio.gather(running, return_exceptions=True) + + +async def test_concurrent_loaded_workflows_retain_each_others_cleanup_imports(tmp_path): + before = _import_state() + spec = _loaded_workflow(tmp_path) + client = OpenCollab(tmp_path, model="offline") + events = [_events(), _events()] + calls = [asyncio.create_task(client.workflow( + spec, {"events": signals, "mode": "recover"}, llm=ResourceModel(), trace=False, + )) for signals in events] + try: + await asyncio.wait_for(asyncio.gather(*(signals["returned"].wait() for signals in events)), 5) + events[0]["release"].set() + assert (await asyncio.wait_for(calls[0], 5)).status == "completed" + assert events[0]["closed"].is_set() + assert not events[1]["closed"].is_set() + assert _import_state() != before + events[1]["release"].set() + assert (await asyncio.wait_for(calls[1], 5)).status == "completed" + assert events[1]["closed"].is_set() + assert _import_state() == before + finally: + for signals in events: + signals["release"].set() + await asyncio.gather(*calls, return_exceptions=True) + + +async def test_candidate_child_context_retains_loaded_workflow_imports(tmp_path): + before = _import_state() + spec = _loaded_workflow(tmp_path) + workspace = tmp_path / "repo" + workspace.mkdir() + for arguments in ( + ("init", "-q"), + ("-c", "user.name=OpenCollab Tests", "-c", "user.email=tests@example.invalid", + "-c", "commit.gpgsign=false", "commit", "--allow-empty", "-qm", "test: create workspace"), + ): + subprocess.run(["git", "-C", str(workspace), *arguments], check=True, capture_output=True) + events = _events() + + async def parent(ctx, args): + return await ctx.candidate_workflow(spec.fn, args, label="resource-worker") + + running = asyncio.create_task(OpenCollab(workspace, model="offline").workflow( + parent, {"events": events, "mode": "recover"}, llm=ResourceModel(), trace=False, + cleanup_timeout=5, + )) + try: + await asyncio.wait_for(events["returned"].wait(), 15) + assert _import_state() != before + events["release"].set() + result = await asyncio.wait_for(running, 15) + assert result.status == "completed", result.reason + assert result.output.output == "recovered" + assert events["closed"].is_set() + assert _import_state() == before + finally: + events["release"].set() + if not running.done(): + running.cancel() + await asyncio.gather(running, return_exceptions=True) diff --git a/tests/workflows/test_workflow_discovery_module_names.py b/tests/workflows/test_workflow_discovery_module_names.py index e0dc122b..ba81f156 100644 --- a/tests/workflows/test_workflow_discovery_module_names.py +++ b/tests/workflows/test_workflow_discovery_module_names.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import sys import pytest @@ -9,6 +10,7 @@ from opencollab import OpenCollab from opencollab.adapters.cli.main import app +from opencollab.application.workflow import WorkflowContext from opencollab.bootstrap import _workflow_runtime_discovery as workflow_discovery @@ -102,3 +104,92 @@ async def test_multiple_workflow_files_keep_sibling_and_self_imports(tmp_path): assert _import_state() == before assert await registry.get("task-entry").fn(None, {}) == "shared task" assert _import_state() == before + + +async def test_concurrent_calls_share_one_loaded_package_until_both_finish(tmp_path): + directory = tmp_path / "workflows" + directory.mkdir() + (directory / "_helper.py").write_text("def answer():\n return 42\n", encoding="utf-8") + (directory / "task.py").write_text( + "from opencollab import workflow\n" + "@workflow\n" + "async def overlap(ctx, args):\n" + " args['entered'].set()\n" + " await args['release'].wait()\n" + " from ._helper import answer\n" + " return answer()\n", + encoding="utf-8", + ) + before = _import_state() + spec = workflow_discovery.discover_workflows(str(directory)).get("overlap") + client = OpenCollab(tmp_path, model="offline") + entered = [asyncio.Event(), asyncio.Event()] + release = [asyncio.Event(), asyncio.Event()] + calls = [asyncio.create_task(client.workflow( + spec, {"entered": entered[index], "release": release[index]}, trace=False, + )) for index in range(2)] + try: + await asyncio.wait_for(asyncio.gather(*(event.wait() for event in entered)), 5) + release[0].set() + assert (await asyncio.wait_for(calls[0], 5)).output == 42 + assert _import_state() != before + release[1].set() + assert (await asyncio.wait_for(calls[1], 5)).output == 42 + assert _import_state() == before + finally: + for event in release: + event.set() + await asyncio.gather(*calls, return_exceptions=True) + + +@pytest.mark.parametrize("awaited_first", [False, True]) +async def test_cleanup_callbacks_keep_imports_for_following_owned_tasks(tmp_path, awaited_first): + directory = tmp_path / "workflows" + directory.mkdir() + (directory / "_helper.py").write_text("def answer():\n return 42\n", encoding="utf-8") + (directory / "task.py").write_text('''import asyncio +from opencollab import workflow + +@workflow +async def cleanup(ctx, args): + async def finish(): + args["following_started"].set() + await args["release_following"].wait() + from ._helper import answer + args["answers"].append(answer()) + + def follow(_task): + asyncio.get_running_loop().call_soon( + lambda: ctx._track_pending_cleanup(asyncio.create_task(finish())), + ) + + first = asyncio.create_task(asyncio.sleep(0)) + first.add_done_callback(follow) + ctx._track_pending_cleanup(first) + if args["awaited_first"]: + await first + return "returned" +''', encoding="utf-8") + before = _import_state() + spec = workflow_discovery.discover_workflows(str(directory)).get("cleanup") + ctx = WorkflowContext(factory=object()) + args = {name: asyncio.Event() for name in ( + "following_started", "release_following", + )} + args["awaited_first"] = awaited_first + args["answers"] = [] + caller = asyncio.current_task() + ctx._active_call_tasks.add(caller) + try: + assert await spec.fn(ctx, args) == "returned" + assert _import_state() != before + await asyncio.wait_for(args["following_started"].wait(), 5) + assert _import_state() != before + args["release_following"].set() + await asyncio.wait_for(ctx.wait_for_pending_cleanup(), 5) + assert args["answers"] == [42] + assert _import_state() == before + finally: + args["release_following"].set() + ctx._active_call_tasks.discard(caller) + await ctx.wait_for_pending_cleanup() From e092227d7ccf79948d43e861861f372f8f429497 Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 17:00:04 +0800 Subject: [PATCH 21/82] fix(files): preserve physical parent traversal before file access --- opencollab/adapters/_docker_snapshot.py | 12 +- opencollab/adapters/_env_local.py | 22 +-- opencollab/adapters/safe_anchored_files.py | 45 +++-- opencollab/adapters/safe_paths.py | 2 +- opencollab/adapters/safety.py | 14 +- .../interfaces/test_sdk_file_path_identity.py | 131 ++++++++++++++ .../test_authored_artifact_transfer.py | 57 ++++++ tests/runtime/test_local_env_path_identity.py | 166 ++++++++++++++++++ 8 files changed, 411 insertions(+), 38 deletions(-) create mode 100644 tests/interfaces/test_sdk_file_path_identity.py create mode 100644 tests/runtime/test_local_env_path_identity.py diff --git a/opencollab/adapters/_docker_snapshot.py b/opencollab/adapters/_docker_snapshot.py index aaa10778..fe04cd99 100644 --- a/opencollab/adapters/_docker_snapshot.py +++ b/opencollab/adapters/_docker_snapshot.py @@ -44,12 +44,6 @@ async def _copy_to_host(environment: Any, source: str, target: Path, *, director async def _copy_artifact_to_host(environment: Any, source: str, target: Path) -> None: - relative = ( - posixpath.relpath(source, environment.workspace) if posixpath.isabs(source) else posixpath.normpath(source) - ) - if relative == ".." or relative.startswith("../"): - raise PermissionError("artifact path escapes workspace") - source = posixpath.join(environment.workspace, relative) delegate = environment._delegate if isinstance(environment, _TeamCandidateEnvironment) else environment if isinstance(delegate, WorktreeEnvironment): delegate._ensure_active() @@ -65,6 +59,12 @@ def copy() -> None: await delegate._run_file_operation(copy) return + relative = ( + posixpath.relpath(source, environment.workspace) if posixpath.isabs(source) else posixpath.normpath(source) + ) + if relative == ".." or relative.startswith("../"): + raise PermissionError("artifact path escapes workspace") + source = posixpath.join(environment.workspace, source) if isinstance(delegate, DockerEnvironment): source = await delegate._contained_file_path(source) if isinstance(delegate, (DockerEnvironment, VMEnvironment)): diff --git a/opencollab/adapters/_env_local.py b/opencollab/adapters/_env_local.py index dc227f02..c5aa1cee 100644 --- a/opencollab/adapters/_env_local.py +++ b/opencollab/adapters/_env_local.py @@ -54,7 +54,7 @@ class LocalEnvironment(Environment): def __init__(self, workspace: str = ".") -> None: super().__init__() - self.workspace = os.path.realpath(os.path.abspath(workspace)) + self.workspace = os.path.realpath(workspace) if not os.path.isdir(self.workspace): raise NotADirectoryError(self.workspace) self._workspace_fd: int | None = open_directory_anchor(self.workspace) @@ -72,17 +72,12 @@ def _relative_path(self, path: str) -> str: candidate = canonicalize_system_path(path) workspace = canonicalize_system_path(self.workspace) try: - inside = os.path.commonpath((workspace, candidate)) == os.fspath(workspace) - except ValueError: - inside = False - if not inside: - raise PermissionError(f"path escapes local workspace: {path}") - normalized = os.path.relpath(candidate, workspace) + relative = os.fspath(candidate.relative_to(workspace)) + except ValueError as exc: + raise PermissionError(f"path escapes local workspace: {path}") from exc else: - normalized = os.path.normpath(path) - if normalized == ".." or normalized.startswith(f"..{os.sep}"): - raise PermissionError(f"relative path escapes local workspace: {path}") - return normalized + relative = path + return relative def _root_fd(self) -> int: if self._workspace_fd is None: @@ -229,7 +224,8 @@ async def write_temp_file( async def remove_file(self, path: str) -> None: self._ensure_active() target = self._relative_path(path) - if target not in self._temporary_files: + owned_target = os.path.normpath(target) + if owned_target not in self._temporary_files: raise OSError(f"refusing to remove unowned local temporary file: {path}") await self._run_file_operation( unlink_regular_file_durable_at, @@ -237,7 +233,7 @@ async def remove_file(self, path: str) -> None: self.workspace, target, ) - self._temporary_files.discard(target) + self._temporary_files.discard(owned_target) async def _cleanup_files(self) -> None: while self._file_operations: diff --git a/opencollab/adapters/safe_anchored_files.py b/opencollab/adapters/safe_anchored_files.py index aeff6f22..1fc951ea 100644 --- a/opencollab/adapters/safe_anchored_files.py +++ b/opencollab/adapters/safe_anchored_files.py @@ -71,12 +71,14 @@ def _relative_components( value = os.fspath(path) if not value or "\0" in value or os.path.isabs(value): raise ValueError("descriptor-relative path must be non-empty relative text") - normalized = os.path.normpath(value) - if normalized in {"", ".", ".."} or normalized.startswith(f"..{os.sep}"): - raise PermissionError(f"path escapes anchored directory: {value}") - parts = normalized.split(os.sep) - if any(part in {"", ".", ".."} for part in parts): + parts = [part for part in value.split(os.sep) if part not in {"", "."}] + if not parts or parts[-1] == "..": raise PermissionError(f"path escapes anchored directory: {value}") + depth = 0 + for part in parts[:-1]: + depth += -1 if part == ".." else 1 + if depth < 0: + raise PermissionError(f"path escapes anchored directory: {value}") return parts[:-1], parts[-1] def open_parent_beneath( @@ -89,9 +91,23 @@ def open_parent_beneath( """Open a target parent without resolving a component by pathname.""" _require_anchor_path(root_fd, root_path) parents, name = _relative_components(path) - current_fd = os.dup(root_fd) + # A directory traversed before .. must exist even when creating a file. + existing_only: set[int] = set() + parent_steps = 0 + for index in range(len(parents) - 1, -1, -1): + if parents[index] == "..": + parent_steps += 1 + elif parent_steps: + existing_only.add(index) + parent_steps -= 1 + opened = [os.dup(root_fd)] + retain_parents = ".." in parents try: - for component in parents: + for index, component in enumerate(parents): + if component == "..": + os.close(opened.pop()) + continue + current_fd = opened[-1] try: next_fd = os.open( component, @@ -99,7 +115,7 @@ def open_parent_beneath( dir_fd=current_fd, ) except FileNotFoundError: - if not create: + if not create or index in existing_only: raise try: os.mkdir(component, 0o700, dir_fd=current_fd) @@ -110,12 +126,13 @@ def open_parent_beneath( _directory_open_flags(), dir_fd=current_fd, ) - os.close(current_fd) - current_fd = next_fd - return current_fd, name - except BaseException: - os.close(current_fd) - raise + if not retain_parents: + os.close(opened.pop()) + opened.append(next_fd) + return opened.pop(), name + finally: + for fd in opened: + os.close(fd) @contextmanager def open_regular_read_at( diff --git a/opencollab/adapters/safe_paths.py b/opencollab/adapters/safe_paths.py index ac706cda..43d8d3e7 100644 --- a/opencollab/adapters/safe_paths.py +++ b/opencollab/adapters/safe_paths.py @@ -28,4 +28,4 @@ def canonicalize_system_path(path: str | os.PathLike[str]) -> Path: value = os.fspath(path) if not value or "\0" in value: raise ValueError("path must be non-empty text without NUL bytes") - return _canonicalize_system_alias(Path(os.path.abspath(value))) + return _canonicalize_system_alias(Path(value).absolute()) diff --git a/opencollab/adapters/safety.py b/opencollab/adapters/safety.py index 5691346b..17dbdafd 100644 --- a/opencollab/adapters/safety.py +++ b/opencollab/adapters/safety.py @@ -16,6 +16,7 @@ import os import re import shlex +import stat from typing import Awaitable, Callable # Commands that warrant human confirmation (not outright blocked) @@ -98,9 +99,9 @@ class SandboxInterceptor: """ def __init__(self, workspace_root: str, allowed_dirs: list[str] | None = None): - self.root = os.path.abspath(workspace_root) + self.root = os.path.realpath(workspace_root) # Additional allowed directories (e.g., /tmp for scratch) - self.allowed_roots = [self.root] + [os.path.abspath(d) for d in (allowed_dirs or [])] + self.allowed_roots = [self.root] + [os.path.realpath(d) for d in (allowed_dirs or [])] def check_path(self, target_path: str) -> str: """Resolve and validate a file path. Returns absolute safe path or raises. @@ -109,9 +110,14 @@ def check_path(self, target_path: str) -> str: """ # Resolve relative to workspace if not os.path.isabs(target_path): - resolved = os.path.abspath(os.path.join(self.root, target_path)) + resolved = os.path.join(self.root, target_path) else: - resolved = os.path.abspath(target_path) + resolved = target_path + prefix = os.sep + for component in resolved.split(os.sep): + if component == ".." and not stat.S_ISDIR(os.stat(prefix).st_mode): + raise NotADirectoryError(prefix) + prefix = os.path.join(prefix, component) resolved = os.path.realpath(resolved) # Check against all allowed roots diff --git a/tests/interfaces/test_sdk_file_path_identity.py b/tests/interfaces/test_sdk_file_path_identity.py new file mode 100644 index 00000000..3f459dd2 --- /dev/null +++ b/tests/interfaces/test_sdk_file_path_identity.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import asyncio +import json +import os + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.adapters.safety import SandboxInterceptor +from opencollab.adapters.tools.base import host_write_lock +from opencollab.adapters.tools.fs import FileWriteTool +from opencollab.application.tool_execution import ToolRuntime +from opencollab.environments import local_environment +from opencollab.tools import builtin_tools + + +class _PathModel: + def __init__(self, tool, arguments): + self.tool = tool + self.arguments = arguments + self.called = False + + async def complete(self, messages, **kwargs): + if self.called: + return LLMResponse(content="finished", usage=Usage(10, 5), finish_reason="stop") + self.called = True + return LLMResponse(tool_calls=[{ + "id": "selected-file", "type": "function", "function": { + "name": self.tool, "arguments": json.dumps(self.arguments), + }, + }], usage=Usage(10, 5), finish_reason="tool_calls") + + +@pytest.mark.parametrize("absolute", [False, True]) +@pytest.mark.parametrize("tool", ["file_read", "file_write", "apply_patch"]) +async def test_sdk_file_tools_match_shell_through_symlink_parent(tmp_path, absolute, tool): + (tmp_path / "actual/child").mkdir(parents=True) + (tmp_path / "alias").symlink_to("actual/child", target_is_directory=True) + target = tmp_path / "actual/module.py" + unrelated = tmp_path / "module.py" + target.write_text("VALUE = 1\n") + unrelated.write_text("UNRELATED = 7\n" if tool == "file_read" else "VALUE = 1\n") + raw = "alias/../module.py" + path = os.path.join(str(tmp_path), raw) if absolute else raw + arguments = {"path": path} + if tool == "file_write": + arguments.update(mode="str_replace", old_str="VALUE = 1", new_str="VALUE = 42") + elif tool == "apply_patch": + arguments.update( + mode="line_replace", start_line=1, end_line=1, + expected_str="VALUE = 1", new_str="VALUE = 42", + ) + model = _PathModel(tool, arguments) + environment = local_environment(tmp_path) + outputs = [] + + async def flow(ctx, _): + result = await ctx.agent_run( + "Use the requested file tool", tools=list(builtin_tools(tool, headless=False)), + isolation="shared", budget=2000, system_prompt="Use the requested file tool", + ) + outputs.extend( + message["content"] for session in ctx.sessions for message in session.state.messages + if message.get("role") == "tool" + ) + return result.status + + try: + result = await OpenCollab(tmp_path, environment=environment, model="local-scripted").workflow( + flow, llm=model, trace=False, timeout=15, budget=5000, concurrency=1, + limit_mode="explicit", agent_profile="single2", + ) + assert result.ok, result.reason + assert result.output == "completed" + assert len(outputs) == 1 and not outputs[0].startswith("Error") + if tool == "file_read": + assert "1\tVALUE = 1" in outputs[0] + assert "UNRELATED" not in outputs[0] + else: + assert target.read_text() == "VALUE = 42\n" + assert unrelated.read_text() == ("UNRELATED = 7\n" if tool == "file_read" else "VALUE = 1\n") + shell = await environment.exec_cmd("cat alias/../module.py") + assert shell.stdout == target.read_text() + finally: + await environment.cleanup() + + +async def test_native_edit_uses_the_selected_physical_file_lock(tmp_path): + (tmp_path / "actual/child").mkdir(parents=True) + (tmp_path / "alias").symlink_to("actual/child", target_is_directory=True) + target = tmp_path / "actual/module.py" + target.write_text("VALUE = 1\n") + unrelated = tmp_path / "module.py" + unrelated.write_text("VALUE = 1\n") + environment = local_environment(tmp_path) + runtime = ToolRuntime(environment, SandboxInterceptor(str(tmp_path)), None) + editor = None + try: + async with host_write_lock(str(target), environment): + editor = asyncio.create_task(FileWriteTool().execute_with_runtime({ + "path": "alias/../module.py", "mode": "str_replace", + "old_str": "VALUE = 1", "new_str": "VALUE = 42", + }, runtime)) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(asyncio.shield(editor), 0.05) + assert target.read_text() == "VALUE = 1\n" + assert unrelated.read_text() == "VALUE = 1\n" + assert not (await asyncio.wait_for(editor, 2)).startswith("Error") + assert target.read_text() == "VALUE = 42\n" + assert unrelated.read_text() == "VALUE = 1\n" + finally: + if editor is not None and not editor.done(): + editor.cancel() + await asyncio.gather(editor, return_exceptions=True) + await environment.cleanup() + + +@pytest.mark.parametrize("component", ["missing", "regular-file"]) +def test_workspace_policy_checks_directory_before_parent_traversal(tmp_path, component): + target = tmp_path / "value.txt" + target.write_text("original\n") + if component == "regular-file": + (tmp_path / "component").write_text("regular\n") + policy = SandboxInterceptor(str(tmp_path)) + with pytest.raises(OSError): + policy.check_path("component/../value.txt") + assert target.read_text() == "original\n" + (tmp_path / "sub").mkdir() + assert policy.check_path("sub/../new/value.txt") == str(tmp_path / "new/value.txt") diff --git a/tests/runtime/test_authored_artifact_transfer.py b/tests/runtime/test_authored_artifact_transfer.py index 35846d06..b9b74fa4 100644 --- a/tests/runtime/test_authored_artifact_transfer.py +++ b/tests/runtime/test_authored_artifact_transfer.py @@ -19,6 +19,63 @@ from opencollab.application.tool_execution import ToolRuntime from opencollab.bootstrap.authored_workflow import AuthoredWorkflowTool, _Child from opencollab.bootstrap.authored_workflow_bridge import AuthoredWorkflowBridge, BridgeClient, BridgeEnvironment +from tests.support.docker_native_edits_support import LocalDockerTransport, docker_environment + + +@pytest.mark.parametrize("absolute", [False, True]) +@pytest.mark.parametrize("component", ["directory", "symlink", "missing"]) +async def test_artifact_transfer_keeps_parent_traversal_identity(tmp_path, absolute, component): + workspace = tmp_path / "workspace" + (workspace / "actual/child").mkdir(parents=True) + (workspace / "result.json").write_text('{"selected": 42}') + (workspace / "actual/result.json").write_text('{"other": 7}') + if component == "directory": + (workspace / "parent").mkdir() + elif component == "symlink": + (workspace / "parent").symlink_to("actual/child", target_is_directory=True) + relative = "parent/../result.json" + source = os.path.join(str(workspace), relative) if absolute else relative + target = tmp_path / "copied.json" + environment = LocalEnvironment(str(workspace)) + try: + if component == "directory": + await _copy_artifact_to_host(environment, source, target) + assert target.read_text() == '{"selected": 42}' + else: + with pytest.raises(OSError): + await _copy_artifact_to_host(environment, source, target) + assert not target.exists() + assert (workspace / "result.json").read_text() == '{"selected": 42}' + assert (workspace / "actual/result.json").read_text() == '{"other": 7}' + finally: + await environment.cleanup() + + +@pytest.mark.parametrize("absolute", [False, True]) +async def test_docker_artifact_transfer_resolves_parent_in_execution_environment(tmp_path, absolute): + (tmp_path / "actual/child").mkdir(parents=True) + (tmp_path / "alias").symlink_to("actual/child", target_is_directory=True) + (tmp_path / "actual/result.json").write_text('{"selected": 42}') + (tmp_path / "result.json").write_text('{"other": 7}') + + class ArtifactTransport(LocalDockerTransport): + async def __call__(self, *argv, **kwargs): + if argv[0] == "cp": + shutil.copyfile(argv[2].split(":", 1)[1], argv[3]) + return ProcessResult(0, b"", b"") + return await super().__call__(*argv, **kwargs) + + transport = ArtifactTransport(tmp_path, pair_reads=False) + environment = docker_environment(tmp_path, transport) + relative = "alias/../result.json" + source = os.path.join(str(tmp_path), relative) if absolute else relative + target = tmp_path / "copied.json" + try: + await _copy_artifact_to_host(environment, source, target) + assert target.read_text() == '{"selected": 42}' + assert (tmp_path / "result.json").read_text() == '{"other": 7}' + finally: + await environment.cleanup() @pytest.mark.parametrize("kind", ["local", "worktree"]) diff --git a/tests/runtime/test_local_env_path_identity.py b/tests/runtime/test_local_env_path_identity.py new file mode 100644 index 00000000..6f315c54 --- /dev/null +++ b/tests/runtime/test_local_env_path_identity.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +import asyncio +import os +import subprocess +import sys +import tempfile +import threading +from pathlib import Path + +import pytest + +from opencollab.adapters import safe_anchored_files +from opencollab.environments import local_environment + + +async def _operate(environment, operation, path): + if operation == "read": + return await environment.read_file(path) + if operation == "range": + result = await environment.read_text_range(path, offset=1, limit=1, max_chars=100) + return "\n".join(result.lines) + "\n" + await environment.write_file(path, "changed\n") + + +@pytest.mark.parametrize("absolute", [False, True]) +@pytest.mark.parametrize("operation", ["read", "range", "write"]) +@pytest.mark.parametrize("component", ["directory", "symlink", "missing", "regular-file"]) +async def test_local_parent_traversal_checks_each_original_component( + tmp_path, absolute, operation, component, +): + (tmp_path / "actual/child").mkdir(parents=True) + target = tmp_path / "value.txt" + target.write_text("original\n") + other = tmp_path / "actual/value.txt" + other.write_text("other\n") + parent = tmp_path / "component" + if component == "directory": + parent.mkdir() + elif component == "symlink": + parent.symlink_to("actual/child", target_is_directory=True) + elif component == "regular-file": + parent.write_text("regular\n") + relative = "component/../value.txt" + path = os.path.join(str(tmp_path), relative) if absolute else relative + environment = local_environment(tmp_path) + try: + if component == "directory": + result = await _operate(environment, operation, path) + if operation != "write": + assert result == "original\n" + assert target.read_text() == ("changed\n" if operation == "write" else "original\n") + else: + with pytest.raises(OSError): + await _operate(environment, operation, path) + assert target.read_text() == "original\n" + assert other.read_text() == "other\n" + if component == "missing": + assert not parent.exists() + finally: + await environment.cleanup() + + +@pytest.mark.parametrize("operation", ["read", "range", "write"]) +async def test_local_final_symlink_remains_rejected(tmp_path, operation): + target = tmp_path / "value.txt" + target.write_text("original\n") + (tmp_path / "link").symlink_to(target.name) + environment = local_environment(tmp_path) + try: + with pytest.raises(OSError): + await _operate(environment, operation, "link") + assert target.read_text() == "original\n" + assert (tmp_path / "link").is_symlink() + finally: + await environment.cleanup() + + +async def test_local_parent_traversal_keeps_new_parent_creation_and_owned_removal(tmp_path): + (tmp_path / "sub").mkdir() + environment = local_environment(tmp_path) + try: + await environment.write_file("sub/../new/value.txt", "created\n") + assert (tmp_path / "new/value.txt").read_text() == "created\n" + temporary = await environment.write_temp_file("temporary", prefix="owned-") + await environment.remove_file("sub/../" + Path(temporary).name) + assert not Path(temporary).exists() + finally: + await environment.cleanup() + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS system-directory aliases") +@pytest.mark.parametrize("alias_root", ["/tmp", "/var/tmp"]) +async def test_local_absolute_system_alias_preserves_parent_checks(alias_root): + with tempfile.TemporaryDirectory(dir=os.path.realpath(alias_root)) as directory: + root = Path(directory) + (root / "sub").mkdir() + (root / "actual/child").mkdir(parents=True) + (root / "alias").symlink_to("actual/child", target_is_directory=True) + target = root / "value.txt" + target.write_text("original\n") + public_root = os.path.join(alias_root, root.name) + environment = local_environment(root) + try: + await environment.write_file(os.path.join(public_root, "sub/../value.txt"), "changed\n") + with pytest.raises(OSError): + await environment.write_file(os.path.join(public_root, "alias/../value.txt"), "wrong\n") + assert target.read_text() == "changed\n" + assert not (root / "actual/value.txt").exists() + finally: + await environment.cleanup() + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX descriptor limits") +def test_deep_ordinary_local_paths_use_bounded_descriptors(tmp_path): + relative = "/".join(["sub"] * 64 + ["value.txt"]) + target = tmp_path / relative + target.parent.mkdir(parents=True) + target.write_text("original\n") + code = ( + "import asyncio, resource, sys\n" + "from opencollab.environments import local_environment\n" + "resource.setrlimit(resource.RLIMIT_NOFILE, (32, resource.getrlimit(resource.RLIMIT_NOFILE)[1]))\n" + "async def run():\n" + " environment = local_environment(sys.argv[1])\n" + " try:\n" + " assert await environment.read_file(sys.argv[2]) == 'original\\n'\n" + " await environment.write_file(sys.argv[2], 'changed\\n')\n" + " finally:\n" + " await environment.cleanup()\n" + "asyncio.run(run())\n" + ) + result = subprocess.run( + [sys.executable, "-c", code, str(tmp_path), relative], capture_output=True, text=True, timeout=10, + ) + assert result.returncode == 0, result.stderr + assert target.read_text() == "changed\n" + + +async def test_parent_step_keeps_the_opened_ancestor_when_its_path_is_replaced(tmp_path, monkeypatch): + (tmp_path / "parent/child").mkdir(parents=True) + (tmp_path / "parent/value.txt").write_text("original\n") + environment = local_environment(tmp_path) + started, release = threading.Event(), threading.Event() + original_open = safe_anchored_files.os.open + + def blocked_open(path, flags, mode=0o777, *, dir_fd=None): + if os.fspath(path) == "value.txt": + started.set() + assert release.wait(2) + return original_open(path, flags, mode, dir_fd=dir_fd) + + monkeypatch.setattr(safe_anchored_files.os, "open", blocked_open) + reading = asyncio.create_task(environment.read_file("parent/child/../value.txt")) + try: + assert await asyncio.to_thread(started.wait, 2) + (tmp_path / "parent").rename(tmp_path / "saved-parent") + (tmp_path / "parent").mkdir() + (tmp_path / "parent/value.txt").write_text("replacement\n") + release.set() + assert await reading == "original\n" + assert (tmp_path / "parent/value.txt").read_text() == "replacement\n" + finally: + release.set() + await asyncio.gather(reading, return_exceptions=True) + await environment.cleanup() From a20799112b1ee111c537b8a822c076d28efda314 Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 17:09:00 +0800 Subject: [PATCH 22/82] fix(files): bound descriptors during parent traversal --- opencollab/adapters/safe_anchored_files.py | 5 ++-- .../interfaces/test_sdk_file_path_identity.py | 7 ++++++ tests/runtime/test_local_env_path_identity.py | 25 +++++++++++++++++-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/opencollab/adapters/safe_anchored_files.py b/opencollab/adapters/safe_anchored_files.py index 1fc951ea..7c674dfe 100644 --- a/opencollab/adapters/safe_anchored_files.py +++ b/opencollab/adapters/safe_anchored_files.py @@ -91,7 +91,7 @@ def open_parent_beneath( """Open a target parent without resolving a component by pathname.""" _require_anchor_path(root_fd, root_path) parents, name = _relative_components(path) - # A directory traversed before .. must exist even when creating a file. + # A directory followed by .. must exist and keep its parent descriptor open. existing_only: set[int] = set() parent_steps = 0 for index in range(len(parents) - 1, -1, -1): @@ -101,7 +101,6 @@ def open_parent_beneath( existing_only.add(index) parent_steps -= 1 opened = [os.dup(root_fd)] - retain_parents = ".." in parents try: for index, component in enumerate(parents): if component == "..": @@ -126,7 +125,7 @@ def open_parent_beneath( _directory_open_flags(), dir_fd=current_fd, ) - if not retain_parents: + if index not in existing_only: os.close(opened.pop()) opened.append(next_fd) return opened.pop(), name diff --git a/tests/interfaces/test_sdk_file_path_identity.py b/tests/interfaces/test_sdk_file_path_identity.py index 3f459dd2..06caec94 100644 --- a/tests/interfaces/test_sdk_file_path_identity.py +++ b/tests/interfaces/test_sdk_file_path_identity.py @@ -129,3 +129,10 @@ def test_workspace_policy_checks_directory_before_parent_traversal(tmp_path, com assert target.read_text() == "original\n" (tmp_path / "sub").mkdir() assert policy.check_path("sub/../new/value.txt") == str(tmp_path / "new/value.txt") + + +def test_workspace_policy_root_resolves_symlink_before_parent(tmp_path): + (tmp_path / "actual/child").mkdir(parents=True) + (tmp_path / "alias").symlink_to("actual/child", target_is_directory=True) + policy = SandboxInterceptor(os.path.join(str(tmp_path), "alias/..")) + assert policy.check_path("module.py") == str(tmp_path / "actual/module.py") diff --git a/tests/runtime/test_local_env_path_identity.py b/tests/runtime/test_local_env_path_identity.py index 6f315c54..6b4ca5c4 100644 --- a/tests/runtime/test_local_env_path_identity.py +++ b/tests/runtime/test_local_env_path_identity.py @@ -89,6 +89,18 @@ async def test_local_parent_traversal_keeps_new_parent_creation_and_owned_remova await environment.cleanup() +async def test_local_workspace_selection_resolves_symlink_before_parent(tmp_path): + (tmp_path / "actual/child").mkdir(parents=True) + (tmp_path / "alias").symlink_to("actual/child", target_is_directory=True) + (tmp_path / "actual/value.txt").write_text("selected\n") + (tmp_path / "value.txt").write_text("unrelated\n") + environment = local_environment(os.path.join(str(tmp_path), "alias/..")) + try: + assert await environment.read_file("value.txt") == "selected\n" + finally: + await environment.cleanup() + + @pytest.mark.skipif(sys.platform != "darwin", reason="macOS system-directory aliases") @pytest.mark.parametrize("alias_root", ["/tmp", "/var/tmp"]) async def test_local_absolute_system_alias_preserves_parent_checks(alias_root): @@ -112,15 +124,23 @@ async def test_local_absolute_system_alias_preserves_parent_checks(alias_root): @pytest.mark.skipif(os.name != "posix", reason="POSIX descriptor limits") -def test_deep_ordinary_local_paths_use_bounded_descriptors(tmp_path): +@pytest.mark.parametrize("traversal", ["plain", "early-parent", "late-parent"]) +def test_deep_ordinary_local_paths_use_bounded_descriptors(tmp_path, traversal): relative = "/".join(["sub"] * 64 + ["value.txt"]) target = tmp_path / relative target.parent.mkdir(parents=True) target.write_text("original\n") + if traversal == "early-parent": + (tmp_path / "cancel").mkdir() + relative = "cancel/../" + relative + elif traversal == "late-parent": + (target.parent / "cancel").mkdir() + relative = "/".join(["sub"] * 64 + ["cancel", "..", "value.txt"]) code = ( - "import asyncio, resource, sys\n" + "import asyncio, os, resource, sys\n" "from opencollab.environments import local_environment\n" "resource.setrlimit(resource.RLIMIT_NOFILE, (32, resource.getrlimit(resource.RLIMIT_NOFILE)[1]))\n" + "before = len(os.listdir('/dev/fd'))\n" "async def run():\n" " environment = local_environment(sys.argv[1])\n" " try:\n" @@ -129,6 +149,7 @@ def test_deep_ordinary_local_paths_use_bounded_descriptors(tmp_path): " finally:\n" " await environment.cleanup()\n" "asyncio.run(run())\n" + "assert len(os.listdir('/dev/fd')) == before\n" ) result = subprocess.run( [sys.executable, "-c", code, str(tmp_path), relative], capture_output=True, text=True, timeout=10, From 458eccca3e599668f6ebc3e6503d62b1ee363239 Mon Sep 17 00:00:00 2001 From: Kai Date: Tue, 17 Nov 2026 22:23:29 +0800 Subject: [PATCH 23/82] test(workflow): avoid parent cleanup waits across Python versions --- tests/workflows/test_workflow_discovery_module_names.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/workflows/test_workflow_discovery_module_names.py b/tests/workflows/test_workflow_discovery_module_names.py index ba81f156..a91f7c27 100644 --- a/tests/workflows/test_workflow_discovery_module_names.py +++ b/tests/workflows/test_workflow_discovery_module_names.py @@ -186,6 +186,15 @@ def follow(_task): await asyncio.wait_for(args["following_started"].wait(), 5) assert _import_state() != before args["release_following"].set() + + async def wait_for_package_release(): + while _import_state() != before: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_package_release(), 5) + assert args["answers"] == [42] + assert _import_state() == before + ctx._active_call_tasks.discard(caller) await asyncio.wait_for(ctx.wait_for_pending_cleanup(), 5) assert args["answers"] == [42] assert _import_state() == before From c98df6cade8398118e400600c372d41497710ca2 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 11:30:16 +0800 Subject: [PATCH 24/82] fix(session): keep Anthropic snapshots compact and restorable --- opencollab/application/session.py | 7 ++- opencollab/application/session_snapshot.py | 50 +++++++++++++++++-- .../test_anthropic_preserved_history.py | 43 +++++++++++++--- 3 files changed, 90 insertions(+), 10 deletions(-) diff --git a/opencollab/application/session.py b/opencollab/application/session.py index 74e2a51c..7fea29c4 100644 --- a/opencollab/application/session.py +++ b/opencollab/application/session.py @@ -25,11 +25,13 @@ ) from opencollab.application.session_run import SessionRunUseCase from opencollab.application.session_snapshot import ( + _anthropic_source_edit_start, _restore_active_turn_start, _restore_anthropic_history, _restore_pending_row, _restore_phase, _restore_queued_external_user_turn, + _serialize_anthropic_history, _serialize_pending_row, _snapshot_int, _snapshot_nonnegative_float, @@ -1004,7 +1006,7 @@ def _snapshot_for_save( "completed_turn_start_message_index": state.completed_turn_start_message_index, "pending_step_latency": state.pending_step_latency, "submitted_summary": state.submitted_summary, - "anthropic_history": state.anthropic_history, + "anthropic_history": _serialize_anthropic_history(state), }, } if self.run_id is not None: @@ -1058,6 +1060,9 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: rewrite_revision = self._auto_save_rewrite_revision if self._auto_save_rewrite_from is not None: replace_from = min(replace_from, self._auto_save_rewrite_from) + edited_from = _anthropic_source_edit_start(self.state) + if edited_from is not None: + replace_from = min(replace_from, edited_from) messages, meta = self._snapshot_for_save(replace_from) current_seen_hashes = set(self.state.turn.seen_result_hashes) seen_hashes_reset = preceding_write_pending or not self._auto_save_seen_result_hashes.issubset( diff --git a/opencollab/application/session_snapshot.py b/opencollab/application/session_snapshot.py index c031540f..e1d184e4 100644 --- a/opencollab/application/session_snapshot.py +++ b/opencollab/application/session_snapshot.py @@ -6,8 +6,9 @@ import math from typing import Any +from opencollab.application._anthropic_history import _edited_anthropic_history from opencollab.domain.pending import PendingRow, RowKind, RowStatus -from opencollab.domain.session import SessionPhase +from opencollab.domain.session import SessionPhase, SessionState # Translate legacy terminals and the removed enqueue phase so old snapshots # retain their disposition instead of falling back to IDLE. @@ -15,6 +16,41 @@ {"cancelled", "budget_exceeded", "step_limit_exceeded", "context_overflow"} ) +def _anthropic_source_edit_start(state: SessionState) -> int | None: + history = state.anthropic_history + source = history.get("source_messages") if history is not None else None + if source is None or state.messages[:len(source)] == source: + return None + for index, (old, current) in enumerate(zip(source, state.messages)): + if old != current: + return index + return len(state.messages) + +def _serialize_anthropic_history(state: SessionState) -> dict | None: + history = state.anthropic_history + if history is None: + return None + source = state.messages + count = history["source_count"] + view = history["messages"] + if _anthropic_source_edit_start(state) is not None: + # Retire the stale projection before restore binds it to the new transcript. + view = _edited_anthropic_history(history["source_messages"], source) + view = _edited_anthropic_history(history["messages"], view) + count = len(source) + + # Aligned and suffix-aligned entries can reuse the persisted transcript. + stored: list[dict | int] = [] + suffix_offset = count - len(view) + for index, message in enumerate(view): + for source_index in (index, index + suffix_offset): + if 0 <= source_index < count and message == source[source_index]: + stored.append(source_index) + break + else: + stored.append(message) + return {"messages": stored, "source_count": count, "tools": history["tools"]} + def _restore_anthropic_history(value: object, messages: list[dict]) -> dict | None: if not isinstance(value, dict): return None @@ -24,7 +60,7 @@ def _restore_anthropic_history(value: object, messages: list[dict]) -> dict | No source = value.get("source_messages") if ( isinstance(count, bool) or not isinstance(count, int) or count < 0 - or not isinstance(view, list) or not all(isinstance(message, dict) for message in view) + or not isinstance(view, list) or (tools is not None and (not isinstance(tools, list) or not all(isinstance(tool, dict) for tool in tools))) ): return None @@ -37,7 +73,15 @@ def _restore_anthropic_history(value: object, messages: list[dict]) -> dict | No or not all(isinstance(message, dict) for message in source) ): return None - return copy.deepcopy({"messages": view, "source_count": count, "tools": tools, "source_messages": source}) + restored_view = [] + for message in view: + if isinstance(message, dict): + restored_view.append(message) + elif type(message) is int and 0 <= message < len(source): + restored_view.append(source[message]) + else: + return None + return copy.deepcopy({"messages": restored_view, "source_count": count, "tools": tools, "source_messages": source}) def _restore_phase(raw: object) -> SessionPhase: value = str(raw) diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index 0c55986e..5361673b 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -161,7 +161,10 @@ async def test_native_steering_prefix_survives_tool_rounds_and_new_user_turns(mo assert _signatures(peer.requests[2]) == ["signed-1", "redacted-1", "signed-2", "redacted-2"] @pytest.mark.parametrize("mode", ["clear", "summary", "first-trim"]) -async def test_compacted_native_view_survives_restore_and_disabling_the_shaper(monkeypatch, tmp_path, mode): +@pytest.mark.parametrize("snapshot_format", ["current", "legacy", "source-copy"]) +async def test_compacted_native_view_survives_restore_and_disabling_the_shaper( + monkeypatch, tmp_path, mode, snapshot_format, +): monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") peer = _BoundThinkingProvider(reads=4) install_sdk_transport(monkeypatch, "anthropic", peer) @@ -182,10 +185,21 @@ async def test_compacted_native_view_survives_restore_and_disabling_the_shaper(m assert len([message for message in raw if message["role"] == "tool"]) == 4 assert all("file evidence" in message["content"] for message in raw if message["role"] == "tool") before = copy.deepcopy(session.state.anthropic_history) - session.save(str(tmp_path / "session.json")) + path = tmp_path / "session.json" + session.save(str(path)) + saved = json.loads(path.read_text()) + native_view = saved["session_state"]["anthropic_history"] + assert "source_messages" not in native_view + assert native_view["messages"][0] == 0 + if snapshot_format != "current": + legacy = dict(before) + if snapshot_format == "legacy": + del legacy["source_messages"] + saved["session_state"]["anthropic_history"] = legacy + path.write_text(json.dumps(saved)) await session.aclose() await session.env.cleanup() - session = load_session(str(tmp_path / "session.json"), agent=_agent(), llm=client, + session = load_session(str(path), agent=_agent(), llm=client, env=LocalEnvironment(str(tmp_path)), max_budget_tokens=100_000, max_steps=20) assert session.state.anthropic_history == before assert session.messages == raw @@ -338,7 +352,7 @@ async def test_prefix_constraint_uses_native_transport_and_documented_models(mod assert client.requires_thinking_prefix is expected -@pytest.mark.parametrize("moment", ["live", "before-save", "after-load"]) +@pytest.mark.parametrize("moment", ["live", "before-save", "before-autosave", "after-load"]) @pytest.mark.parametrize("edit", ["text", "nested", "replacement", "deletion", "tail-deletion", "system"]) async def test_source_edits_reach_native_requests_and_retire_only_invalid_thinking( monkeypatch, tmp_path, moment, edit, @@ -366,21 +380,31 @@ def apply_edit(session): async with _client() as client: session = build_session(agent=_agent(), llm=client, env=LocalEnvironment(str(tmp_path)), - max_budget_tokens=100_000, max_steps=20) + max_budget_tokens=100_000, max_steps=20, + auto_save_path=str(path) if moment == "before-autosave" else None) try: await session.add_user_message("Answer the first question") assert await session.run_loop() == "done" await session.add_user_message("Choose red") session.messages[-1]["content"] = [{"type": "text", "text": "Choose red"}] assert await session.run_loop() == "done" + if moment == "before-autosave": + session.save(str(path)) if moment != "after-load": apply_edit(session) if moment != "live": - session.save(str(path)) + if moment == "before-autosave": + edited = copy.deepcopy(session.messages) + await session.enqueue_auto_save() + assert not session.persistence_errors + else: + session.save(str(path)) await session.aclose() await session.env.cleanup() session = load_session(str(path), agent=_agent(), llm=client, env=LocalEnvironment(str(tmp_path)), max_budget_tokens=100_000, max_steps=20) + if moment == "before-autosave": + assert session.messages == edited if moment == "after-load": apply_edit(session) edited = copy.deepcopy(session.messages) @@ -436,6 +460,13 @@ def test_restore_rejects_malformed_optional_source_prefix(source): }, [message]) is None +@pytest.mark.parametrize("reference", [True, -1, 1, "0"]) +def test_restore_rejects_invalid_native_message_references(reference): + assert _restore_anthropic_history({ + "messages": [reference], "source_count": 1, "tools": None, + }, [{"role": "user", "content": "Request"}]) is None + + @pytest.mark.parametrize("count", [0, 2]) def test_legacy_native_history_restores_without_an_optional_source_prefix(count): messages = [{"role": "user", "content": "First"}, {"role": "assistant", "content": "Answer"}] From aee56ba916067a9db497e72eef93edabd93627f9 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 11:30:16 +0800 Subject: [PATCH 25/82] fix(docker): retain edit ownership across directory changes --- opencollab/adapters/_env_docker.py | 71 ++++++--- opencollab/adapters/safety.py | 17 ++- opencollab/adapters/single2_safety.py | 4 + opencollab/adapters/tools/_paths.py | 23 +-- opencollab/adapters/tools/apply_patch.py | 2 +- opencollab/adapters/tools/fs.py | 4 +- opencollab/bootstrap/container.py | 2 + opencollab/bootstrap/runtime_context.py | 4 +- .../interfaces/test_sdk_file_path_identity.py | 26 +++- tests/support/docker_native_edits_support.py | 12 ++ tests/tools/test_docker_edit_path_identity.py | 15 ++ .../test_candidate_docker_alias_mutations.py | 140 ++++++++++++++---- 12 files changed, 244 insertions(+), 76 deletions(-) diff --git a/opencollab/adapters/_env_docker.py b/opencollab/adapters/_env_docker.py index 88d7c187..468d7dc0 100644 --- a/opencollab/adapters/_env_docker.py +++ b/opencollab/adapters/_env_docker.py @@ -698,13 +698,31 @@ async def file_write_locks(self, paths: Sequence[str]) -> AsyncIterator[None]: await self._bind_attached() if self._container_id is None: raise RuntimeError("Container not started. Call setup() first.") - targets = { - await self._resolve_write_target(self._validate_container_path(path)) - for path in paths - } + ordered = await self._grouped_write_targets(paths) + while True: + async with contextlib.AsyncExitStack() as locks: + for target in ordered: + await locks.enter_async_context(self._resolved_file_write_lock(target)) + # An earlier publisher may have changed a directory alias while + # we waited. Release the group before acquiring any new target. + current = await self._grouped_write_targets(paths) + if current == ordered: + yield None + return + ordered = current + + async def _grouped_write_targets(self, paths: Sequence[str]) -> list[str]: + requested = {self._validate_container_path(path) for path in paths} + if not requested: + return [] + # The first path sorts before any of its descendants, so it can + # establish the physical workspace using ordinary parent resolution. + first = min(requested) + targets = {await self._resolve_write_target(first)} physical_workspace = self._resolved_workspace or self.workspace - ancestors = set() - for path in paths: + ancestors_by_path: dict[str, tuple[str, ...]] = {} + for path in sorted(requested): + ancestors = [] parent = posixpath.dirname(path) while parent not in ("", ".", "/", self.workspace, physical_workspace): if posixpath.isabs(parent) and not any( @@ -713,19 +731,22 @@ async def file_write_locks(self, paths: Sequence[str]) -> AsyncIterator[None]: ): break if posixpath.basename(parent) not in (".", ".."): - ancestors.add(parent) + ancestors.append(parent) parent = posixpath.dirname(parent) + ancestors_by_path[path] = tuple(ancestors) + resolve = dict(ancestors_by_path) + for ancestors in ancestors_by_path.values(): + for index, ancestor in enumerate(ancestors): + if ancestor not in resolve: + resolve[ancestor] = ancestors[index + 1:] + resolve.pop(first) targets.update({ - await self._resolve_write_target(self._validate_container_path(path)) - for path in ancestors + await self._resolve_write_target( + path, replaceable_parents=tuple(parent for parent in ancestors if parent in requested), + ) + for path, ancestors in resolve.items() }) - # Preserve alias nodes and add physical ancestors before taking any - # locks, so directory recovery and alias edits share one lock order. - ordered = _write_lock_paths(targets, self, workspace=physical_workspace) - async with contextlib.AsyncExitStack() as locks: - for target in ordered: - await locks.enter_async_context(self._resolved_file_write_lock(target)) - yield None + return _write_lock_paths(targets, self, workspace=physical_workspace) @contextlib.asynccontextmanager async def file_write_lock( @@ -806,15 +827,22 @@ async def _resolve_write_target( target: str, *, containment_root: str | None = None, + replaceable_parents: Sequence[str] = (), ) -> str: # Resolve the parent through the file execution wrapper, including its # cwd and command prefix. Keep the final component so atomic writes # replace a file symlink rather than modifying its referent. + # A patch may name both a regular file and its future descendants. directory, filename = posixpath.split(target) command = ( - 'directory=$1; workspace=$2; suffix=; ' + 'directory=$1; workspace=$2; shift 2; suffix=; ' 'while [ ! -d "$directory" ]; do ' - '[ ! -e "$directory" ] && [ ! -L "$directory" ] || exit 66; ' + 'case "${directory##*/}" in ..) exit 66;; esac; ' + '[ ! -L "$directory" ] || exit 66; ' + 'if [ -e "$directory" ]; then ' + '[ -f "$directory" ] || exit 66; ' + 'for parent do [ "$parent" != "$directory" ] || break; done; ' + '[ "$#" -gt 0 ] && [ "$parent" = "$directory" ] || exit 66; fi; ' 'suffix="/${directory##*/}$suffix"; directory=${directory%/*}; ' '[ -n "$directory" ] || directory=/; done; ' 'cd -P -- "$directory" || exit 66; ' @@ -827,10 +855,11 @@ async def _resolve_write_target( if containment_root not in (None, "/tmp"): raise ValueError("container temporary files must stay under /tmp") root = self.workspace if containment_root is None else containment_root + arguments = [parent, root, *( + path if posixpath.isabs(path) else "./" + path for path in replaceable_parents + )] result = await self._exec( - "bash -c " - f"{shlex.quote(command)} opencollab-file-parent {shlex.quote(parent)} " - f"{shlex.quote(root)}", + f"bash -c {shlex.quote(command)} opencollab-file-parent {shlex.join(arguments)}", timeout=DOCKER_CONTROL_TIMEOUT_SECONDS, login=False, ) diff --git a/opencollab/adapters/safety.py b/opencollab/adapters/safety.py index 17dbdafd..3d4b8bee 100644 --- a/opencollab/adapters/safety.py +++ b/opencollab/adapters/safety.py @@ -14,6 +14,7 @@ from __future__ import annotations import os +import posixpath import re import shlex import stat @@ -98,8 +99,11 @@ class SandboxInterceptor: await interceptor.check_cmd_interactive("rm -rf ./build", confirm_fn) # asks user """ - def __init__(self, workspace_root: str, allowed_dirs: list[str] | None = None): + def __init__( + self, workspace_root: str, allowed_dirs: list[str] | None = None, *, remote_filesystem: bool = False, + ): self.root = os.path.realpath(workspace_root) + self._remote_workspace = workspace_root if remote_filesystem and posixpath.isabs(workspace_root) else None # Additional allowed directories (e.g., /tmp for scratch) self.allowed_roots = [self.root] + [os.path.realpath(d) for d in (allowed_dirs or [])] @@ -134,6 +138,17 @@ def check_path(self, target_path: str) -> str: f"(allowed roots: {self.allowed_roots})" ) + def check_remote_file_path(self, target_path: str) -> str | None: + workspace = self._remote_workspace + if workspace is None: + return None + path = posixpath.join(workspace, target_path) + root = posixpath.normpath(workspace) + if posixpath.commonpath((root, posixpath.normpath(path))) != root: + raise PermissionError(f"Path escapes workspace: '{target_path}' (workspace: {workspace})") + # Native file operations validate the actual parent and symlink target. + return path + def check_cmd(self, cmd: str) -> None: """Hard-block destructive commands. Raises PermissionError.""" for pattern in BLOCKED_RE: diff --git a/opencollab/adapters/single2_safety.py b/opencollab/adapters/single2_safety.py index 6c373306..4a159fbc 100644 --- a/opencollab/adapters/single2_safety.py +++ b/opencollab/adapters/single2_safety.py @@ -28,6 +28,10 @@ def __init__(self, policy: SafetyPolicyPort, workspace: str): def check_path(self, target_path: str) -> str: return self._policy.check_path(target_path) + def check_remote_file_path(self, target_path: str) -> str | None: + check = getattr(self._policy, "check_remote_file_path", None) + return check(target_path) if callable(check) else None + def check_cmd(self, cmd: str) -> None: self._policy.check_cmd(cmd) self._check_container_search(cmd) diff --git a/opencollab/adapters/tools/_paths.py b/opencollab/adapters/tools/_paths.py index b71560f3..e75e54e6 100644 --- a/opencollab/adapters/tools/_paths.py +++ b/opencollab/adapters/tools/_paths.py @@ -1,11 +1,4 @@ -"""Shared path-jail helper for filesystem tools. - -Every filesystem tool resolves a user-supplied path through the sandbox's -``check_path`` before touching the environment, so no path can escape the -workspace (ref: ``adapters/safety.py``). This module is the single copy of that -guard; ``fs.py`` and ``apply_patch.py`` delegate here instead of repeating the -``policy and policy.check_path(...)`` dance. -""" +"""Check workspace paths for filesystem tools.""" from __future__ import annotations @@ -15,14 +8,14 @@ from opencollab.application.tool_execution import ToolRuntime -def checked_path(runtime: ToolRuntime, path: str) -> str: - """Jail ``path`` to the workspace via the runtime's safety policy. - - Returns the resolved absolute path, or ``path`` unchanged when no policy is - wired (yolo / tests). Raises ``PermissionError`` if the path escapes the - workspace — callers that prefer a soft error catch it and return a string. - """ +def checked_path(runtime: ToolRuntime, path: str, *, native_file: bool = False) -> str: + """Apply the runtime policy and preserve remote components for native file operations.""" policy = runtime.safety_policy + remote_check = getattr(policy, "check_remote_file_path", None) + if native_file and callable(remote_check): + remote_path = remote_check(path) + if remote_path is not None: + return remote_path checked = policy.check_path(path) if policy else path environment = runtime.environment workspace = getattr(environment, "workspace", None) diff --git a/opencollab/adapters/tools/apply_patch.py b/opencollab/adapters/tools/apply_patch.py index 20497067..cee8f016 100644 --- a/opencollab/adapters/tools/apply_patch.py +++ b/opencollab/adapters/tools/apply_patch.py @@ -108,7 +108,7 @@ async def execute_with_runtime( if env is None: return "Error: no execution environment available." - path = checked_path(runtime, path) + path = checked_path(runtime, path, native_file=True) try: async with host_write_lock(path, env): diff --git a/opencollab/adapters/tools/fs.py b/opencollab/adapters/tools/fs.py index ad1ec24b..509f2bac 100644 --- a/opencollab/adapters/tools/fs.py +++ b/opencollab/adapters/tools/fs.py @@ -90,7 +90,7 @@ async def execute_with_runtime( if env is None: return "Error: no execution environment available." - path = checked_path(runtime, path) + path = checked_path(runtime, path, native_file=True) try: range_reader = getattr(env, "read_text_range", None) @@ -250,7 +250,7 @@ async def execute_with_runtime( if mode == "create" and "content" not in params: return "Error: content is required for create mode." - path = checked_path(runtime, path) + path = checked_path(runtime, path, native_file=True) try: async with host_write_lock(path, env): diff --git a/opencollab/bootstrap/container.py b/opencollab/bootstrap/container.py index 86f15ee8..1e5ea104 100644 --- a/opencollab/bootstrap/container.py +++ b/opencollab/bootstrap/container.py @@ -492,6 +492,8 @@ def build_session_runtime( resolved_llm = _resolve_llm(agent, llm, llm_timeout, provider_retry_budget) if agent_profile is not None: + if safety_policy is None: + safety_policy = build_workspace_safety_policy(resolved_env) safety_policy = agent_profile.wrap_safety( safety_policy, resolved_env.workspace, diff --git a/opencollab/bootstrap/runtime_context.py b/opencollab/bootstrap/runtime_context.py index 79e579b2..6dc7c941 100644 --- a/opencollab/bootstrap/runtime_context.py +++ b/opencollab/bootstrap/runtime_context.py @@ -12,7 +12,7 @@ import uuid from dataclasses import dataclass -from opencollab.adapters.env import Environment +from opencollab.adapters.env import DockerEnvironment, Environment, VMEnvironment from opencollab.adapters.safety import SandboxInterceptor from opencollab.adapters.trace import Tracer from opencollab.application.ports import ( @@ -61,4 +61,4 @@ def build_workspace_safety_policy(env: Environment) -> SafetyPolicyPort | None: """A sandbox interceptor scoped to ``env``'s workspace, or None without one.""" if env is None or not getattr(env, "workspace", None): return None - return SandboxInterceptor(env.workspace) + return SandboxInterceptor(env.workspace, remote_filesystem=isinstance(env, (DockerEnvironment, VMEnvironment))) diff --git a/tests/interfaces/test_sdk_file_path_identity.py b/tests/interfaces/test_sdk_file_path_identity.py index 06caec94..9dbcbd89 100644 --- a/tests/interfaces/test_sdk_file_path_identity.py +++ b/tests/interfaces/test_sdk_file_path_identity.py @@ -7,6 +7,7 @@ import pytest from opencollab import OpenCollab +from opencollab.adapters._env_docker import DockerEnvironment from opencollab.adapters.llm.types import LLMResponse, Usage from opencollab.adapters.safety import SandboxInterceptor from opencollab.adapters.tools.base import host_write_lock @@ -14,6 +15,7 @@ from opencollab.application.tool_execution import ToolRuntime from opencollab.environments import local_environment from opencollab.tools import builtin_tools +from tests.support.docker_native_edits_support import RemoteDockerTransport class _PathModel: @@ -35,15 +37,26 @@ async def complete(self, messages, **kwargs): @pytest.mark.parametrize("absolute", [False, True]) @pytest.mark.parametrize("tool", ["file_read", "file_write", "apply_patch"]) -async def test_sdk_file_tools_match_shell_through_symlink_parent(tmp_path, absolute, tool): +@pytest.mark.parametrize(("backend", "parent"), [("local", "alias"), ("docker", "alias"), ("docker", "sub")]) +async def test_sdk_file_tools_match_shell_through_parent(tmp_path, absolute, tool, backend, parent): (tmp_path / "actual/child").mkdir(parents=True) (tmp_path / "alias").symlink_to("actual/child", target_is_directory=True) - target = tmp_path / "actual/module.py" - unrelated = tmp_path / "module.py" + (tmp_path / "sub").mkdir() + target = tmp_path / ("actual/module.py" if parent == "alias" else "module.py") + unrelated = tmp_path / ("module.py" if parent == "alias" else "actual/module.py") target.write_text("VALUE = 1\n") unrelated.write_text("UNRELATED = 7\n" if tool == "file_read" else "VALUE = 1\n") - raw = "alias/../module.py" - path = os.path.join(str(tmp_path), raw) if absolute else raw + if backend == "docker": + transport = RemoteDockerTransport(tmp_path) + environment = DockerEnvironment( + workspace=transport.remote, container_id="a" * 64, exec_workdir=transport.remote, + ) + transport.attach(environment) + assert not os.path.exists(transport.remote) + else: + environment = local_environment(tmp_path) + raw = f"{parent}/../module.py" + path = os.path.join(environment.workspace, raw) if absolute else raw arguments = {"path": path} if tool == "file_write": arguments.update(mode="str_replace", old_str="VALUE = 1", new_str="VALUE = 42") @@ -53,7 +66,6 @@ async def test_sdk_file_tools_match_shell_through_symlink_parent(tmp_path, absol expected_str="VALUE = 1", new_str="VALUE = 42", ) model = _PathModel(tool, arguments) - environment = local_environment(tmp_path) outputs = [] async def flow(ctx, _): @@ -81,7 +93,7 @@ async def flow(ctx, _): else: assert target.read_text() == "VALUE = 42\n" assert unrelated.read_text() == ("UNRELATED = 7\n" if tool == "file_read" else "VALUE = 1\n") - shell = await environment.exec_cmd("cat alias/../module.py") + shell = await environment.exec_cmd(f"cat {raw}") assert shell.stdout == target.read_text() finally: await environment.cleanup() diff --git a/tests/support/docker_native_edits_support.py b/tests/support/docker_native_edits_support.py index 6478f2ac..0fe3ef6b 100644 --- a/tests/support/docker_native_edits_support.py +++ b/tests/support/docker_native_edits_support.py @@ -80,6 +80,18 @@ async def __call__(self, *argv, timeout=120, input_bytes=None): pass return ProcessResult(proc.returncode, out, err) +class RemoteDockerTransport(LocalDockerTransport): + def __init__(self, workspace: Path, *, pair_reads: bool = False): + super().__init__(workspace, pair_reads=pair_reads) + self.remote = f"/oc-native-test-{uuid.uuid4().hex}" + + async def __call__(self, *argv, **kwargs): + actual = str(self.workspace) + mapped = tuple(arg.replace(self.remote, actual) if isinstance(arg, str) else arg for arg in argv) + result = await super().__call__(*mapped, **kwargs) + return ProcessResult(result.returncode, result.stdout.replace(actual.encode(), self.remote.encode()), + result.stderr.replace(actual.encode(), self.remote.encode())) + def docker_environment(workspace: Path, transport: LocalDockerTransport, *, container_id=None): env = DockerEnvironment( workspace=str(workspace), diff --git a/tests/tools/test_docker_edit_path_identity.py b/tests/tools/test_docker_edit_path_identity.py index 6ecc95ad..742aa6d4 100644 --- a/tests/tools/test_docker_edit_path_identity.py +++ b/tests/tools/test_docker_edit_path_identity.py @@ -191,6 +191,21 @@ async def test_new_parent_directories_resolve_through_existing_alias(tmp_path): assert (directory / "new/nested/file.txt").read_text() == "created" +@pytest.mark.parametrize("path", ["config/default.toml", "missing/../module.py", "config/../module.py"]) +async def test_docker_edit_requires_a_directory_before_parent_traversal_or_file_creation(tmp_path, path): + (tmp_path / "config").write_text("config") + (tmp_path / "module.py").write_text("original") + environment = docker_environment(tmp_path, LocalDockerTransport(tmp_path, pair_reads=False)) + try: + with pytest.raises(OSError, match="execution directory"): + await FileWriteTool().execute_with_runtime({ + "path": path, "mode": "create", "content": "changed", + }, ToolRuntime(environment, None, None)) + assert (tmp_path / "config").read_text() == "config" + assert (tmp_path / "module.py").read_text() == "original" + finally: + await environment.cleanup() + @pytest.mark.parametrize("result", [ ExecResult(1, "", "directory unavailable"), ExecResult(0, "/unframed/path\n", ""), diff --git a/tests/workflows/test_candidate_docker_alias_mutations.py b/tests/workflows/test_candidate_docker_alias_mutations.py index 11c0af4e..d8ba0f93 100644 --- a/tests/workflows/test_candidate_docker_alias_mutations.py +++ b/tests/workflows/test_candidate_docker_alias_mutations.py @@ -6,30 +6,20 @@ import pytest from opencollab import OpenCollab +from opencollab.adapters import _env_docker as docker_module +from opencollab.adapters._candidate_increment import _apply_increment from opencollab.adapters._env_docker import DockerEnvironment -from opencollab.adapters._env_process import ProcessResult from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace from opencollab.adapters.tools.base import host_write_locks from opencollab.adapters.tools.fs import FileWriteTool from opencollab.application.tool_execution import ToolRuntime from opencollab.tools import builtin_tools -from tests.support.docker_native_edits_support import LocalDockerTransport +from tests.support.docker_native_edits_support import LocalDockerTransport, RemoteDockerTransport, docker_environment from tests.workflows.test_candidate_adoption_recovery import _flatten, _repository, _run from tests.workflows.test_candidate_shared_mutations import _WriterModel from tests.workflows.test_workflow_candidate_workspace import _git -class _RemoteNamespaceTransport(LocalDockerTransport): - remote = "/oc-native-alias-workspace" - - async def __call__(self, *argv, **kwargs): - actual = str(self.workspace) - mapped = tuple(arg.replace(self.remote, actual) if isinstance(arg, str) else arg for arg in argv) - result = await super().__call__(*mapped, **kwargs) - return ProcessResult(result.returncode, result.stdout.replace(actual.encode(), self.remote.encode()), - result.stderr.replace(actual.encode(), self.remote.encode())) - - class _RemoteCandidates(EnvCandidateWorkspace): def __init__(self, environment, transport): super().__init__(environment) @@ -42,7 +32,7 @@ async def _candidate_environment(self, path): def _remote_environment(repo): - transport = _RemoteNamespaceTransport(repo, pair_reads=False) + transport = RemoteDockerTransport(repo) environment = DockerEnvironment( workspace=transport.remote, container_id="a" * 64, exec_workdir=transport.remote, command_prefix="export OC_NATIVE_EDIT_PROBE=wrapped", @@ -139,34 +129,34 @@ async def flow(ctx, _): await environment.cleanup() -async def test_remote_alias_group_prepares_locks_before_acquisition_and_releases_cancelled_waiter( +async def test_remote_alias_group_resolves_before_and_after_ordered_acquisition_and_releases_cancelled_waiter( tmp_path, monkeypatch, ): (tmp_path / "settings").mkdir() (tmp_path / "settings/value.py").write_text("VALUE = 1\n") (tmp_path / "alias").symlink_to("settings", target_is_directory=True) environment, _ = _remote_environment(tmp_path) - from opencollab.adapters import _env_docker as docker_module resolve = environment._resolve_write_target resolving_task = asyncio.current_task() resolutions = [] + expected = [environment.workspace + suffix for suffix in ("/alias", "/settings", "/settings/value.py")] - async def resolve_before_locks(target, **kwargs): - assert not any(owner is resolving_task for owner in docker_module._WRITE_LOCK_OWNERS.values()) - result = await resolve(target, **kwargs) - resolutions.append(result) - return result + async def resolve_with_lock_check(target, **kwargs): + owned = [key.split("\0", 1)[1] for key, owner in docker_module._WRITE_LOCK_OWNERS.items() + if owner is resolving_task] + assert owned in ([], expected) + resolutions.append(bool(owned)) + return await resolve(target, **kwargs) - monkeypatch.setattr(environment, "_resolve_write_target", resolve_before_locks) + monkeypatch.setattr(environment, "_resolve_write_target", resolve_with_lock_check) waiting = None try: async with host_write_locks((environment.workspace + "/alias/value.py",), environment): owned = [key.split("\0", 1)[1] for key, owner in docker_module._WRITE_LOCK_OWNERS.items() if owner is resolving_task] - assert owned == sorted(owned) - assert owned == [environment.workspace + suffix for suffix in ( - "/alias", "/settings", "/settings/value.py", - )] + assert owned == expected + assert resolutions == sorted(resolutions) + assert set(resolutions) == {False, True} monkeypatch.setattr(environment, "_resolve_write_target", resolve) async with host_write_locks((environment.workspace + "/settings",), environment): waiting = asyncio.create_task(FileWriteTool().execute_with_runtime({ @@ -180,7 +170,6 @@ async def resolve_before_locks(target, **kwargs): with pytest.raises(asyncio.CancelledError): await waiting assert alias_key not in docker_module._WRITE_LOCK_OWNERS - assert resolutions output = await asyncio.wait_for(FileWriteTool().execute_with_runtime({ "path": "alias/value.py", "mode": "create", "content": "VALUE = 100\n", }, ToolRuntime(environment, None, None)), 5) @@ -191,3 +180,100 @@ async def resolve_before_locks(target, **kwargs): if waiting is not None: await asyncio.gather(waiting, return_exceptions=True) await environment.cleanup() + + +async def test_docker_edits_serialize_on_the_target_after_candidate_retargets_directory_alias( + tmp_path, monkeypatch, +): + repo = _repository(tmp_path) + for directory in ("old", "new"): + (repo / directory).mkdir() + (repo / directory / "module.py").write_text("alpha = 1\nbeta = 1\n") + alias = repo / "alias" + alias.symlink_to("old", target_is_directory=True) + _git(repo, "add", "old", "new", "alias") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: keep two module directories") + alias.unlink() + alias.symlink_to("new", target_is_directory=True) + patch = _git(repo, "diff", "--binary", "--", "alias") + _git(repo, "restore", "--", "alias") + + environment = docker_environment(repo, LocalDockerTransport(repo)) + runtime = ToolRuntime(environment, None, None) + entered, release = asyncio.Event(), asyncio.Event() + execute, read = environment.exec_cmd, environment.read_file + + async def pause_publication(cmd, *args, **kwargs): + if " apply --binary " in cmd and ".candidate-adopt-" in cmd: + entered.set() + await release.wait() + return await execute(cmd, *args, **kwargs) + + async def read_under_target_lock(path): + target_key = environment.container_reference + "\0" + str(repo / "new/module.py") + assert docker_module._WRITE_LOCK_OWNERS.get(target_key) is asyncio.current_task() + return await read(path) + + async def edit(path, name): + return await FileWriteTool().execute_with_runtime({ + "path": path, "mode": "str_replace", "old_str": f"{name} = 1", "new_str": f"{name} = 2", + }, runtime) + + async def wait_for_alias_editor(): + alias_key = environment.container_reference + "\0" + str(alias) + while getattr(docker_module._WRITE_LOCKS.get(alias_key), "users", 0) < 2: + await asyncio.sleep(0.005) + + monkeypatch.setattr(environment, "exec_cmd", pause_publication) + monkeypatch.setattr(environment, "read_file", read_under_target_lock) + publication = asyncio.create_task(_apply_increment(environment, str(repo), patch)) + writers = [] + try: + await asyncio.wait_for(entered.wait(), 5) + writers.append(asyncio.create_task(edit("alias/module.py", "alpha"))) + await asyncio.wait_for(wait_for_alias_editor(), 5) + assert not writers[0].done() + release.set() + await asyncio.wait_for(publication, 5) + writers.append(asyncio.create_task(edit("new/module.py", "beta"))) + receipts = await asyncio.wait_for(asyncio.gather(*writers), 5) + assert all(receipt.startswith("Replaced") for receipt in receipts), receipts + assert (repo / "old/module.py").read_text() == "alpha = 1\nbeta = 1\n" + assert (repo / "new/module.py").read_text() == "alpha = 2\nbeta = 2\n" + finally: + release.set() + await asyncio.gather(publication, *writers, return_exceptions=True) + await environment.cleanup() + + +@pytest.mark.parametrize("new_file", ["default.toml", "nested/default.toml"]) +async def test_sdk_docker_candidate_can_replace_a_file_with_a_directory(tmp_path, new_file): + repo = _repository(tmp_path) + (repo / "config").write_text("enabled = true\n") + _git(repo, "add", "config") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: keep a flat config") + index = (repo / ".git/index").read_bytes() + environment, transport = _remote_environment(repo) + + async def edit(child, _): + config = Path(child.workspace_root) / "config" + config.unlink() + target = config / new_file + target.parent.mkdir(parents=True) + target.write_text("enabled = false\n") + + async def flow(ctx, _): + candidate = await ctx.candidate_workflow(edit, {}, label="nested-config") + await ctx.adopt_candidate(candidate) + return "adopted" + + try: + result = await OpenCollab(repo, environment=environment).workflow( + flow, candidate_workspace=_RemoteCandidates(environment, transport), trace=False, timeout=30, + ) + assert result.ok, result.reason + assert result.output == "adopted" + assert (repo / "config" / new_file).read_text() == "enabled = false\n" + assert (repo / ".git/index").read_bytes() == index + finally: + await environment.cleanup() From 6637691ca509394f7f6df31e3962032f7943a0a0 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 11:30:16 +0800 Subject: [PATCH 26/82] fix(workflow): stage oversized recovery scripts in bounded fragments --- opencollab/adapters/_candidate_adoption.py | 26 ++++++++++++++++--- .../test_candidate_adoption_command_size.py | 4 ++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index d27b93ea..52ae4df0 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -8,6 +8,7 @@ from contextlib import AsyncExitStack, asynccontextmanager from typing import Any +from opencollab.adapters._env_base import ENV_FILE_WRITE_LIMIT_BYTES from opencollab.adapters._git_patch_bytes import _complete, _patch_temporary from opencollab.application.async_timeout import await_owned_operation from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError @@ -17,15 +18,32 @@ async def _adoption_command( environment: Any, workspace: str, command: str, *, repository: bool, ) -> AsyncIterator[str]: - if len(command.encode("utf-8", errors="surrogateescape")) <= 32_768: + size = len(command.encode("utf-8", errors="surrogateescape")) + if size <= 32_768: yield command return resources = AsyncExitStack() - try: - script = await resources.enter_async_context(_patch_temporary( - environment, workspace, command + "\n", prefix=".candidate-command-", suffix=".sh", + + async def stage(content: str) -> str: + return await resources.enter_async_context(_patch_temporary( + environment, workspace, content, prefix=".candidate-command-", suffix=".sh", cleanup_best_effort=True, repository=repository, )) + + try: + content = command + "\n" + if size < ENV_FILE_WRITE_LIMIT_BYTES: + script = await stage(content) + else: + # Keep each transfer within the file and bridge limits. Assemble + # the complete script before it can change the workspace. + chunk_size = ENV_FILE_WRITE_LIMIT_BYTES // 8 + script = await stage("") + for start in range(0, len(content), chunk_size): + fragment = await stage(content[start:start + chunk_size]) + _complete(await environment.exec_cmd( + f"cat -- {shlex.quote(fragment)} >> {shlex.quote(script)}", timeout=120, + ), "candidate command staging") yield f"sh {shlex.quote(script)}" finally: try: diff --git a/tests/workflows/test_candidate_adoption_command_size.py b/tests/workflows/test_candidate_adoption_command_size.py index c9e6f9b1..af21e3ed 100644 --- a/tests/workflows/test_candidate_adoption_command_size.py +++ b/tests/workflows/test_candidate_adoption_command_size.py @@ -19,7 +19,9 @@ async def exec_cmd(self, command, timeout=120.0, **kwargs): @pytest.mark.parametrize("blocked", [False, True]) async def test_bulk_candidate_adoption_keeps_recovery_below_argument_limits(tmp_path, blocked): - repo = tmp_path / "repo" + # Long absolute paths expand the recovery script past the temp-file limit + # even though the repository-relative patch fits under one megabyte. + repo = tmp_path.joinpath(*("nested-" + "workspace-" * 8,) * 5, "repo") (repo / "src").mkdir(parents=True) (repo / "settings").mkdir() names = [f"{number:04d}_" + "module_" * 10 + ".py" for number in range(900)] From babdf4d22df58d9f3b8dc1b5a66d0b5e8778570b Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 11:30:17 +0800 Subject: [PATCH 27/82] fix(llm): settle usage before releasing request slots --- opencollab/adapters/llm/client.py | 83 +++++++--------- .../application/_session_run_completion.py | 26 ++--- opencollab/application/_session_run_usage.py | 21 ++-- opencollab/application/session_run.py | 8 +- .../bootstrap/authored_workflow_bridge.py | 29 +++--- opencollab/bootstrap/team_workflow.py | 5 +- .../test_authored_request_budget_http.py | 3 + .../runtime/test_authored_workflow_bridge.py | 20 +++- .../runtime/test_pooled_generation_timeout.py | 75 ++++++++++++++ .../test_pooled_request_budget_http.py | 98 +++++++++++++++++++ .../test_provider_attempt_budget_http.py | 41 +++++++- 11 files changed, 324 insertions(+), 85 deletions(-) diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index 7e8579cf..dbc51451 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -125,6 +125,7 @@ def __init__( self.supports_request_admission = request_pool is not None self.supports_request_revalidation = True self.supports_attempt_usage = True + self.supports_usage_settlement = True self.model = model self.provider = provider self.max_retries = max(0, max_retries) @@ -313,10 +314,12 @@ async def complete( ``before_request`` runs before each physical attempt. Failed attempts report their known usage through ``on_attempt_usage`` before retry - backoff; ``on_usage`` receives the final aggregate once. + backoff. ``on_usage`` receives the final aggregate before slot release. """ start = time.monotonic() response = None + provider_error: BaseException | None = None + settlement_error: BaseException | None = None settled_usage = None first_attempt = True initial_limit: int | None = None @@ -340,9 +343,9 @@ def prepare_attempt() -> int | None: if self._retry_budget_explicit else _scoped_native_retry_budget(self.provider_retry_budget) ) - # Measure here so every provider path records the call timing. - with recording() as first_token: + try: async with AsyncExitStack() as request_owner: + first_token = request_owner.enter_context(recording()) try: await request_owner.enter_async_context(request_slot(self.request_pool, self.request_capacity)) initial_limit = request_limit() @@ -414,53 +417,43 @@ def prepare_attempt() -> int | None: first_event_timeout=self.first_event_timeout, stream_idle_timeout=self.stream_idle_timeout, ) - settled_usage = getattr(response, "usage", None) - transport_timing = first_token.snapshot() - try: - response.transport_timing = transport_timing - except (AttributeError, TypeError): - # Custom provider responses may reject new attributes. The usage ledger - # still records their timing. - pass - await _record_api_usage_async( - provider=self.provider, - model=self.model, - wire_protocol=self.wire_protocol, - reasoning_effort=reasoning_effort, - base_url=self.base_url, - latency_s=time.monotonic() - start, - status="success", - response=response, - transport_timing=transport_timing, - ) - return response except BaseException as exc: - settled_usage = getattr(response if response is not None else exc, "usage", None) - if response is not None: - exc.usage = settled_usage - else: - try: - await _record_api_usage_async( - provider=self.provider, - model=self.model, - wire_protocol=self.wire_protocol, - reasoning_effort=reasoning_effort, - base_url=self.base_url, - latency_s=time.monotonic() - start, - status="error", - error=exc, - transport_timing=first_token.snapshot(), - ) - except asyncio.CancelledError as cancellation: - usage = getattr(exc, "usage", None) - if usage is not None: - cancellation.usage = usage - raise + provider_error = exc raise finally: + settled_usage = getattr(response if response is not None else provider_error, "usage", None) + transport_timing = first_token.snapshot() + latency_s = time.monotonic() - start + if response is not None: + try: + response.transport_timing = transport_timing + except (AttributeError, TypeError): + pass if on_usage is not None and settled_usage is not None: try: on_usage(settled_usage) except BaseException as exc: exc.usage = settled_usage - raise + settlement_error = exc + finally: + try: + await _record_api_usage_async( + provider=self.provider, + model=self.model, + wire_protocol=self.wire_protocol, + reasoning_effort=reasoning_effort, + base_url=self.base_url, + latency_s=latency_s, + status="success" if response is not None else "error", + response=response, + error=provider_error, + transport_timing=transport_timing, + ) + except BaseException as exc: + if settled_usage is not None: + exc.usage = settled_usage + raise + finally: + if settlement_error is not None: + raise settlement_error + return response diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index c6e2764e..a6f0659c 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -29,8 +29,8 @@ ) from opencollab.application._session_run_trace import _SessionRunTraceMixin from opencollab.application._session_run_usage import ( - _AttemptUsageAccounting, _complete_with_error_usage, + _CompletionUsageAccounting, _normalize_completion_usage, ) from opencollab.application._tool_loop_execution import _apply_completed_prefix_progress @@ -809,12 +809,14 @@ async def _invoke_summary( kwargs["max_output_tokens"] = max_output_tokens start = time.monotonic() protected_call = self.state.wind_down_done - attempt_usage = _AttemptUsageAccounting(self, protected_call=protected_call) + usage_accounting = _CompletionUsageAccounting(self, protected_call=protected_call) if getattr(self.llm, "supports_attempt_usage", False): - kwargs["on_attempt_usage"] = attempt_usage.record + kwargs["on_attempt_usage"] = usage_accounting.record_attempt admitted = asyncio.Event() if getattr(self.llm, "supports_request_admission", False) else None if admitted is not None: kwargs["on_admitted"] = admitted.set + if getattr(self.llm, "supports_usage_settlement", False): + kwargs["on_usage"] = usage_accounting.settle if getattr(self.llm, "supports_request_revalidation", False): def revalidate() -> int: limit = self._revalidate_request( @@ -832,9 +834,8 @@ def revalidate() -> int: def account_response(response: CompletionResponse) -> None: input_tokens, total_tokens = _normalize_completion_usage(response.usage) - self.state.add_used_tokens(total_tokens - attempt_usage.tokens) + usage_accounting.settle(response.usage) self._record_usage_event(response.usage, total_tokens, purpose="summary", late=abandoned) - self._mark_budget_reserve_consumed(protected_call=protected_call) self.state.add_markup_recovered(getattr(response.usage, "markup_recovered", 0)) self.state.set_context_tokens(input_tokens) if abandoned: @@ -848,8 +849,8 @@ async def complete_owned() -> CompletionResponse: if self._run_control is not None: self._raise_if_run_cancelled() return await _complete_with_error_usage( - self, complete, messages, protected_call=protected_call, - usage_purpose="summary", on_response=account_response, attempt_usage=attempt_usage, **kwargs, + self, complete, messages, usage_accounting=usage_accounting, + usage_purpose="summary", on_response=account_response, **kwargs, ) finally: self._draining_provider_tasks.discard(asyncio.current_task()) @@ -890,12 +891,14 @@ async def _invoke_llm(self, **kwargs: Any) -> CompletionResponse: if self._run_control is not None: self._raise_if_run_cancelled() protected_call = self.state.wind_down_done - attempt_usage = _AttemptUsageAccounting(self, protected_call=protected_call) + usage_accounting = _CompletionUsageAccounting(self, protected_call=protected_call) if getattr(self.llm, "supports_attempt_usage", False): - kwargs["on_attempt_usage"] = attempt_usage.record + kwargs["on_attempt_usage"] = usage_accounting.record_attempt admitted = asyncio.Event() if getattr(self.llm, "supports_request_admission", False) else None if admitted is not None: kwargs["on_admitted"] = admitted.set + if getattr(self.llm, "supports_usage_settlement", False): + kwargs["on_usage"] = usage_accounting.settle if getattr(self.llm, "supports_request_revalidation", False): kwargs["before_request"] = lambda: self._revalidate_request( kwargs["messages"], kwargs.get("tools"), thinking=kwargs.get("thinking", False), @@ -907,13 +910,12 @@ async def complete_owned() -> CompletionResponse: nonlocal late_accounted try: response = await _complete_with_error_usage( - self, self.llm.complete, protected_call=protected_call, **kwargs, - attempt_usage=attempt_usage, + self, self.llm.complete, usage_accounting=usage_accounting, **kwargs, ) late_accounted = asyncio.current_task() in self._draining_provider_tasks self._account_completion_usage( response, protected_call=protected_call, late=late_accounted, - accounted_tokens=attempt_usage.tokens, + accounted_tokens=usage_accounting.tokens, ) return response finally: diff --git a/opencollab/application/_session_run_usage.py b/opencollab/application/_session_run_usage.py index 3a4cd1db..66833881 100644 --- a/opencollab/application/_session_run_usage.py +++ b/opencollab/application/_session_run_usage.py @@ -34,16 +34,21 @@ def _normalize_completion_usage(usage: Any) -> tuple[int, int]: return input_tokens if context_tokens is None else context_tokens, max(reported_total, input_tokens + output_tokens) -class _AttemptUsageAccounting: +class _CompletionUsageAccounting: def __init__(self, runner: Any, *, protected_call: bool) -> None: self.runner = runner self.protected_call = protected_call self.tokens = 0 - def record(self, usage: Any) -> None: - total = _normalize_completion_usage(usage)[1] - self.runner.state.add_used_tokens(total) - self.tokens += total + def record_attempt(self, usage: Any) -> None: + self._charge(_normalize_completion_usage(usage)[1]) + + def settle(self, usage: Any) -> None: + self._charge(_normalize_completion_usage(usage)[1] - self.tokens) + + def _charge(self, tokens: int) -> None: + self.runner._charge_completion_tokens(tokens) + self.tokens += tokens self.runner._mark_budget_reserve_consumed(protected_call=self.protected_call) @@ -51,9 +56,8 @@ async def _complete_with_error_usage( runner: Any, complete: Callable[..., Awaitable[Any]], *args: Any, - protected_call: bool, + usage_accounting: _CompletionUsageAccounting, usage_purpose: str = "completion", - attempt_usage: _AttemptUsageAccounting | None = None, **kwargs: Any, ) -> Any: """Charge reported error usage inside the provider owner exactly once.""" @@ -64,13 +68,12 @@ async def _complete_with_error_usage( usage = getattr(exc, "usage", None) if usage is not None: _input_tokens, total_tokens = _normalize_completion_usage(usage) - runner.state.add_used_tokens(total_tokens - (attempt_usage.tokens if attempt_usage is not None else 0)) + usage_accounting.settle(usage) late = asyncio.current_task() in runner._draining_provider_tasks record_usage = getattr(runner, "_record_usage_event", None) if callable(record_usage): record_usage(usage, total_tokens, purpose=usage_purpose, late=late, error=exc) - runner._mark_budget_reserve_consumed(protected_call=protected_call) if late: runner._late_provider_usage += (total_tokens,) if runner.late_provider_usage_checkpoint is not None: diff --git a/opencollab/application/session_run.py b/opencollab/application/session_run.py index e71e0cfa..49bf68d9 100644 --- a/opencollab/application/session_run.py +++ b/opencollab/application/session_run.py @@ -185,6 +185,7 @@ def __init__( self._late_provider_usage: tuple[int, ...] = () # The session supplies its owned persistence queue after adopting this runner. self.late_provider_usage_checkpoint: Callable[[], object] | None = None + self._on_completion_tokens: Callable[[int], None] | None = None def reset_runtime_for_user_turn(self) -> None: """Restore agent capabilities narrowed by the previous turn.""" @@ -264,13 +265,18 @@ def _record_late_provider_result( self._draining_provider_tasks.discard(task) self._finish_run_control_if_quiesced() + def _charge_completion_tokens(self, tokens: int) -> None: + self.state.add_used_tokens(tokens) + if tokens and self._on_completion_tokens is not None: + self._on_completion_tokens(tokens) + def _account_completion_usage( self, response: Any, *, protected_call: bool, late: bool, usage_accounted: bool = False, accounted_tokens: int = 0, ) -> None: _input_tokens, total_tokens = _normalize_completion_usage(response.usage) if not usage_accounted: - self.state.add_used_tokens(total_tokens - accounted_tokens) + self._charge_completion_tokens(total_tokens - accounted_tokens) self._record_usage_event(response.usage, total_tokens, purpose="completion", late=late) self._mark_budget_reserve_consumed(protected_call=protected_call) if late: diff --git a/opencollab/bootstrap/authored_workflow_bridge.py b/opencollab/bootstrap/authored_workflow_bridge.py index 2066ffc0..f9dbb3d4 100644 --- a/opencollab/bootstrap/authored_workflow_bridge.py +++ b/opencollab/bootstrap/authored_workflow_bridge.py @@ -39,20 +39,25 @@ def _write_json(path: Path, value: Any) -> None: def _restore_error(value: dict[str, Any]) -> Exception: - name = value.pop("type") - if name in {"_RunCancelled", "_TeamBudgetStop"}: - return {"_RunCancelled": _RunCancelled, "_TeamBudgetStop": _TeamBudgetStop}[name]() - if name == "_TokenBudgetStop": - return _TokenBudgetStop( + error_type = value["type"] + error: Exception + if error_type == "_RunCancelled": + error = _RunCancelled() + elif error_type == "_TeamBudgetStop": + error = _TeamBudgetStop() + elif error_type == "_TokenBudgetStop": + error = _TokenBudgetStop( reserved_input_tokens=value["reserved_input_tokens"], remaining_budget=value["remaining_budget"], minimum_output_tokens=value["minimum_output_tokens"], ) - cls = getattr(llm_errors, name, getattr(builtins, name, RuntimeError)) - if not isinstance(cls, type) or not issubclass(cls, Exception): - cls = RuntimeError - error = cls(value.pop("message")) + else: + cls = getattr(llm_errors, error_type, getattr(builtins, error_type, RuntimeError)) + if not isinstance(cls, type) or not issubclass(cls, Exception): + cls = RuntimeError + error = cls(value["message"]) for name, item in value.items(): - setattr(error, name, Usage(**item) if name == "usage" else item) + if name not in {"type", "message"}: + setattr(error, name, Usage(**item) if name == "usage" else item) return error @@ -241,7 +246,9 @@ async def _respond(self, identity: str, request: dict[str, Any]) -> None: "remaining_budget", "minimum_output_tokens"): value = getattr(exc, name, None) if value is not None: - error[name] = to_plain_data(value) + error[name] = ( + asdict(value) if name == "usage" and isinstance(value, Usage) else to_plain_data(value) + ) reply = {"error": error} await await_owned_operation(self._environment.write_file( posixpath.join(self._directory, f"{identity}.json"), json.dumps(reply, ensure_ascii=False), diff --git a/opencollab/bootstrap/team_workflow.py b/opencollab/bootstrap/team_workflow.py index 1792b440..4988f620 100644 --- a/opencollab/bootstrap/team_workflow.py +++ b/opencollab/bootstrap/team_workflow.py @@ -85,7 +85,6 @@ def build_child(**kwargs: Any) -> Session: def on_event(event: RunEvent) -> None: if event.type == "usage": - parent.state.add_used_tokens(event.data["total_tokens"]) parent.enqueue_auto_save() if control.on_event is not None: control.on_event(event) @@ -97,7 +96,9 @@ def on_event(event: RunEvent) -> None: runtime.safety_policy if kwargs["env"] is runtime.environment else build_workspace_safety_policy(kwargs["env"]) ) - return build_session(**kwargs) + session = build_session(**kwargs) + session.runner._on_completion_tokens = parent.runner._charge_completion_tokens + return session save_dir = ( tempfile.mkdtemp(prefix=f"workflow-agent-{runtime.aid}-", dir=self._save_dir) diff --git a/tests/runtime/test_authored_request_budget_http.py b/tests/runtime/test_authored_request_budget_http.py index eaa35023..5abcfa1c 100644 --- a/tests/runtime/test_authored_request_budget_http.py +++ b/tests/runtime/test_authored_request_budget_http.py @@ -32,6 +32,7 @@ async def test_native_usage_is_reported_once_inside_the_slot( ): config, _field = configuration usages = [] + released = asyncio.Event() install_sdk_transport(monkeypatch, config["provider"], completion_http_response) native_slot = transport.request_slot @@ -42,8 +43,10 @@ async def observed_slot(*args): yield finally: assert [usage.total_tokens for usage in usages] == [4] + released.set() async def record(**kwargs): + assert released.is_set() if outcome == "ledger_error": raise ValueError("controlled ledger failure") if outcome == "ledger_cancellation": diff --git a/tests/runtime/test_authored_workflow_bridge.py b/tests/runtime/test_authored_workflow_bridge.py index f1ac07af..32be958f 100644 --- a/tests/runtime/test_authored_workflow_bridge.py +++ b/tests/runtime/test_authored_workflow_bridge.py @@ -2,6 +2,7 @@ import asyncio import os +from dataclasses import asdict from types import SimpleNamespace import pytest @@ -13,23 +14,40 @@ from opencollab.adapters.llm.types import LLMResponse, Usage from opencollab.adapters.session_environments import SessionEnvironmentPool from opencollab.adapters.tools.fs import FileWriteTool -from opencollab.application._session_run_shared import _TokenBudgetStop +from opencollab.application._session_run_shared import _TeamBudgetStop, _TokenBudgetStop from opencollab.application.ports import ( ContextTokenEstimatorPort, RequestOutputRequirementPort, RequestTokenEstimatorPort, ) +from opencollab.application.run_control import _RunCancelled from opencollab.application.tool_execution import ToolRuntime from opencollab.bootstrap.authored_workflow_bridge import ( AuthoredWorkflowBridge, BridgeClient, BridgeEnvironment, BridgeLLM, + _restore_error, ) from opencollab.bootstrap.session_factory import build_session from opencollab.domain.agent import Agent +@pytest.mark.parametrize("error", [ + _TokenBudgetStop(reserved_input_tokens=1027, remaining_budget=500, minimum_output_tokens=1), + _TeamBudgetStop(), + _RunCancelled(), +]) +def test_bridge_restores_usage_on_control_errors(error): + usage = Usage(1_498, 2, cache_read_tokens=19, raw_usage={"total_tokens": 1_500}) + payload = {"type": type(error).__name__, "message": str(error), "usage": asdict(usage), **vars(error)} + restored = _restore_error(payload) + + assert type(restored) is type(error) + assert str(restored) == str(error) + assert vars(restored) == {**vars(error), "usage": usage} + + @pytest.mark.parametrize("docker", [False, True]) async def test_bridge_retains_native_whole_edit_lock(tmp_path, monkeypatch, docker): image = os.environ.get("OPENCOLLAB_DOCKER_TEST_IMAGE") diff --git a/tests/runtime/test_pooled_generation_timeout.py b/tests/runtime/test_pooled_generation_timeout.py index ceefde22..8d3b09f7 100644 --- a/tests/runtime/test_pooled_generation_timeout.py +++ b/tests/runtime/test_pooled_generation_timeout.py @@ -1,6 +1,8 @@ from __future__ import annotations import asyncio +import json +import threading import pytest @@ -8,6 +10,7 @@ from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.llm.request_admission import request_slot from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.application.run_control import RunControl from opencollab.application.session_run import GenerationTimeoutError from opencollab.bootstrap import build_session from opencollab.domain.agent import Agent @@ -134,3 +137,75 @@ async def complete(*_args, **_kwargs): assert session.pending_cleanup_tasks == () assert (await asyncio.wait_for(client.complete([]), 1)).content == "answer" await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("provider_error", [False, True]) +async def test_blocked_ledger_releases_slot_and_preserves_late_session_usage( + tmp_path, monkeypatch, local_environment_factory, summary, provider_error, +): + writing, release = threading.Event(), threading.Event() + second_started = asyncio.Event() + events, calls = [], [] + log_path = tmp_path / "usage.jsonl" + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(log_path)) + original_record = transport.record_api_usage + + def delayed_record(**kwargs): + if not writing.is_set(): + writing.set() + assert release.wait(5) + original_record(**kwargs) + + async def complete(*_args, **_kwargs): + calls.append(True) + if len(calls) == 2: + second_started.set() + return LLMResponse(content="next", usage=Usage(2, 2)) + usage = Usage(13, 24) + if provider_error: + exc = ValueError("provider disconnected") + exc.usage = usage + raise exc + return LLMResponse(content="Earlier context.", usage=usage) + + monkeypatch.setattr(transport, "record_api_usage", delayed_record) + monkeypatch.setattr(transport, "complete_openai", complete) + async with LLMClient( + api_key="local", request_pool=str(tmp_path), request_capacity=1, context_window=40_000, + ) as client: + session = build_session( + agent=Agent(name="blocked-ledger", system_prompt="system"), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.2, + run_control=RunControl(on_event=lambda event: events.append(event) if event.type == "usage" else None), + ) + session.messages = history(summary) + owner = asyncio.create_task(session.run_loop()) + following = None + try: + assert await asyncio.to_thread(writing.wait, 5) + assert session.used_tokens == 37 + following = asyncio.create_task(client.complete([])) + await asyncio.wait_for(second_started.wait(), 2) + with pytest.raises(GenerationTimeoutError): + await asyncio.wait_for(owner, 2) + assert session.pending_cleanup_tasks + assert len(calls) == 2 + finally: + release.set() + await asyncio.gather(owner, *session.pending_cleanup_tasks, return_exceptions=True) + if following is not None: + assert (await asyncio.wait_for(following, 2)).content == "next" + await session.aclose() + + assert session.used_tokens == 37 + assert session.runner.late_provider_usage == (37,) + assert not session.pending_cleanup_tasks + assert len(events) == 1 + assert events[0].data["purpose"] == ("summary" if summary else "completion") + assert events[0].data["total_tokens"] == 37 + assert events[0].data["late"] is True + rows = [json.loads(line) for line in log_path.read_text().splitlines()] + assert [(row["status"], row["usage"]["total_tokens"]) for row in rows] == [ + ("error" if provider_error else "success", 37), ("success", 4), + ] diff --git a/tests/runtime/test_pooled_request_budget_http.py b/tests/runtime/test_pooled_request_budget_http.py index 04ae0327..885ac0cb 100644 --- a/tests/runtime/test_pooled_request_budget_http.py +++ b/tests/runtime/test_pooled_request_budget_http.py @@ -289,3 +289,101 @@ async def complete_summary(messages, *, on_response, **kwargs): assert session.used_tokens == (20_000 if change == "exhausted" else 0) finally: await session.aclose() + + +async def test_nested_team_workflow_charges_parent_before_other_native_admission( + tmp_path, monkeypatch, local_environment_factory, +): + from opencollab.application.event_bus import EventBus + from opencollab.application.scheduler import Scheduler + from opencollab.application.tool_execution import ToolRuntime + from opencollab.application.workflow_registry import Registry, WorkflowSpec + from opencollab.bootstrap.context_builder import SpawnConfig + from opencollab.bootstrap.team_workflow import TeamWorkflowTool + from opencollab.domain.scheduler import SessionControlBlock + + provider_started, release_provider, queued, ledger_started, release_ledger = ( + asyncio.Event() for _ in range(5) + ) + requests = [] + + async def reply(request): + requests.append(json.loads(request.content)) + provider_started.set() + await release_provider.wait() + payload = completion_http_response(request, text="child answer").json() + payload["usage"].update(prompt_tokens=20_198, total_tokens=20_200) + return http_response(request, 200, json=payload) + + async def flow(ctx, _args): + return await ctx.agent("child request", tools=[]) + + native_slot, record = transport.request_slot, transport._record_api_usage_async + entries = 0 + + @asynccontextmanager + async def observed_slot(*args): + nonlocal entries + entries += 1 + if entries == 2: + queued.set() + async with native_slot(*args): + yield + + async def delayed_record(**kwargs): + if kwargs["status"] == "success": + ledger_started.set() + await release_ledger.wait() + await record(**kwargs) + + install_sdk_transport(monkeypatch, "openai", reply) + monkeypatch.setattr(transport, "request_slot", observed_slot) + monkeypatch.setattr(transport, "_record_api_usage_async", delayed_record) + scheduler = Scheduler(session_factory=None, worktree_pool=None, event_sink=EventBus(), max_budget_tokens=20_000) + registry = Registry() + registry.register(WorkflowSpec(name="registered", description="Return a result", fn=flow)) + env = local_environment_factory(str(tmp_path)) + async with LLMClient( + api_key="local-test", request_pool=str(tmp_path), request_capacity=1, max_retries=0, + ) as client: + parent, sibling = [ + build_session( + env=env, llm=client, max_budget_tokens=30_000, + agent=Agent(name=name, system_prompt="system", max_tokens_per_step=128), + team_budget_exhausted=lambda: scheduler.budget_exhausted, + ) for name in ("parent", "sibling") + ] + for aid, session in enumerate((parent, sibling)): + scheduler.table.add(SessionControlBlock( + aid=aid, parent_aid=0 if aid else None, agent=session.agent, state=session.state, + )) + config = SpawnConfig(model="gpt-4o", provider="openai", api_key=None, base_url=None, + llm_timeout=5, tracer=None, event_bus=EventBus(), permission_policy=None) + tool = TeamWorkflowTool(lambda: parent, registry, config, None) + active = asyncio.create_task(tool.execute_with_runtime( + {"name": "registered", "args": {}}, ToolRuntime(env, None, None), + )) + following = None + try: + await asyncio.wait_for(provider_started.wait(), 5) + await sibling.add_user_message("queued request") + following = asyncio.create_task(sibling.run_loop()) + await asyncio.wait_for(queued.wait(), 5) + release_provider.set() + await asyncio.wait_for(ledger_started.wait(), 5) + assert parent.used_tokens == scheduler.used_tokens == 20_200 + assert scheduler.budget_exhausted + assert await asyncio.wait_for(following, 5) == "" + assert "team budget exceeded" in sibling.state.terminal_reason + assert not active.done() + assert len(requests) == 1 + release_ledger.set() + result = json.loads(await asyncio.wait_for(active, 5)) + assert result["tokens"] == parent.used_tokens == 20_200 + assert result["output"] == "child answer" + finally: + release_provider.set() + release_ledger.set() + await asyncio.gather(active, *([following] if following is not None else []), return_exceptions=True) + await sibling.aclose() + await parent.aclose() diff --git a/tests/runtime/test_provider_attempt_budget_http.py b/tests/runtime/test_provider_attempt_budget_http.py index 7a629929..e76d8590 100644 --- a/tests/runtime/test_provider_attempt_budget_http.py +++ b/tests/runtime/test_provider_attempt_budget_http.py @@ -6,6 +6,7 @@ import sys import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path import pytest @@ -334,20 +335,42 @@ async def flow(ctx, _args): assert [request["observed"] for request in requests] == [0, 1500] -@pytest.mark.parametrize("budget, retries, count", [(2000, 1, 1), (20000, 1, 2), (None, 1, 2), (2000, 0, 1)]) +@pytest.mark.parametrize("budget, retries, count, stopped_by", [ + (2000, 1, 1, "_TokenBudgetStop"), + (20000, 1, 1, "_TeamBudgetStop"), + (20000, 1, 1, "_RunCancelled"), + (20000, 1, 2, None), + (None, 1, 2, None), + (2000, 0, 1, None), +]) async def test_authored_subprocess_accounts_failure_before_retry( - tmp_path, monkeypatch, attempt_server, budget, retries, count, + tmp_path, monkeypatch, attempt_server, budget, retries, count, stopped_by, ): monkeypatch.setenv("PYTHONPATH", str(REPO_ROOT)) hosts = [] url, requests = attempt_server("chat", lambda: (hosts[0].used_tokens, hosts[1].bridge.tokens)) (tmp_path / "authored.py").write_text( + 'from opencollab import RunControl\n' 'async def main(ctx, args):\n' - f' return await ctx.agent({("context " * 300)!r}, tools=[], budget={budget!r})\n', + ' events = []\n' + ' def observe(event):\n' + ' if event.type == "usage":\n' + ' events.append([event.data["total_tokens"], event.data["error_type"],\n' + ' event.data["cache_read_tokens"]])\n' + f' goal = {"context " * 300!r}\n' + f' await ctx.agent_run(goal, tools=[], budget={budget!r}, run_control=RunControl(on_event=observe))\n' + ' return {"workflow": ctx.tokens_spent(), "session": ctx.sessions[0].used_tokens, "events": events}\n', encoding="utf-8", ) environment = LocalEnvironment(str(tmp_path)) parent, tool, runtime = _manager(environment, budget=budget) + if stopped_by == "_TeamBudgetStop": + parent.runner._team_budget_exhausted = lambda: parent.used_tokens >= 1500 + if stopped_by == "_RunCancelled": + def request_stop(_error): + hosts[1].stop_requested = True + return 0.0 + monkeypatch.setattr(retry, "extract_retry_after_seconds", request_stop) for target in (tool._config, parent.agent): target.base_url = url target.api_key = "controlled-test" # pragma: allowlist secret @@ -364,7 +387,17 @@ async def test_authored_subprocess_accounts_failure_before_retry( await asyncio.wait_for(child.task, 20) assert child.result["status"] == "completed", child.result assert len(requests) == count - assert child.tokens == child.bridge.tokens == parent.used_tokens == 1500 + 4 * (count - 1) + expected = 1500 + 4 * (count - 1) + assert child.tokens == child.bridge.tokens == parent.used_tokens == expected + artifacts = Path(child.artifacts) + assert json.loads((artifacts / "usage.json").read_text()) == {"tokens": expected} + result = json.loads((artifacts / "result.json").read_text()) + assert result["tokens"] == expected + output = result["output"] + assert output["workflow"] == output["session"] == expected + assert [event[0] for event in output["events"]] == [expected] + if stopped_by is not None: + assert output["events"] == [[expected, stopped_by, None]] if count == 2: assert requests[1]["observed"] == (1500, 1500) records = _records(tmp_path) From 2b21cb599601e1c9870eeab95006be01ac02cb53 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 11:30:17 +0800 Subject: [PATCH 28/82] fix(scheduler): retry inbox delivery after individual budget releases --- opencollab/application/_scheduler_run.py | 30 ++++--- ...st_scheduler_cross_agent_budget_cleanup.py | 84 +++++++++++++++++++ 2 files changed, 101 insertions(+), 13 deletions(-) diff --git a/opencollab/application/_scheduler_run.py b/opencollab/application/_scheduler_run.py index b69e6231..dcf9f3de 100644 --- a/opencollab/application/_scheduler_run.py +++ b/opencollab/application/_scheduler_run.py @@ -191,13 +191,7 @@ async def _run_turn_exclusive(self, aid: int, user_message: str) -> str: break await self._wait_for_scheduler_progress(aid) continue - # The waiter joins the wait, never the liveness question: it - # never completes on its own, so counting it as pending work - # would keep a quiescent team's turn running until someone - # cancelled it. - if cancel_waiter is not None: - pending.add(cancel_waiter) - await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + await self._wait_for_active_scheduler_tasks(pending, cancel_waiter) finally: if cancel_waiter is not None: cancel_waiter.cancel() @@ -244,9 +238,7 @@ async def _wait_for_restored_turn(self, aid: int, turn_start: int) -> None: if not pending: await self._wait_for_scheduler_progress(aid) continue - if cancel_waiter is not None: - pending.add(cancel_waiter) - await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + await self._wait_for_active_scheduler_tasks(pending, cancel_waiter) except asyncio.CancelledError: # The public caller performs bounded team cleanup after propagating # cancellation. A suspended recovery driver has already returned, @@ -390,8 +382,8 @@ def _message_budget_cleanup_tasks(self) -> set[asyncio.Task]: # Any member's deferred grant can hold the shared budget an inbox needs. return { task - for _aid, _grant, _baseline, tasks, _finished in self._pending_provider_budget_leases.values() - for task in tasks + for _aid, _grant, _baseline, tasks, finished in self._pending_provider_budget_leases.values() + for task in tasks - finished } def _active_scheduler_tasks(self) -> set[asyncio.Task]: @@ -419,6 +411,18 @@ def _active_scheduler_tasks(self) -> set[asyncio.Task]: if not task.done() and task is not current } + async def _wait_for_active_scheduler_tasks( + self, pending: set[asyncio.Task], cancel_waiter: asyncio.Task | None = None, + ) -> None: + budget_owners = self._message_budget_cleanup_tasks() + budget_owners.discard(asyncio.current_task()) + waiting = pending | budget_owners + if cancel_waiter is not None: + waiting.add(cancel_waiter) + done, _ = await asyncio.wait(waiting, return_when=asyncio.FIRST_COMPLETED) + if done & budget_owners: + await self._drain_ready_message_inboxes() + async def _wait_for_scheduler_progress(self, aid: int) -> None: """Block for a producer completion or report a state with no producer.""" settled_budget_owners: set[asyncio.Task] = set() @@ -430,7 +434,7 @@ async def _wait_for_scheduler_progress(self, aid: int) -> None: return producers = self._active_scheduler_tasks() if producers: - await asyncio.wait(producers, return_when=asyncio.FIRST_COMPLETED) + await self._wait_for_active_scheduler_tasks(producers) return budget_owners.update(self._message_budget_cleanup_tasks()) callbacks = budget_owners - settled_budget_owners diff --git a/tests/workflows/test_scheduler_cross_agent_budget_cleanup.py b/tests/workflows/test_scheduler_cross_agent_budget_cleanup.py index 881f152d..3f090161 100644 --- a/tests/workflows/test_scheduler_cross_agent_budget_cleanup.py +++ b/tests/workflows/test_scheduler_cross_agent_budget_cleanup.py @@ -156,6 +156,90 @@ def observe_progress(): await asyncio.gather(active, return_exceptions=True) +async def test_inbox_runs_after_first_of_two_deferred_grants_releases(monkeypatch, tmp_path): + started = [asyncio.Event(), asyncio.Event()] + release = [asyncio.Event(), asyncio.Event()] + both_deferred = asyncio.Event() + delivered = asyncio.Event() + recipient_requests = [] + + async def handler(request): + body = json.loads(request.content) + text = next(m["content"] for m in reversed(body["messages"]) if m["role"] == "user") + if text in {"owner 0 operation", "owner 1 operation"}: + index = int(text.split()[1]) + started[index].set() + try: + await release[index].wait() + except asyncio.CancelledError: + await release[index].wait() + if "accepted follow-up" in text: + recipient_requests.append(text) + delivered.set() + return completion_http_response(request, text="done", output_tokens=19_986 if text == "lead warmup" else 2) + + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + install_sdk_transport(monkeypatch, "openai", handler) + async with LLMClient( + model="gpt-4o", provider="openai", + api_key="local-controlled", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + ) as client: + scheduler = Scheduler( + session_factory=_Factory(client, 1.0), + worktree_pool=WorktreePool(str(tmp_path), use_worktrees=False), + event_sink=EventBus(), max_budget_tokens=40_000, + ) + lead = build_session(agent=Agent(name="lead", system_prompt="lead"), llm=client, max_budget_tokens=40_000) + scheduler.register_lead(lead) + assert await scheduler.run("lead warmup") == "done" + for aid, role in enumerate(("recipient", "owner-one", "owner-two"), 1): + assert await scheduler.spawn(0, role, f"{role} warmup") == aid + await scheduler._tasks[aid] + assert scheduler.used_tokens == 20_000 + + original_active = scheduler._active_scheduler_tasks + + def observe(): + tasks = original_active() + if len(scheduler._pending_provider_budget_leases) == 2: + both_deferred.set() + return tasks + + monkeypatch.setattr(scheduler, "_active_scheduler_tasks", observe) + for aid in (2, 3): + scheduler._sessions[aid].runner._per_call_timeout = 0.5 + work = [asyncio.create_task(scheduler.run_turn(aid, f"owner {aid - 2} operation")) for aid in (2, 3)] + try: + await asyncio.wait_for(asyncio.gather(*(event.wait() for event in started)), 3) + assert scheduler.allocated_tokens == 40_000 + assert await scheduler.send_message(0, 1, "work", "accepted follow-up") == "Message queued to aid 1." + await asyncio.wait_for(both_deferred.wait(), 3) + release[0].set() + await asyncio.wait_for(delivered.wait(), 3) + + assert not scheduler._message_inbox[1] + assert {lease[0] for lease in scheduler._pending_provider_budget_leases.values()} == {3} + assert not work[1].done() + assert len(recipient_requests) == 1 + second_owner, = next(iter(scheduler._pending_provider_budget_leases.values()))[3] + settled = asyncio.Event() + second_owner.add_done_callback(lambda _: settled.set()) + release[1].set() + results = await asyncio.wait_for(asyncio.gather(*work, return_exceptions=True), 3) + assert all(isinstance(error, SchedulerTurnError) for error in results) + await asyncio.wait_for(settled.wait(), 3) + assert not scheduler._pending_provider_budget_leases + assert scheduler.allocated_tokens == scheduler.used_tokens + finally: + for event in release: + event.set() + await scheduler.cleanup() + await asyncio.gather(*work, return_exceptions=True) + + def _budget_callback_scheduler(tmp_path, owner): state = SessionState(messages=[]) state.set_phase(SessionPhase.DONE) From 40412bcea51fcab643d21a54e774c0f26b2c1f7d Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 11:30:17 +0800 Subject: [PATCH 29/82] fix(workflow): retain loaded modules through runtime finalization --- opencollab/application/workflow.py | 2 + opencollab/application/workflow_candidates.py | 6 ++ .../bootstrap/_workflow_runtime_cleanup.py | 23 ++++++-- .../bootstrap/_workflow_runtime_discovery.py | 10 +++- .../bootstrap/_workflow_runtime_execution.py | 6 ++ .../test_sdk_workflow_import_cleanup.py | 59 +++++++++++++++++++ 6 files changed, 100 insertions(+), 6 deletions(-) diff --git a/opencollab/application/workflow.py b/opencollab/application/workflow.py index 25ed7efc..49ab29c0 100644 --- a/opencollab/application/workflow.py +++ b/opencollab/application/workflow.py @@ -31,6 +31,7 @@ import math import time from collections.abc import Mapping, Sequence +from contextlib import ExitStack from copy import deepcopy from typing import Any @@ -210,6 +211,7 @@ def __init__( _TaskConcurrencyPermit | None ] = contextvars.ContextVar("workflow_task_concurrency_permit", default=None) self._pending_cleanup_tasks: set[asyncio.Task[Any]] = set() + self._workflow_imports: ExitStack | None = None self._active_call_tasks: set[asyncio.Task[Any]] = set() self._active_session_tasks: set[asyncio.Task[Any]] = set() self._agent_failures: list[dict[str, Any]] = [] diff --git a/opencollab/application/workflow_candidates.py b/opencollab/application/workflow_candidates.py index 459cc3bd..03c728c4 100644 --- a/opencollab/application/workflow_candidates.py +++ b/opencollab/application/workflow_candidates.py @@ -6,6 +6,7 @@ import inspect import os from collections.abc import Awaitable, Callable, Mapping, Sequence +from contextlib import ExitStack from dataclasses import dataclass, replace from typing import Any @@ -495,6 +496,7 @@ async def run() -> CandidateRun: lease = await self._candidate_workspace.acquire(label) budget_lease = None child = None + imports = ExitStack() candidate: CandidateRun | None = None failure: BaseException | None = None preserve_lease = False @@ -531,6 +533,7 @@ async def run() -> CandidateRun: child._active_task_concurrency_permit = self._active_task_concurrency_permit child._budget_escape_state = self._budget_escape_state child.budget._parent_allocation = (self.budget, budget_lease) + child._workflow_imports = imports try: output = await workflow_fn(child, dict(args)) except Exception as exc: # noqa: BLE001 - preserve candidate edits @@ -627,6 +630,9 @@ async def run() -> CandidateRun: getattr(child._factory, "has_pending_isolated_cleanup", False) ) finally: + if child is not None: + child._workflow_imports = None + imports.close() if budget_lease is not None: self.budget.release(budget_lease) return await self._finish_candidate_run( diff --git a/opencollab/bootstrap/_workflow_runtime_cleanup.py b/opencollab/bootstrap/_workflow_runtime_cleanup.py index aafdff67..aa4b4513 100644 --- a/opencollab/bootstrap/_workflow_runtime_cleanup.py +++ b/opencollab/bootstrap/_workflow_runtime_cleanup.py @@ -80,11 +80,24 @@ def _positive_cleanup_timeout(value: float) -> float: return timeout async def _wait_for_context_cleanup(ctx: WorkflowContext, *, timeout: float) -> bool: - """Wait once for the cleanup tasks currently owned by the context.""" - pending = set(ctx.pending_cleanup_tasks) - if pending: - _done, pending = await asyncio.wait(pending, timeout=timeout) - return not pending and not ctx.pending_cleanup_tasks + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + saw_empty = False + while True: + pending = set(ctx.pending_cleanup_tasks) + if pending: + saw_empty = False + remaining = deadline - loop.time() + if remaining <= 0: + return False + _done, unfinished = await asyncio.wait(pending, timeout=remaining) + if unfinished: + return False + elif saw_empty: + return True + else: + saw_empty = True + await asyncio.sleep(0) def _session_environments(ctx: WorkflowContext) -> tuple[Any, ...]: environments: list[Any] = [] diff --git a/opencollab/bootstrap/_workflow_runtime_discovery.py b/opencollab/bootstrap/_workflow_runtime_discovery.py index c53283f8..bf12bf29 100644 --- a/opencollab/bootstrap/_workflow_runtime_discovery.py +++ b/opencollab/bootstrap/_workflow_runtime_discovery.py @@ -160,7 +160,7 @@ def release(self) -> None: self.capture_modules() _remove_workflow_package(self.package_name) - def release_after_cleanup(self, ctx: object) -> None: + def release_after_cleanup(self, ctx: object, *, finalized: bool = False) -> None: if not isinstance(ctx, WorkflowContext): self.release() return @@ -169,6 +169,10 @@ def release_after_cleanup(self, ctx: object) -> None: def pending_tasks(): return {task for task in getattr(ctx, "pending_cleanup_tasks", ()) if task is not caller} + if finalized and not pending_tasks(): + self.release() + return + loop = asyncio.get_running_loop() watched = set() scheduled = False @@ -209,6 +213,10 @@ def _bind_runtime_package(spec: WorkflowSpec, owner: _WorkflowModuleOwner) -> Wo @wraps(original) async def run(ctx, args): owner.acquire() + imports = ctx._workflow_imports if isinstance(ctx, WorkflowContext) else None + if imports is not None: + imports.callback(owner.release_after_cleanup, ctx, finalized=True) + return await original(ctx, args) try: return await original(ctx, args) finally: diff --git a/opencollab/bootstrap/_workflow_runtime_execution.py b/opencollab/bootstrap/_workflow_runtime_execution.py index 0d82f927..7e91c5e1 100644 --- a/opencollab/bootstrap/_workflow_runtime_execution.py +++ b/opencollab/bootstrap/_workflow_runtime_execution.py @@ -5,6 +5,7 @@ import asyncio import uuid from collections.abc import Callable +from contextlib import ExitStack from typing import Any, Literal from opencollab.adapters.trace import Tracer @@ -155,6 +156,8 @@ async def run_workflow( stop_reason: Literal["budget_exceeded"] | None = None workflow_failure: BaseException | None = None cancellation: asyncio.CancelledError | None = None + imports = ExitStack() + ctx._workflow_imports = imports try: result = await fn(ctx, args) except WorkflowBudgetExceeded as exc: @@ -181,6 +184,9 @@ async def run_workflow( cancellation = cancellation or cleanup_cancellation except BaseException as exc: cleanup_failure = exc + finally: + ctx._workflow_imports = None + imports.close() tracer_failure = _close_tracer_capture(tracer) if owns_tracer else None tracer_write_error, tracer_dropped_steps, inspection_failure = _inspect_tracer(tracer) diff --git a/tests/interfaces/test_sdk_workflow_import_cleanup.py b/tests/interfaces/test_sdk_workflow_import_cleanup.py index d2cc7852..7f57de67 100644 --- a/tests/interfaces/test_sdk_workflow_import_cleanup.py +++ b/tests/interfaces/test_sdk_workflow_import_cleanup.py @@ -220,3 +220,62 @@ async def parent(ctx, args): if not running.done(): running.cancel() await asyncio.gather(running, return_exceptions=True) + + +async def test_named_workflow_keeps_imports_for_followup_dispatched_by_completion_callback(tmp_path): + directory = tmp_path / "workflows" + directory.mkdir() + (directory / "_cleanup.py").write_text("def close(events):\n events['closed'] = True\n", encoding="utf-8") + (directory / "callback.py").write_text('''import asyncio +from opencollab import workflow + +class CloseResource: + name = "resource" + description = "Close the resource managed by this workflow" + parameters = {"type": "object", "properties": {}} + + def __init__(self, events): + self.events = events + + def to_openai_schema(self): + return {"type": "function", "function": { + "name": self.name, "description": self.description, "parameters": self.parameters, + }} + + async def execute_with_runtime(self, args, runtime): + from ._cleanup import close + close(self.events) + return "closed" + +@workflow +async def callback_cleanup(ctx, args): + events = args["events"] + def start_followup(): + events["tasks"].append(asyncio.create_task(ctx.agent("finish", tools=[CloseResource(events)]))) + + first = asyncio.create_task(ctx.agent("start")) + first.add_done_callback(lambda _: asyncio.get_running_loop().call_soon(start_followup)) + await first + return "returned" +''', encoding="utf-8") + + class Model(ResourceModel): + async def complete(self, messages, **kwargs): + if any(m["role"] == "user" and "finish" in str(m["content"]) for m in messages) and not any( + m["role"] == "tool" for m in messages + ): + return await super().complete(messages, **kwargs) + return LLMResponse(content="done", usage=Usage(input_tokens=5, output_tokens=5), finish_reason="end_turn") + + events = {"closed": False, "tasks": []} + before = _import_state() + result = await OpenCollab(tmp_path, model="offline").workflow( + "callback_cleanup", {"events": events}, llm=Model(), trace=False, cleanup_timeout=2, + ) + assert result.status == "completed", result.reason + assert result.output == "returned" + assert len(events["tasks"]) == 1 + followup, = events["tasks"] + assert followup.result() == "done" + assert events["closed"] + assert _import_state() == before From 8d69a3f2627e68f62b20658bea43882a06e46908 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 11:44:07 +0800 Subject: [PATCH 30/82] fix(anthropic): preserve the default output limit during admission --- opencollab/adapters/llm/anthropic_provider.py | 2 +- .../test_anthropic_request_budget_http.py | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/opencollab/adapters/llm/anthropic_provider.py b/opencollab/adapters/llm/anthropic_provider.py index 9de632d1..a0f09108 100644 --- a/opencollab/adapters/llm/anthropic_provider.py +++ b/opencollab/adapters/llm/anthropic_provider.py @@ -407,7 +407,7 @@ async def complete_anthropic( def prepare_attempt() -> None: if before_attempt is not None and (limit := before_attempt()) is not None: - kwargs["max_tokens"] = limit + kwargs["max_tokens"] = int(limit or DEFAULT_MAX_OUTPUT_TOKENS) def report_failure(_exc: BaseException) -> None: if failed_usages and (usage := failed_usages[-1]) is not None and on_attempt_usage is not None: diff --git a/tests/runtime/test_anthropic_request_budget_http.py b/tests/runtime/test_anthropic_request_budget_http.py index fde65e30..1de21e69 100644 --- a/tests/runtime/test_anthropic_request_budget_http.py +++ b/tests/runtime/test_anthropic_request_budget_http.py @@ -32,6 +32,26 @@ def _wire_input_tokens(request): messages.insert(0, {"role": "system", "content": request["system"]}) return estimate_request_tokens(messages, request.get("tools")) + +@pytest.mark.parametrize(("requested", "expected"), [(None, 8192), (0, 8192), (32, 32)]) +async def test_anthropic_admission_applies_the_native_output_default(monkeypatch, requested, expected): + requests = [] + + async def handler(request): + requests.append(json.loads(request.content)) + return completion_http_response(request) + + install_sdk_transport(monkeypatch, "anthropic", handler) + async with LLMClient( + model=_MODEL, provider="anthropic", api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + ) as client: + result = await client.complete([{"role": "user", "content": "Answer ok"}], max_output_tokens=requested) + + assert result.content == "ok" + assert [body["max_tokens"] for body in requests] == [expected] + + @pytest.mark.parametrize("budget,thinking,expected_tokens,expected_calls", [ (11_000, True, 9702, 2), (None, True, 9702, 2), From 8145f2bd2cdf82694f233a3bde0d819b7976951c Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 11:44:07 +0800 Subject: [PATCH 31/82] fix(docker): resolve grouped edits within the candidate repository --- opencollab/adapters/_candidate_increment.py | 16 ++-- opencollab/adapters/_env_docker.py | 74 ++++++++++++------- opencollab/bootstrap/runtime_context.py | 3 +- .../interfaces/test_sdk_file_path_identity.py | 25 +++++++ .../test_candidate_docker_alias_mutations.py | 62 +++++++++++++++- 5 files changed, 143 insertions(+), 37 deletions(-) diff --git a/opencollab/adapters/_candidate_increment.py b/opencollab/adapters/_candidate_increment.py index 244276bc..3f04f1b9 100644 --- a/opencollab/adapters/_candidate_increment.py +++ b/opencollab/adapters/_candidate_increment.py @@ -10,6 +10,7 @@ from opencollab.adapters._candidate_adoption import _adoption_transaction from opencollab.adapters._candidate_physical import _physical_git +from opencollab.adapters._env_docker import DockerEnvironment from opencollab.adapters._git_patch_bytes import _complete, _patch_temporary from opencollab.adapters.git_patch import _git_command_and_config_guard from opencollab.adapters.tools.base import host_write_locks @@ -33,8 +34,10 @@ async def _apply_increment( if not isinstance(patch, str) or not patch.strip(): raise ValueError("candidate patch must be non-empty") preserved = tuple(_safe_path(path) for path in preserve_paths) + delegate = getattr(environment, "_delegate", environment) + docker = isinstance(delegate, DockerEnvironment) lock_root = ( - workspace if getattr(environment, "local_filesystem", False) + workspace if docker or getattr(environment, "local_filesystem", False) else getattr(environment, "workspace", workspace) ) async with _patch_temporary( @@ -54,10 +57,13 @@ async def _apply_increment( for protected in preserved for path in candidate_paths ): raise ValueError("candidate patch overlaps a preserved path") - async with host_write_locks( - (posixpath.join(lock_root, ".candidate-adoption"), - *(posixpath.join(workspace, path) for path in sorted(candidate_paths))), environment, - ): + targets = (posixpath.join(lock_root, ".candidate-adoption"), + *(posixpath.join(workspace, path) for path in sorted(candidate_paths))) + locks = ( + delegate.file_write_locks(targets, _repository_root=workspace) + if docker else host_write_locks(targets, environment) + ) + async with locks: git = "git -c core.filemode=true" guard = "" if strict and repository: diff --git a/opencollab/adapters/_env_docker.py b/opencollab/adapters/_env_docker.py index 468d7dc0..2d3559b3 100644 --- a/opencollab/adapters/_env_docker.py +++ b/opencollab/adapters/_env_docker.py @@ -693,41 +693,49 @@ async def write_file_with_change(self, path: str, content: str) -> bool | None: return await self._write_file(path, content, observe_change=True) @contextlib.asynccontextmanager - async def file_write_locks(self, paths: Sequence[str]) -> AsyncIterator[None]: + async def file_write_locks( + self, paths: Sequence[str], *, _repository_root: str | None = None, + ) -> AsyncIterator[None]: self._ensure_active() await self._bind_attached() if self._container_id is None: raise RuntimeError("Container not started. Call setup() first.") - ordered = await self._grouped_write_targets(paths) + ordered = await self._grouped_write_targets(paths, repository_root=_repository_root) while True: async with contextlib.AsyncExitStack() as locks: for target in ordered: await locks.enter_async_context(self._resolved_file_write_lock(target)) # An earlier publisher may have changed a directory alias while # we waited. Release the group before acquiring any new target. - current = await self._grouped_write_targets(paths) + current = await self._grouped_write_targets(paths, repository_root=_repository_root) if current == ordered: yield None return ordered = current - async def _grouped_write_targets(self, paths: Sequence[str]) -> list[str]: + async def _grouped_write_targets( + self, paths: Sequence[str], *, repository_root: str | None = None, + ) -> list[str]: requested = {self._validate_container_path(path) for path in paths} if not requested: return [] # The first path sorts before any of its descendants, so it can # establish the physical workspace using ordinary parent resolution. + parent_cache: dict[tuple[str, str, tuple[str, ...]], tuple[str, str]] = {} first = min(requested) - targets = {await self._resolve_write_target(first)} - physical_workspace = self._resolved_workspace or self.workspace + workspace = repository_root or self.workspace + targets = {await self._resolve_write_target( + first, parent_cache=parent_cache, repository_root=repository_root, + )} + physical_workspace = parent_cache[(workspace, posixpath.dirname(first), ())][0] ancestors_by_path: dict[str, tuple[str, ...]] = {} for path in sorted(requested): ancestors = [] parent = posixpath.dirname(path) - while parent not in ("", ".", "/", self.workspace, physical_workspace): + while parent not in ("", ".", "/", workspace, physical_workspace): if posixpath.isabs(parent) and not any( _within_directory(root, posixpath.normpath(parent)) - for root in (self.workspace, physical_workspace) + for root in (workspace, physical_workspace) ): break if posixpath.basename(parent) not in (".", ".."): @@ -743,6 +751,7 @@ async def _grouped_write_targets(self, paths: Sequence[str]) -> list[str]: targets.update({ await self._resolve_write_target( path, replaceable_parents=tuple(parent for parent in ancestors if parent in requested), + parent_cache=parent_cache, repository_root=repository_root, ) for path, ancestors in resolve.items() }) @@ -828,19 +837,40 @@ async def _resolve_write_target( *, containment_root: str | None = None, replaceable_parents: Sequence[str] = (), + parent_cache: dict[tuple[str, str, tuple[str, ...]], tuple[str, str]] | None = None, + repository_root: str | None = None, ) -> str: - # Resolve the parent through the file execution wrapper, including its - # cwd and command prefix. Keep the final component so atomic writes - # replace a file symlink rather than modifying its referent. - # A patch may name both a regular file and its future descendants. directory, filename = posixpath.split(target) + if containment_root not in (None, "/tmp"): + raise ValueError("container temporary files must stay under /tmp") + root = containment_root or repository_root or self.workspace + key = (root, directory, tuple(replaceable_parents)) + parent = parent_cache.get(key) if parent_cache is not None else None + if parent is None: + parent = await self._resolve_write_parent(directory, root, replaceable_parents) + if parent_cache is not None: + parent_cache[key] = parent + resolved_root, resolved = parent + # Owned temporary files do not change the cached worktree root. + if containment_root is None and repository_root is None: + self._resolved_workspace = resolved_root + # Keep the final component for atomic replacement of file symlinks. + resolved_target = posixpath.normpath(posixpath.join(resolved, filename)) + if not _within_directory(resolved_root, resolved_target): + boundary = "workspace" if containment_root is None else "temporary directory" + raise PermissionError(f"container file path escapes {boundary}: {target}") + return resolved_target + + async def _resolve_write_parent( + self, directory: str, root: str, replaceable_parents: Sequence[str], + ) -> tuple[str, str]: + # A patch may replace an existing file or symlink with a directory. command = ( 'directory=$1; workspace=$2; shift 2; suffix=; ' 'while [ ! -d "$directory" ]; do ' 'case "${directory##*/}" in ..) exit 66;; esac; ' - '[ ! -L "$directory" ] || exit 66; ' - 'if [ -e "$directory" ]; then ' - '[ -f "$directory" ] || exit 66; ' + 'if [ -e "$directory" ] || [ -L "$directory" ]; then ' + '[ -f "$directory" ] || [ -L "$directory" ] || exit 66; ' 'for parent do [ "$parent" != "$directory" ] || break; done; ' '[ "$#" -gt 0 ] && [ "$parent" = "$directory" ] || exit 66; fi; ' 'suffix="/${directory##*/}$suffix"; directory=${directory%/*}; ' @@ -852,9 +882,6 @@ async def _resolve_write_target( parent = directory or "." if not posixpath.isabs(parent): parent = "./" + parent - if containment_root not in (None, "/tmp"): - raise ValueError("container temporary files must stay under /tmp") - root = self.workspace if containment_root is None else containment_root arguments = [parent, root, *( path if posixpath.isabs(path) else "./" + path for path in replaceable_parents )] @@ -873,16 +900,7 @@ async def _resolve_write_target( or not posixpath.isabs(resolved) ): raise OSError("cannot resolve container file execution directory") - resolved_root = posixpath.normpath(resolved_root) - # Owned /tmp files must not replace the cached workspace root. A later - # workspace write would then treat the temporary directory as the project. - if containment_root is None: - self._resolved_workspace = resolved_root - resolved_target = posixpath.normpath(posixpath.join(resolved, filename)) - if not _within_directory(resolved_root, resolved_target): - boundary = "workspace" if containment_root is None else "temporary directory" - raise PermissionError(f"container file path escapes {boundary}: {target}") - return resolved_target + return posixpath.normpath(resolved_root), resolved async def _physical_workspace(self) -> str: if self._resolved_workspace is not None: diff --git a/opencollab/bootstrap/runtime_context.py b/opencollab/bootstrap/runtime_context.py index 6dc7c941..dd786125 100644 --- a/opencollab/bootstrap/runtime_context.py +++ b/opencollab/bootstrap/runtime_context.py @@ -61,4 +61,5 @@ def build_workspace_safety_policy(env: Environment) -> SafetyPolicyPort | None: """A sandbox interceptor scoped to ``env``'s workspace, or None without one.""" if env is None or not getattr(env, "workspace", None): return None - return SandboxInterceptor(env.workspace, remote_filesystem=isinstance(env, (DockerEnvironment, VMEnvironment))) + backend = getattr(env, "_delegate", env) + return SandboxInterceptor(env.workspace, remote_filesystem=isinstance(backend, (DockerEnvironment, VMEnvironment))) diff --git a/tests/interfaces/test_sdk_file_path_identity.py b/tests/interfaces/test_sdk_file_path_identity.py index 9dbcbd89..a7797b86 100644 --- a/tests/interfaces/test_sdk_file_path_identity.py +++ b/tests/interfaces/test_sdk_file_path_identity.py @@ -3,16 +3,19 @@ import asyncio import json import os +from types import SimpleNamespace import pytest from opencollab import OpenCollab from opencollab.adapters._env_docker import DockerEnvironment +from opencollab.adapters._team_candidate_environment import _TeamCandidateEnvironment from opencollab.adapters.llm.types import LLMResponse, Usage from opencollab.adapters.safety import SandboxInterceptor from opencollab.adapters.tools.base import host_write_lock from opencollab.adapters.tools.fs import FileWriteTool from opencollab.application.tool_execution import ToolRuntime +from opencollab.bootstrap.runtime_context import build_workspace_safety_policy from opencollab.environments import local_environment from opencollab.tools import builtin_tools from tests.support.docker_native_edits_support import RemoteDockerTransport @@ -99,6 +102,28 @@ async def flow(ctx, _): await environment.cleanup() +async def test_wrapped_docker_workspace_resolves_parent_components_in_the_container(tmp_path): + (tmp_path / "sub").mkdir() + target = tmp_path / "module.py" + target.write_text("VALUE = 1\n") + transport = RemoteDockerTransport(tmp_path) + environment = DockerEnvironment( + workspace=transport.remote, container_id="a" * 64, exec_workdir=transport.remote, + ) + transport.attach(environment) + wrapped = _TeamCandidateEnvironment(SimpleNamespace(environment=environment), transport.remote) + runtime = ToolRuntime(wrapped, build_workspace_safety_policy(wrapped), None) + try: + result = await FileWriteTool().execute_with_runtime({ + "path": "sub/../module.py", "mode": "str_replace", + "old_str": "VALUE = 1", "new_str": "VALUE = 42", + }, runtime) + assert result.startswith("Replaced"), result + assert target.read_text() == "VALUE = 42\n" + finally: + await environment.cleanup() + + async def test_native_edit_uses_the_selected_physical_file_lock(tmp_path): (tmp_path / "actual/child").mkdir(parents=True) (tmp_path / "alias").symlink_to("actual/child", target_is_directory=True) diff --git a/tests/workflows/test_candidate_docker_alias_mutations.py b/tests/workflows/test_candidate_docker_alias_mutations.py index d8ba0f93..c12374d2 100644 --- a/tests/workflows/test_candidate_docker_alias_mutations.py +++ b/tests/workflows/test_candidate_docker_alias_mutations.py @@ -247,10 +247,16 @@ async def wait_for_alias_editor(): @pytest.mark.parametrize("new_file", ["default.toml", "nested/default.toml"]) -async def test_sdk_docker_candidate_can_replace_a_file_with_a_directory(tmp_path, new_file): +@pytest.mark.parametrize("symlink", [False, True]) +async def test_sdk_docker_candidate_can_replace_a_file_with_a_directory(tmp_path, new_file, symlink): repo = _repository(tmp_path) - (repo / "config").write_text("enabled = true\n") - _git(repo, "add", "config") + if symlink: + (repo / "config-source").write_text("enabled = true\n") + (repo / "config").symlink_to("config-source") + _git(repo, "add", "config", "config-source") + else: + (repo / "config").write_text("enabled = true\n") + _git(repo, "add", "config") _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: keep a flat config") index = (repo / ".git/index").read_bytes() environment, transport = _remote_environment(repo) @@ -274,6 +280,56 @@ async def flow(ctx, _): assert result.ok, result.reason assert result.output == "adopted" assert (repo / "config" / new_file).read_text() == "enabled = false\n" + if symlink: + assert (repo / "config-source").read_text() == "enabled = true\n" assert (repo / ".git/index").read_bytes() == index finally: await environment.cleanup() + + +async def test_sdk_docker_candidate_uses_the_same_git_root_for_capture_and_adoption(tmp_path): + repo = _repository(tmp_path) + package = repo / "package" + package.mkdir() + (package / "module.py").write_text("VALUE = 1\n") + _git(repo, "add", "package") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare a subdirectory workflow") + index = (repo / ".git/index").read_bytes() + transport = LocalDockerTransport(repo, pair_reads=False) + environment = docker_environment(package, transport) + + async def edit(child, _): + (Path(child.workspace_root).parent / "other.py").write_text("VALUE = 2\n") + + async def flow(ctx, _): + candidate = await ctx.candidate_workflow(edit, {}, label="update-neighbor") + assert "b/other.py" in candidate.effective_diff + await ctx.adopt_candidate(candidate) + return "adopted" + + try: + result = await OpenCollab(package, environment=environment).workflow( + flow, candidate_workspace=_RemoteCandidates(environment, transport), trace=False, timeout=30, + ) + assert result.ok, result.reason + assert (repo / "other.py").read_text() == "VALUE = 2\n" + assert (repo / ".git/index").read_bytes() == index + assert environment._resolved_workspace in {None, str(package)} + finally: + await environment.cleanup() + + +async def test_docker_group_reuses_sibling_parent_resolution_in_each_pass(tmp_path): + source = tmp_path / "src" + source.mkdir() + transport = LocalDockerTransport(tmp_path, pair_reads=False) + environment = docker_environment(tmp_path, transport) + try: + paths = tuple(str(source / f"module_{index}.py") for index in range(48)) + async with environment.file_write_locks(paths): + resolutions = [row for row in transport.calls if "opencollab-file-parent" in row["command"]] + assert len(resolutions) == 4 + await environment.write_file(paths[0], "VALUE = 1\n") + assert (source / "module_0.py").read_text() == "VALUE = 1\n" + finally: + await environment.cleanup() From ccc7e3c8537f8849ae89729153b72d5dc4809069 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 11:44:07 +0800 Subject: [PATCH 32/82] fix(workflow): retain ownership of interrupted recovery allocation --- opencollab/adapters/_candidate_adoption.py | 18 ++++++--- .../test_candidate_adoption_cancellation.py | 39 +++++++++++++++++++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index 52ae4df0..435df3a1 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -107,12 +107,18 @@ async def _adoption_transaction( storage = shlex.quote(_complete(await environment.exec_cmd( f"git -C {shlex.quote(workspace)} rev-parse --absolute-git-dir", timeout=30, ), "candidate recovery storage").removesuffix("\n")) - backup = _complete(await environment.exec_cmd( - f'mktemp -d {storage}/candidate-recovery-XXXXXXXX', timeout=30, - ), "candidate recovery storage").removesuffix("\n") - backup_quoted = shlex.quote(backup) + backup = None keep_backup = False + + async def allocate_backup() -> None: + nonlocal backup + backup = _complete(await environment.exec_cmd( + f'mktemp -d {storage}/candidate-recovery-XXXXXXXX', timeout=30, + ), "candidate recovery storage").removesuffix("\n") + try: + await await_owned_operation(allocate_backup(), propagate_cancellation=True) + assert backup is not None copies = [] restores = [] recreate_directories = [] @@ -187,10 +193,10 @@ async def _adoption_transaction( ) from recovery raise finally: - if not keep_backup: + if backup is not None and not keep_backup: try: await await_owned_operation(environment.exec_cmd( - f"rm -rf -- {backup_quoted}", timeout=120, + f"rm -rf -- {shlex.quote(backup)}", timeout=120, )) except Exception: pass diff --git a/tests/workflows/test_candidate_adoption_cancellation.py b/tests/workflows/test_candidate_adoption_cancellation.py index 6e202bbb..2fb6b65c 100644 --- a/tests/workflows/test_candidate_adoption_cancellation.py +++ b/tests/workflows/test_candidate_adoption_cancellation.py @@ -7,8 +7,12 @@ import pytest +from opencollab.adapters._candidate_increment import _apply_increment +from opencollab.adapters._env_local import LocalEnvironment from opencollab.application.workflow import WorkflowContext from opencollab.application.workflow_candidates import CandidateRun +from tests.workflows.test_candidate_adoption_recovery import _repository +from tests.workflows.test_workflow_candidate_workspace import _git @pytest.mark.parametrize("modern", [False, True]) @@ -71,3 +75,38 @@ async def deliver(candidate, preserve_paths): with pytest.raises(OSError) as raised: await context.adopt_candidate(selected) assert raised.value is failure + + +async def test_backend_cancellation_cleans_recovery_directory_created_before_receipt(tmp_path, monkeypatch): + repo = _repository(tmp_path) + original = (repo / "other.py").read_text() + (repo / "other.py").write_text("VALUE = 42\n") + patch = _git(repo, "diff", "--binary", "--", "other.py") + _git(repo, "restore", "--worktree", "--", "other.py") + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + allocated, release_receipt = asyncio.Event(), asyncio.Event() + + async def hold_recovery_directory_receipt(command, *args, **kwargs): + result = await execute(command, *args, **kwargs) + if command.startswith("mktemp -d ") and "candidate-recovery-" in command: + allocated.set() + await release_receipt.wait() + return result + + monkeypatch.setattr(environment, "exec_cmd", hold_recovery_directory_receipt) + task = asyncio.create_task(_apply_increment(environment, str(repo), patch)) + try: + await asyncio.wait_for(allocated.wait(), 5) + assert list((repo / ".git").glob("candidate-recovery-*")) + task.cancel() + await asyncio.sleep(0) + release_receipt.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 5) + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert (repo / "other.py").read_text() == original + finally: + release_receipt.set() + await asyncio.gather(task, return_exceptions=True) + await environment.cleanup() From 98b7c6e25b12a3f5193a43936f9ace6253d31167 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 11:44:08 +0800 Subject: [PATCH 33/82] fix(session): retain journal rewrites across native history refreshes --- opencollab/application/_anthropic_history.py | 33 ++++++++-- .../application/_session_run_completion.py | 12 +++- opencollab/application/session.py | 13 ++-- opencollab/application/session_run.py | 1 + opencollab/application/session_snapshot.py | 12 +--- .../test_anthropic_preserved_history.py | 65 +++++++++++++++++++ 6 files changed, 111 insertions(+), 25 deletions(-) diff --git a/opencollab/application/_anthropic_history.py b/opencollab/application/_anthropic_history.py index d8fc4a61..be924483 100644 --- a/opencollab/application/_anthropic_history.py +++ b/opencollab/application/_anthropic_history.py @@ -3,6 +3,7 @@ from __future__ import annotations import copy +from collections.abc import Callable from typing import Any from opencollab.domain.session import SessionState @@ -34,20 +35,37 @@ def _edited_anthropic_history(before: list[dict], after: list[dict]) -> list[dic return _strip_anthropic_thinking(after, index) return after -def _anthropic_history_input(state: SessionState) -> list[dict]: +def _anthropic_source_edit_start(state: SessionState) -> int | None: + history = state.anthropic_history + source = history.get("source_messages") if history is not None else None + if source is None or state.messages[:len(source)] == source: + return None + for index, (old, current) in enumerate(zip(source, state.messages)): + if old != current: + return index + return len(state.messages) + +def _anthropic_history_input( + state: SessionState, *, on_source_edit: Callable[[int], None] | None = None, +) -> list[dict]: """Reuse the native view until its original transcript prefix changes.""" history = state.anthropic_history if history is None: return state.messages - source = history.get("source_messages") - if source is not None and state.messages[:len(source)] != source: - return _edited_anthropic_history(source, state.messages) + edit_start = _anthropic_source_edit_start(state) + if edit_start is not None: + if on_source_edit is not None: + on_source_edit(edit_start) + return _edited_anthropic_history(history["source_messages"], state.messages) return copy.deepcopy([*history["messages"], *state.messages[history["source_count"]:]]) -def _remember_anthropic_history(state: SessionState, messages: list[dict], tools: list[dict] | None) -> list[dict]: +def _remember_anthropic_history( + state: SessionState, messages: list[dict], tools: list[dict] | None, + *, on_source_edit: Callable[[int], None] | None = None, +) -> list[dict]: """Persist edits so removed blocks and compacted content stay removed.""" history = state.anthropic_history - messages = _edited_anthropic_history(_anthropic_history_input(state), messages) + messages = _edited_anthropic_history(_anthropic_history_input(state, on_source_edit=on_source_edit), messages) if history is not None: messages = _edited_anthropic_history(history["messages"], messages) if history["tools"] != tools: @@ -60,6 +78,7 @@ def _remember_anthropic_history(state: SessionState, messages: list[dict], tools def _recover_anthropic_history( state: SessionState, messages: list[dict], tools: list[dict] | None, error: Exception, + *, on_source_edit: Callable[[int], None] | None = None, ) -> tuple[list[dict], int] | None: """Retire replayed thinking after an explicit provider prefix rejection.""" status = getattr(error, "status_code", None) @@ -82,4 +101,4 @@ def _recover_anthropic_history( ) if not removed: return None - return _remember_anthropic_history(state, recovered, tools), removed + return _remember_anthropic_history(state, recovered, tools, on_source_edit=on_source_edit), removed diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index a6f0659c..9341bfd7 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -623,7 +623,9 @@ async def _complete_with_choice( params = None await self._prepare_request_budget(messages, tools, thinking=thinking, thinking_params=params) if self._requires_thinking_prefix(): - messages = _remember_anthropic_history(self.state, messages, tools) + messages = _remember_anthropic_history( + self.state, messages, tools, on_source_edit=self._on_anthropic_source_edit, + ) max_output_tokens = self._request_output_limit( messages, tools, thinking=thinking, thinking_params=params, ) @@ -643,7 +645,9 @@ async def _complete_with_choice( ) except Exception as exc: recovery = ( - _recover_anthropic_history(self.state, messages, tools, exc) + _recover_anthropic_history( + self.state, messages, tools, exc, on_source_edit=self._on_anthropic_source_edit, + ) if self._requires_thinking_prefix() else None ) if recovery is None: @@ -651,7 +655,9 @@ async def _complete_with_choice( rejected_by = type(exc).__name__ messages, removed = recovery await self._prepare_request_budget(messages, tools, thinking=thinking, thinking_params=params) - messages = _remember_anthropic_history(self.state, messages, tools) + messages = _remember_anthropic_history( + self.state, messages, tools, on_source_edit=self._on_anthropic_source_edit, + ) max_output_tokens = self._request_output_limit( messages, tools, thinking=thinking, thinking_params=params, ) diff --git a/opencollab/application/session.py b/opencollab/application/session.py index 7fea29c4..f5c8759f 100644 --- a/opencollab/application/session.py +++ b/opencollab/application/session.py @@ -9,6 +9,7 @@ from enum import Enum from typing import TYPE_CHECKING, Any, Awaitable, Callable +from opencollab.application._anthropic_history import _anthropic_source_edit_start from opencollab.application.async_timeout import await_owned_operation from opencollab.application.autosave import AutoSaveSubscriber from opencollab.application.event_bus import EventBus @@ -25,7 +26,6 @@ ) from opencollab.application.session_run import SessionRunUseCase from opencollab.application.session_snapshot import ( - _anthropic_source_edit_start, _restore_active_turn_start, _restore_anthropic_history, _restore_pending_row, @@ -153,6 +153,7 @@ def __init__( self._auto_save_rewrite_from: int | None = 0 self._auto_save_rewrite_revision = 0 self._auto_save_lock = threading.Lock() + self.runner._on_anthropic_source_edit = self._mark_auto_save_rewrite self._auto_save_checkpoint_lock = threading.RLock() self._auto_save_pending_checkpoints: set[int] = set() self._auto_save_seen_result_hashes: set[str] = set() @@ -319,9 +320,7 @@ def messages(self) -> list[dict]: def messages(self, value: list[dict]) -> None: self.state.replace_messages(value) if hasattr(self, "_auto_save_rewrite_from"): - with self._auto_save_lock: - self._auto_save_rewrite_from = 0 - self._auto_save_rewrite_revision += 1 + self._mark_auto_save_rewrite(0) @property def used_tokens(self) -> int: @@ -1039,6 +1038,12 @@ def _prepare_auto_save(self) -> Callable[[], None] | None: with self._auto_save_lock: return self._prepare_journal_auto_save(path) + def _mark_auto_save_rewrite(self, start: int) -> None: + with self._auto_save_lock: + previous = self._auto_save_rewrite_from + self._auto_save_rewrite_from = start if previous is None else min(previous, start) + self._auto_save_rewrite_revision += 1 + def _reserve_auto_save_sequence(self) -> int: self._auto_save_reserved_sequence = max( self._auto_save_reserved_sequence, self._auto_save_sequence, diff --git a/opencollab/application/session_run.py b/opencollab/application/session_run.py index 49bf68d9..171b4d7e 100644 --- a/opencollab/application/session_run.py +++ b/opencollab/application/session_run.py @@ -186,6 +186,7 @@ def __init__( # The session supplies its owned persistence queue after adopting this runner. self.late_provider_usage_checkpoint: Callable[[], object] | None = None self._on_completion_tokens: Callable[[int], None] | None = None + self._on_anthropic_source_edit: Callable[[int], None] | None = None def reset_runtime_for_user_turn(self) -> None: """Restore agent capabilities narrowed by the previous turn.""" diff --git a/opencollab/application/session_snapshot.py b/opencollab/application/session_snapshot.py index e1d184e4..6835b1c6 100644 --- a/opencollab/application/session_snapshot.py +++ b/opencollab/application/session_snapshot.py @@ -6,7 +6,7 @@ import math from typing import Any -from opencollab.application._anthropic_history import _edited_anthropic_history +from opencollab.application._anthropic_history import _anthropic_source_edit_start, _edited_anthropic_history from opencollab.domain.pending import PendingRow, RowKind, RowStatus from opencollab.domain.session import SessionPhase, SessionState @@ -16,16 +16,6 @@ {"cancelled", "budget_exceeded", "step_limit_exceeded", "context_overflow"} ) -def _anthropic_source_edit_start(state: SessionState) -> int | None: - history = state.anthropic_history - source = history.get("source_messages") if history is not None else None - if source is None or state.messages[:len(source)] == source: - return None - for index, (old, current) in enumerate(zip(source, state.messages)): - if old != current: - return index - return len(state.messages) - def _serialize_anthropic_history(state: SessionState) -> dict | None: history = state.anthropic_history if history is None: diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index 5361673b..b6dea7dd 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import copy import json @@ -452,6 +453,70 @@ def test_restored_unchanged_prefix_preserves_projection_and_tracks_future_nested assert state.anthropic_history["source_messages"][1]["content"][0]["text"] == "Old requirement" +@pytest.mark.parametrize("checkpoint", [False, True], ids=["journal", "checkpoint"]) +async def test_first_tool_step_autosaves_an_edit_already_used_in_the_request(monkeypatch, tmp_path, checkpoint): + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + peer = _BoundThinkingProvider(reads=0) + held_request, resume = asyncio.Event(), asyncio.Event() + hold_after_read = False + + async def provider(request): + if hold_after_read and len(peer.requests) == peer.reads: + held_request.set() + await resume.wait() + return peer(request) + + install_sdk_transport(monkeypatch, "anthropic", provider) + path = tmp_path / "session.json" + + async with _client() as client: + environment = LocalEnvironment(str(tmp_path)) + session = build_session(agent=_agent(), llm=client, env=environment, + auto_save_path=str(path), max_budget_tokens=100_000, max_steps=20) + running = restored = None + try: + for index in range(3): + await session.add_user_message(f"Choose red for turn {index}") + assert await session.run_loop() == "done" + await session.add_user_message("Read the current evidence") + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + until_checkpoint = session._next_auto_save_checkpoint - session._auto_save_sequence + extra_saves = until_checkpoint - 1 if checkpoint else int(until_checkpoint == 1) + for _ in range(extra_saves): + await session.enqueue_auto_save() + saved = session.store.load_snapshot(str(path), session.agent.system_prompt) + assert saved["messages"][1]["content"] == "Choose red for turn 0" + + session.messages[1]["content"] = "Choose green for turn 0" + peer.reads = len(peer.requests) + 1 + hold_after_read = True + running = asyncio.create_task(session.run_loop()) + await asyncio.wait_for(held_request.wait(), 5) + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + assert not running.done() + assert "Choose green for turn 0" in json.dumps(peer.requests[-1]) + assert not session.persistence_errors + + saved = session.store.load_snapshot(str(path), session.agent.system_prompt) + base = json.loads(path.read_text()) + assert (base["_autosave_sequence"] == saved["_autosave_sequence"]) is checkpoint + restored = load_session(str(path), agent=_agent(), llm=client, env=environment) + assert restored.messages[1]["content"] == "Choose green for turn 0" + assert restored.state.anthropic_history["messages"][1]["content"] == "Choose green for turn 0" + assert sum(message["role"] == "tool" for message in restored.messages) == 1 + + resume.set() + assert await running == "done" + finally: + resume.set() + if running is not None: + await asyncio.gather(running, return_exceptions=True) + if restored is not None: + await restored.aclose() + await session.aclose() + await environment.cleanup() + + @pytest.mark.parametrize("source", ["invalid", [], [{}, {}], [False]]) def test_restore_rejects_malformed_optional_source_prefix(source): message = {"role": "user", "content": "Request"} From 188d99289387064246ce8ee066246d54883ad7ec Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 11:53:28 +0800 Subject: [PATCH 34/82] fix(files): resolve remote search and workflow paths in their backend --- opencollab/adapters/_env_docker.py | 11 +++ opencollab/adapters/_env_vm.py | 4 + opencollab/adapters/_vm_remote.py | 3 + opencollab/adapters/tools/_paths.py | 21 ++++- opencollab/adapters/tools/fs.py | 10 +- opencollab/bootstrap/authored_workflow.py | 2 +- .../bootstrap/authored_workflow_bridge.py | 8 +- opencollab/bootstrap/runtime_context.py | 5 +- .../test_member_authored_workflow.py | 36 ++++++++ .../interfaces/test_sdk_file_path_identity.py | 13 ++- tests/tools/test_remote_search_paths.py | 91 +++++++++++++++++++ 11 files changed, 185 insertions(+), 19 deletions(-) create mode 100644 tests/tools/test_remote_search_paths.py diff --git a/opencollab/adapters/_env_docker.py b/opencollab/adapters/_env_docker.py index 2d3559b3..ca53c580 100644 --- a/opencollab/adapters/_env_docker.py +++ b/opencollab/adapters/_env_docker.py @@ -211,6 +211,7 @@ class DockerEnvironment(Environment): """ process_isolated = True + remote_filesystem = True def __init__( self, @@ -954,6 +955,16 @@ async def _contained_file_path(self, path: str) -> str: raise PermissionError(f"container file path escapes workspace: {path}") return referent + async def resolve_search_path(self, path: str) -> str: + if not isinstance(path, str) or not path or "\0" in path: + raise ValueError("container search path must be non-empty text without NUL bytes") + target = await self._symlink_referent(posixpath.join(self.workspace, path)) + if target is None: + raise FileNotFoundError(f"cannot resolve container search path: {path}") + if not _within_directory(await self._physical_workspace(), target): + raise PermissionError(f"container search path escapes workspace: {path}") + return target + @staticmethod def _validate_container_path(path: str) -> str: if not isinstance(path, str) or not path or "\0" in path: diff --git a/opencollab/adapters/_env_vm.py b/opencollab/adapters/_env_vm.py index 2ddbec44..21d46d3b 100644 --- a/opencollab/adapters/_env_vm.py +++ b/opencollab/adapters/_env_vm.py @@ -57,6 +57,7 @@ class VMEnvironment(Environment): """ process_isolated = True + remote_filesystem = True file_write_lock_scope = "workspace" def __init__(self, connection: VMConnection, workspace: str, *, state_directory: str, run_id: str | None = None): @@ -282,6 +283,9 @@ async def recover_operation(self, operation_id: str) -> Any: async def read_file(self, path: str) -> str: return await self._operate("read", {"path": path}) + async def resolve_search_path(self, path: str) -> str: + return await self._operate("resolve_search_path", {"path": path}) + async def read_text_range(self, path: str, *, offset: int, limit: int, max_chars: int) -> TextFileRange: return TextFileRange(**await self._operate("range", {"path": path, "offset": offset, "limit": limit, "max_chars": max_chars})) diff --git a/opencollab/adapters/_vm_remote.py b/opencollab/adapters/_vm_remote.py index e15b1d11..21f3b2d2 100644 --- a/opencollab/adapters/_vm_remote.py +++ b/opencollab/adapters/_vm_remote.py @@ -34,6 +34,7 @@ unlink_regular_file_durable_at, write_regular_bytes_atomic_at, ) +from opencollab.adapters.safety import SandboxInterceptor from opencollab.adapters.vm_config import validate_vm_path _MAX_MESSAGE = 32 * 1024 * 1024 @@ -205,6 +206,8 @@ async def _perform(request: dict[str, Any], directory: Path) -> Any: return result if action == "read": return await env.read_file(args["path"]) + if action == "resolve_search_path": + return SandboxInterceptor(env.workspace).check_path(args["path"]) if action == "range": return asdict(await env.read_text_range(**args)) if action == "write": diff --git a/opencollab/adapters/tools/_paths.py b/opencollab/adapters/tools/_paths.py index e75e54e6..bb3c9785 100644 --- a/opencollab/adapters/tools/_paths.py +++ b/opencollab/adapters/tools/_paths.py @@ -8,14 +8,18 @@ from opencollab.application.tool_execution import ToolRuntime +def _checked_remote_path(runtime: ToolRuntime, path: str) -> str | None: + check = getattr(runtime.safety_policy, "check_remote_file_path", None) + return check(path) if callable(check) else None + + def checked_path(runtime: ToolRuntime, path: str, *, native_file: bool = False) -> str: """Apply the runtime policy and preserve remote components for native file operations.""" - policy = runtime.safety_policy - remote_check = getattr(policy, "check_remote_file_path", None) - if native_file and callable(remote_check): - remote_path = remote_check(path) + if native_file: + remote_path = _checked_remote_path(runtime, path) if remote_path is not None: return remote_path + policy = runtime.safety_policy checked = policy.check_path(path) if policy else path environment = runtime.environment workspace = getattr(environment, "workspace", None) @@ -25,3 +29,12 @@ def checked_path(runtime: ToolRuntime, path: str, *, native_file: bool = False) if os.path.isabs(checked) and os.path.commonpath((host_root, checked)) == host_root: checked = posixpath.join(workspace, os.path.relpath(checked, host_root)) return checked + + +async def checked_search_path(runtime: ToolRuntime, path: str) -> str: + resolve = getattr(runtime.environment, "resolve_search_path", None) + if callable(resolve): + remote_path = _checked_remote_path(runtime, path) + if remote_path is not None: + return await resolve(remote_path) + return checked_path(runtime, path) diff --git a/opencollab/adapters/tools/fs.py b/opencollab/adapters/tools/fs.py index 509f2bac..b32754a8 100644 --- a/opencollab/adapters/tools/fs.py +++ b/opencollab/adapters/tools/fs.py @@ -11,7 +11,7 @@ from opencollab.adapters._safe_file_common import _normalize_newlines, _split_lines from opencollab.adapters.tools._output import require_positive_int, truncate from opencollab.adapters.tools._parameters import integer_parameter -from opencollab.adapters.tools._paths import checked_path +from opencollab.adapters.tools._paths import checked_path, checked_search_path from opencollab.adapters.tools.apply_patch_engine import ( _MIXED_NEWLINES, _detect_newline_style, @@ -427,15 +427,15 @@ async def execute_with_runtime( if env is None: return "Error: no execution environment available." - try: - search_path = checked_path(runtime, search_path) - except PermissionError as exc: - return f"Error: {exc}" try: max_results = int(raw_max_results) except (TypeError, ValueError): return "Error: max_results must be an integer." max_results = max(1, min(max_results, 500)) + try: + search_path = await checked_search_path(runtime, search_path) + except OSError as exc: + return f"Error: {exc}" quoted_pattern = shlex.quote(pattern) quoted_search_path = shlex.quote(search_path) diff --git a/opencollab/bootstrap/authored_workflow.py b/opencollab/bootstrap/authored_workflow.py index 5a6941e3..fae8e606 100644 --- a/opencollab/bootstrap/authored_workflow.py +++ b/opencollab/bootstrap/authored_workflow.py @@ -241,7 +241,7 @@ async def _launch(self, params: dict[str, Any], runtime: ToolRuntime) -> _Child: source = params.get("source") if not isinstance(source, str) or not source or "\x00" in source: raise ValueError("source must name a Python file in the workspace") - source = checked_path(runtime, source) + source = checked_path(runtime, source, native_file=True) await environment.read_file(source) budget = params.get("budget") if budget is not None and (isinstance(budget, bool) or not isinstance(budget, int) or budget < 1): diff --git a/opencollab/bootstrap/authored_workflow_bridge.py b/opencollab/bootstrap/authored_workflow_bridge.py index f9dbb3d4..5743ff70 100644 --- a/opencollab/bootstrap/authored_workflow_bridge.py +++ b/opencollab/bootstrap/authored_workflow_bridge.py @@ -29,7 +29,8 @@ _METADATA = {"context_window", "estimate_request_tokens", "estimate_context_tokens", "minimum_output_tokens"} _COMPLETION = {"messages", "tools", "temperature", "thinking", "thinking_params", "reasoning_effort", "tool_choice", "top_p", "max_output_tokens", "response_session_id"} -_ENVIRONMENT = {"exec_cmd", "read_file", "read_text_range", "write_file", "write_temp_file", "remove_file"} +_ENVIRONMENT = {"exec_cmd", "read_file", "read_text_range", "write_file", "write_temp_file", "remove_file", + "resolve_search_path"} def _write_json(path: Path, value: Any) -> None: @@ -341,6 +342,7 @@ def admitted() -> None: self._next_environment += 1 self._environments[self._next_environment] = acquired return {"handle": self._next_environment, "workspace": acquired.workspace, + "remote_filesystem": getattr(acquired, "remote_filesystem", False) is True, "process_isolated": bool(getattr(acquired, "process_isolated", False))} if action == "release": if arguments or handle == 0: @@ -491,6 +493,7 @@ def __init__(self, client: BridgeClient, options: dict[str, Any]) -> None: super().__init__() self.client, self.handle = client, options["handle"] self.workspace, self.process_isolated = options["workspace"], options["process_isolated"] + self.remote_filesystem = options.get("remote_filesystem") is True self._released = False self._edit: ContextVar[str | None] = ContextVar("authored_file_edit", default=None) @@ -527,6 +530,9 @@ async def exec_cmd(self, cmd: str, timeout: float = 120.0, *, input_bytes: bytes async def read_file(self, path: str) -> str: return await self._call("read_file", {"path": path}) + async def resolve_search_path(self, path: str) -> str: + return await self._call("resolve_search_path", {"path": path}) + async def read_text_range(self, path: str, **kwargs: Any) -> TextFileRange: return TextFileRange(**await self._call("read_text_range", {"path": path, **kwargs})) diff --git a/opencollab/bootstrap/runtime_context.py b/opencollab/bootstrap/runtime_context.py index dd786125..3ed1ff9a 100644 --- a/opencollab/bootstrap/runtime_context.py +++ b/opencollab/bootstrap/runtime_context.py @@ -12,7 +12,7 @@ import uuid from dataclasses import dataclass -from opencollab.adapters.env import DockerEnvironment, Environment, VMEnvironment +from opencollab.adapters.env import Environment from opencollab.adapters.safety import SandboxInterceptor from opencollab.adapters.trace import Tracer from opencollab.application.ports import ( @@ -61,5 +61,4 @@ def build_workspace_safety_policy(env: Environment) -> SafetyPolicyPort | None: """A sandbox interceptor scoped to ``env``'s workspace, or None without one.""" if env is None or not getattr(env, "workspace", None): return None - backend = getattr(env, "_delegate", env) - return SandboxInterceptor(env.workspace, remote_filesystem=isinstance(backend, (DockerEnvironment, VMEnvironment))) + return SandboxInterceptor(env.workspace, remote_filesystem=getattr(env, "remote_filesystem", False) is True) diff --git a/tests/interfaces/test_member_authored_workflow.py b/tests/interfaces/test_member_authored_workflow.py index c74f0b57..1f0109fe 100644 --- a/tests/interfaces/test_member_authored_workflow.py +++ b/tests/interfaces/test_member_authored_workflow.py @@ -3,14 +3,18 @@ import asyncio import json import os +import shutil import sys import threading from contextlib import contextmanager +from dataclasses import replace from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import pytest +from opencollab.adapters._env_docker import DockerEnvironment +from opencollab.adapters._env_process import ProcessResult from opencollab.adapters.llm.types import LLMResponse, Usage from opencollab.adapters.tools.bash import BashTool from opencollab.application.event_bus import EventBus @@ -19,11 +23,13 @@ from opencollab.bootstrap import container from opencollab.bootstrap.authored_workflow import AuthoredWorkflowTool from opencollab.bootstrap.context_builder import SpawnConfig +from opencollab.bootstrap.runtime_context import build_workspace_safety_policy from opencollab.bootstrap.session_factory import build_session from opencollab.bootstrap.team_workflow import TeamWorkflowTool from opencollab.bootstrap.workflow_runtime import run_workflow from opencollab.domain.agent import Agent from opencollab.sdk import OpenCollab +from tests.support.docker_native_edits_support import RemoteDockerTransport from tests.support.environment_test_support import local_environment_factory as local_environment_factory from tests.support.paths import REPO_ROOT @@ -203,6 +209,36 @@ async def wait(): await asyncio.wait_for(wait(), 10) +async def test_docker_authored_source_resolves_parent_in_the_runner(tmp_path): + class RunnerTransport(RemoteDockerTransport): + async def __call__(self, *argv, input_bytes=None, **kwargs): + if argv[0] == "cp": + source = argv[-2].split(":", 1)[1].replace(self.remote, str(self.workspace)) + shutil.copyfile(source, argv[-1]) + return ProcessResult(0, b"", b"") + if input_bytes is not None and "opencollab.bootstrap.authored_workflow_runner" in argv[-1]: + input_bytes = input_bytes.replace(self.remote.encode(), str(self.workspace).encode()) + return await super().__call__(*argv, input_bytes=input_bytes, **kwargs) + + (tmp_path / "nested").mkdir() + (tmp_path / "authored.py").write_text('async def main(ctx, args):\n return "WORKFLOW_MARKER"\n') + transport = RunnerTransport(tmp_path) + environment = DockerEnvironment( + workspace=transport.remote, exec_workdir=transport.remote, container_id="b" * 64, + ) + transport.attach(environment) + parent, tool, runtime = _manager(environment) + runtime = replace(runtime, safety_policy=build_workspace_safety_policy(environment)) + try: + child = await _launch(tool, runtime, source="nested/../authored.py") + result = json.loads(await tool.execute_with_runtime({"action": "wait", "id": child["id"]}, runtime)) + assert result["status"] == "completed", result + assert result["result"]["output"] == "WORKFLOW_MARKER" + finally: + await parent.aclose() + await environment.cleanup() + + @pytest.mark.parametrize("stop", ["explicit", "parent-close", "cancel-wait"]) async def test_stop_cleans_child_before_returning(local_environment_factory, tmp_path, stop): (tmp_path / "authored.py").write_text('''import asyncio diff --git a/tests/interfaces/test_sdk_file_path_identity.py b/tests/interfaces/test_sdk_file_path_identity.py index a7797b86..12ffae44 100644 --- a/tests/interfaces/test_sdk_file_path_identity.py +++ b/tests/interfaces/test_sdk_file_path_identity.py @@ -39,7 +39,7 @@ async def complete(self, messages, **kwargs): @pytest.mark.parametrize("absolute", [False, True]) -@pytest.mark.parametrize("tool", ["file_read", "file_write", "apply_patch"]) +@pytest.mark.parametrize("tool", ["file_read", "file_write", "apply_patch", "grep"]) @pytest.mark.parametrize(("backend", "parent"), [("local", "alias"), ("docker", "alias"), ("docker", "sub")]) async def test_sdk_file_tools_match_shell_through_parent(tmp_path, absolute, tool, backend, parent): (tmp_path / "actual/child").mkdir(parents=True) @@ -48,7 +48,8 @@ async def test_sdk_file_tools_match_shell_through_parent(tmp_path, absolute, too target = tmp_path / ("actual/module.py" if parent == "alias" else "module.py") unrelated = tmp_path / ("module.py" if parent == "alias" else "actual/module.py") target.write_text("VALUE = 1\n") - unrelated.write_text("UNRELATED = 7\n" if tool == "file_read" else "VALUE = 1\n") + reading = tool in {"file_read", "grep"} + unrelated.write_text("UNRELATED = 7\n" if reading else "VALUE = 1\n") if backend == "docker": transport = RemoteDockerTransport(tmp_path) environment = DockerEnvironment( @@ -68,6 +69,8 @@ async def test_sdk_file_tools_match_shell_through_parent(tmp_path, absolute, too mode="line_replace", start_line=1, end_line=1, expected_str="VALUE = 1", new_str="VALUE = 42", ) + elif tool == "grep": + arguments["pattern"] = "VALUE" model = _PathModel(tool, arguments) outputs = [] @@ -90,12 +93,12 @@ async def flow(ctx, _): assert result.ok, result.reason assert result.output == "completed" assert len(outputs) == 1 and not outputs[0].startswith("Error") - if tool == "file_read": - assert "1\tVALUE = 1" in outputs[0] + if reading: + assert ("1\tVALUE = 1" if tool == "file_read" else "1:VALUE = 1") in outputs[0] assert "UNRELATED" not in outputs[0] else: assert target.read_text() == "VALUE = 42\n" - assert unrelated.read_text() == ("UNRELATED = 7\n" if tool == "file_read" else "VALUE = 1\n") + assert unrelated.read_text() == ("UNRELATED = 7\n" if reading else "VALUE = 1\n") shell = await environment.exec_cmd(f"cat {raw}") assert shell.stdout == target.read_text() finally: diff --git a/tests/tools/test_remote_search_paths.py b/tests/tools/test_remote_search_paths.py new file mode 100644 index 00000000..5abe0ee0 --- /dev/null +++ b/tests/tools/test_remote_search_paths.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from dataclasses import asdict + +from opencollab.adapters import _vm_remote +from opencollab.adapters.env import DockerEnvironment, LocalEnvironment, VMEnvironment +from opencollab.adapters.session_environments import SessionEnvironmentPool +from opencollab.adapters.tools.fs import GrepTool +from opencollab.adapters.vm_config import VMConnection +from opencollab.application.tool_execution import ToolRuntime +from opencollab.bootstrap.authored_workflow_bridge import AuthoredWorkflowBridge, BridgeEnvironment +from opencollab.bootstrap.runtime_context import build_workspace_safety_policy +from tests.support.docker_native_edits_support import RemoteDockerTransport + + +async def test_vm_search_path_is_resolved_in_the_guest(tmp_path, monkeypatch): + workspace = tmp_path / "workspace" + (workspace / "nested").mkdir(parents=True) + (workspace / "module.py").write_text("search marker\n") + guest = LocalEnvironment(str(workspace)) + vm = VMEnvironment( + VMConnection("guest", "worker"), str(tmp_path / "guest"), state_directory=str(tmp_path / "state"), + ) + info = workspace.stat() + identity = {"device": info.st_dev, "inode": info.st_ino} + vm._identity = identity + monkeypatch.setattr(_vm_remote, "_identity", lambda path: identity) + calls = [] + + async def rpc(payload): + request = payload["request"] + action, args = request["action"], request["args"] + calls.append(action) + if action == "resolve_search_path": + path = args["path"].replace(vm.workspace, str(workspace), 1) + value = await _vm_remote._perform({ + **request, "workspace": str(workspace), "args": {"path": path}, + }, tmp_path) + value = value.replace(str(workspace), vm.workspace, 1) + else: + assert action == "exec" + command = args["cmd"].replace(vm.workspace, str(workspace), 1) + value = {**asdict(await guest.exec_cmd(command)), "stdout_bytes": None} + return {"status": "completed", "value": value} + + monkeypatch.setattr(vm, "_rpc", rpc) + runtime = ToolRuntime(vm, build_workspace_safety_policy(vm), None) + try: + result = await GrepTool().execute_with_runtime( + {"path": "nested/../module.py", "pattern": "search marker"}, runtime, + ) + assert result == "1:search marker" + assert calls[:2] == ["resolve_search_path", "exec"] + finally: + await guest.cleanup() + await vm.cleanup() + + +async def test_authored_bridge_preserves_remote_search_resolution(tmp_path): + (tmp_path / "nested").mkdir() + (tmp_path / "module.py").write_text("search marker\n") + transport = RemoteDockerTransport(tmp_path) + environment = DockerEnvironment( + workspace=transport.remote, exec_workdir=transport.remote, container_id="b" * 64, + ) + transport.attach(environment) + pool = SessionEnvironmentPool(environment.workspace, isolation="shared", base_environment=environment) + bridge = AuthoredWorkflowBridge( + environment, environment.workspace + "/artifacts", pool, {}, ToolRuntime(environment, None, None), + allow_unisolated_shell=True, + ) + + class BridgeConnection: + async def call(self, action, arguments, handle, *, edit=None): + return await bridge._dispatch("local-search", { + "action": action, "arguments": arguments, "environment": handle, "edit": edit, + }) + + client = BridgeConnection() + handle = await client.call("acquire", {"strategy": "shared", "label": "child"}, 0) + proxy = BridgeEnvironment(client, handle) + try: + runtime = ToolRuntime(proxy, build_workspace_safety_policy(proxy), None) + result = await GrepTool().execute_with_runtime( + {"path": "nested/..", "pattern": "search marker"}, runtime, + ) + assert "module.py:1:search marker" in result + finally: + await proxy.cleanup() + await bridge.aclose() + await environment.cleanup() From 6be9001309cc3d554d574dd6256e352444f0075e Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 14:05:17 +0800 Subject: [PATCH 35/82] fix(session): reconcile journal rewrites with persisted history --- opencollab/application/_anthropic_history.py | 25 ++---- .../application/_session_run_completion.py | 12 +-- opencollab/application/session.py | 41 +++++++--- opencollab/application/session_run.py | 1 - opencollab/application/session_snapshot.py | 4 +- .../test_anthropic_preserved_history.py | 81 +++++++++++++++++++ tests/runtime/test_exported_session_resume.py | 2 +- 7 files changed, 123 insertions(+), 43 deletions(-) diff --git a/opencollab/application/_anthropic_history.py b/opencollab/application/_anthropic_history.py index be924483..9e6ab2d1 100644 --- a/opencollab/application/_anthropic_history.py +++ b/opencollab/application/_anthropic_history.py @@ -3,7 +3,6 @@ from __future__ import annotations import copy -from collections.abc import Callable from typing import Any from opencollab.domain.session import SessionState @@ -35,37 +34,26 @@ def _edited_anthropic_history(before: list[dict], after: list[dict]) -> list[dic return _strip_anthropic_thinking(after, index) return after -def _anthropic_source_edit_start(state: SessionState) -> int | None: +def _anthropic_source_changed(state: SessionState) -> bool: history = state.anthropic_history source = history.get("source_messages") if history is not None else None - if source is None or state.messages[:len(source)] == source: - return None - for index, (old, current) in enumerate(zip(source, state.messages)): - if old != current: - return index - return len(state.messages) + return source is not None and state.messages[:len(source)] != source -def _anthropic_history_input( - state: SessionState, *, on_source_edit: Callable[[int], None] | None = None, -) -> list[dict]: +def _anthropic_history_input(state: SessionState) -> list[dict]: """Reuse the native view until its original transcript prefix changes.""" history = state.anthropic_history if history is None: return state.messages - edit_start = _anthropic_source_edit_start(state) - if edit_start is not None: - if on_source_edit is not None: - on_source_edit(edit_start) + if _anthropic_source_changed(state): return _edited_anthropic_history(history["source_messages"], state.messages) return copy.deepcopy([*history["messages"], *state.messages[history["source_count"]:]]) def _remember_anthropic_history( state: SessionState, messages: list[dict], tools: list[dict] | None, - *, on_source_edit: Callable[[int], None] | None = None, ) -> list[dict]: """Persist edits so removed blocks and compacted content stay removed.""" history = state.anthropic_history - messages = _edited_anthropic_history(_anthropic_history_input(state, on_source_edit=on_source_edit), messages) + messages = _edited_anthropic_history(_anthropic_history_input(state), messages) if history is not None: messages = _edited_anthropic_history(history["messages"], messages) if history["tools"] != tools: @@ -78,7 +66,6 @@ def _remember_anthropic_history( def _recover_anthropic_history( state: SessionState, messages: list[dict], tools: list[dict] | None, error: Exception, - *, on_source_edit: Callable[[int], None] | None = None, ) -> tuple[list[dict], int] | None: """Retire replayed thinking after an explicit provider prefix rejection.""" status = getattr(error, "status_code", None) @@ -101,4 +88,4 @@ def _recover_anthropic_history( ) if not removed: return None - return _remember_anthropic_history(state, recovered, tools, on_source_edit=on_source_edit), removed + return _remember_anthropic_history(state, recovered, tools), removed diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index 9341bfd7..a6f0659c 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -623,9 +623,7 @@ async def _complete_with_choice( params = None await self._prepare_request_budget(messages, tools, thinking=thinking, thinking_params=params) if self._requires_thinking_prefix(): - messages = _remember_anthropic_history( - self.state, messages, tools, on_source_edit=self._on_anthropic_source_edit, - ) + messages = _remember_anthropic_history(self.state, messages, tools) max_output_tokens = self._request_output_limit( messages, tools, thinking=thinking, thinking_params=params, ) @@ -645,9 +643,7 @@ async def _complete_with_choice( ) except Exception as exc: recovery = ( - _recover_anthropic_history( - self.state, messages, tools, exc, on_source_edit=self._on_anthropic_source_edit, - ) + _recover_anthropic_history(self.state, messages, tools, exc) if self._requires_thinking_prefix() else None ) if recovery is None: @@ -655,9 +651,7 @@ async def _complete_with_choice( rejected_by = type(exc).__name__ messages, removed = recovery await self._prepare_request_budget(messages, tools, thinking=thinking, thinking_params=params) - messages = _remember_anthropic_history( - self.state, messages, tools, on_source_edit=self._on_anthropic_source_edit, - ) + messages = _remember_anthropic_history(self.state, messages, tools) max_output_tokens = self._request_output_limit( messages, tools, thinking=thinking, thinking_params=params, ) diff --git a/opencollab/application/session.py b/opencollab/application/session.py index f5c8759f..1afb3805 100644 --- a/opencollab/application/session.py +++ b/opencollab/application/session.py @@ -9,7 +9,6 @@ from enum import Enum from typing import TYPE_CHECKING, Any, Awaitable, Callable -from opencollab.application._anthropic_history import _anthropic_source_edit_start from opencollab.application.async_timeout import await_owned_operation from opencollab.application.autosave import AutoSaveSubscriber from opencollab.application.event_bus import EventBus @@ -150,10 +149,10 @@ def __init__( self._auto_save_sequence = 0 self._auto_save_reserved_sequence = 0 self._auto_save_message_count = 0 + self._auto_save_messages: list[dict] | None = None self._auto_save_rewrite_from: int | None = 0 self._auto_save_rewrite_revision = 0 self._auto_save_lock = threading.Lock() - self.runner._on_anthropic_source_edit = self._mark_auto_save_rewrite self._auto_save_checkpoint_lock = threading.RLock() self._auto_save_pending_checkpoints: set[int] = set() self._auto_save_seen_result_hashes: set[str] = set() @@ -755,6 +754,12 @@ def _restore_auto_save_tracking(self, snapshot: dict[str, Any]) -> None: len(stored_messages) if isinstance(stored_messages, list) else 0, len(self.state.messages), ) + self._auto_save_messages = ( + copy.deepcopy(stored_messages) + if isinstance(stored_messages, list) and self._auto_save_path + and isinstance(self.store, JournalSnapshotStorePort) and self._tracks_native_history() + else None + ) self._auto_save_rewrite_from = None self._auto_save_seen_result_hashes = set( self.state.turn.seen_result_hashes @@ -937,6 +942,7 @@ def save(self, path: str) -> None: # skip its sequence and its completion cannot rewind our cursor. with self._auto_save_lock: messages, meta = self._snapshot_for_save() + tracked_messages = messages if self._tracks_native_history() else None sequence = self._reserve_auto_save_sequence() rewrite_revision = self._auto_save_rewrite_revision self._auto_save_pending_checkpoints.add(sequence) @@ -949,6 +955,7 @@ def save(self, path: str) -> None: with self._auto_save_lock: self._auto_save_sequence = sequence self._auto_save_message_count = len(messages) + self._auto_save_messages = tracked_messages if self._auto_save_rewrite_revision == rewrite_revision: self._auto_save_rewrite_from = None self._auto_save_seen_result_hashes = set(meta["session_state"]["seen_result_hashes"]) @@ -1050,6 +1057,9 @@ def _reserve_auto_save_sequence(self) -> int: ) + 1 return self._auto_save_reserved_sequence + def _tracks_native_history(self) -> bool: + return self.state.anthropic_history is not None or self.runner._requires_thinking_prefix() + def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: preceding_write_pending = self._auto_save_reserved_sequence > self._auto_save_sequence sequence = self._reserve_auto_save_sequence() @@ -1065,10 +1075,19 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: rewrite_revision = self._auto_save_rewrite_revision if self._auto_save_rewrite_from is not None: replace_from = min(replace_from, self._auto_save_rewrite_from) - edited_from = _anthropic_source_edit_start(self.state) - if edited_from is not None: - replace_from = min(replace_from, edited_from) + track_messages = self._tracks_native_history() + stored_messages = self._auto_save_messages or [] + if track_messages: + # Native history references resolve against the last committed transcript. + replace_from = min(replace_from, len(stored_messages)) + if replace_from: + current_messages = self.state.enriched_messages() + for index in range(replace_from): + if current_messages[index] != stored_messages[index]: + replace_from = index + break messages, meta = self._snapshot_for_save(replace_from) + tracked_messages = [*stored_messages[:replace_from], *messages] if track_messages else None current_seen_hashes = set(self.state.turn.seen_result_hashes) seen_hashes_reset = preceding_write_pending or not self._auto_save_seen_result_hashes.issubset( current_seen_hashes @@ -1078,12 +1097,11 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: if seen_hashes_reset else current_seen_hashes - self._auto_save_seen_result_hashes ) - checkpoint_due = sequence >= self._next_auto_save_checkpoint - checkpoint_messages = ( - copy.deepcopy(self.state.enriched_messages()) - if checkpoint_due - else None - ) + checkpoint_messages = None + if sequence >= self._next_auto_save_checkpoint: + checkpoint_messages = tracked_messages + if checkpoint_messages is None: + checkpoint_messages = copy.deepcopy(self.state.enriched_messages()) def persist() -> None: self.store.append_snapshot_delta( @@ -1103,6 +1121,7 @@ def persist() -> None: # cannot cause the durable delta to be skipped on the next save. self._auto_save_sequence = sequence self._auto_save_message_count = message_count + self._auto_save_messages = tracked_messages if self._auto_save_rewrite_revision == rewrite_revision: self._auto_save_rewrite_from = None self._auto_save_seen_result_hashes = current_seen_hashes diff --git a/opencollab/application/session_run.py b/opencollab/application/session_run.py index 171b4d7e..49bf68d9 100644 --- a/opencollab/application/session_run.py +++ b/opencollab/application/session_run.py @@ -186,7 +186,6 @@ def __init__( # The session supplies its owned persistence queue after adopting this runner. self.late_provider_usage_checkpoint: Callable[[], object] | None = None self._on_completion_tokens: Callable[[int], None] | None = None - self._on_anthropic_source_edit: Callable[[int], None] | None = None def reset_runtime_for_user_turn(self) -> None: """Restore agent capabilities narrowed by the previous turn.""" diff --git a/opencollab/application/session_snapshot.py b/opencollab/application/session_snapshot.py index 6835b1c6..d35dccd9 100644 --- a/opencollab/application/session_snapshot.py +++ b/opencollab/application/session_snapshot.py @@ -6,7 +6,7 @@ import math from typing import Any -from opencollab.application._anthropic_history import _anthropic_source_edit_start, _edited_anthropic_history +from opencollab.application._anthropic_history import _anthropic_source_changed, _edited_anthropic_history from opencollab.domain.pending import PendingRow, RowKind, RowStatus from opencollab.domain.session import SessionPhase, SessionState @@ -23,7 +23,7 @@ def _serialize_anthropic_history(state: SessionState) -> dict | None: source = state.messages count = history["source_count"] view = history["messages"] - if _anthropic_source_edit_start(state) is not None: + if _anthropic_source_changed(state): # Retire the stale projection before restore binds it to the new transcript. view = _edited_anthropic_history(history["source_messages"], source) view = _edited_anthropic_history(history["messages"], view) diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index b6dea7dd..11c1501a 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -5,6 +5,7 @@ import asyncio import copy import json +import shutil import pytest @@ -19,6 +20,7 @@ from opencollab.bootstrap import build_session, load_session from opencollab.domain.agent import Agent from opencollab.domain.session import SessionState +from tests.runtime.test_manual_autosave_ordering import PausedAppendStore from tests.support.provider_sdk_http import http_response, install_sdk_transport _THINKING = {"thinking", "redacted_thinking"} @@ -517,6 +519,85 @@ async def provider(request): await environment.cleanup() +@pytest.mark.parametrize(("first_save", "revert"), [ + ("journal", True), ("manual", True), ("periodic", True), ("manual-after-freeze", True), + ("journal", False), +]) +async def test_native_history_delta_uses_the_last_saved_transcript(monkeypatch, tmp_path, first_save, revert): + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + peer = _BoundThinkingProvider(reads=0) + install_sdk_transport(monkeypatch, "anthropic", peer) + path = tmp_path / "session.json" + journal = path.with_suffix(".json.journal") + resumed_path = tmp_path / "resumed.json" + red, green = "Use the red option.", "Use the green option." + store = PausedAppendStore() + paused_append = first_save == "manual-after-freeze" + + async with _client() as client: + environment = LocalEnvironment(str(tmp_path)) + session = build_session( + agent=_agent(), llm=client, env=environment, store=store, auto_save_path=str(path), + seed_user_messages=[{"role": "user", "content": _blocks(red)}], + ) + restored = None + try: + for index in range(3): + await session.add_user_message(f"Warmup {index}") + assert await session.run_loop() == "done" + until_checkpoint = session._next_auto_save_checkpoint - session._auto_save_sequence + if first_save == "periodic": + for _ in range(until_checkpoint - 1): + await session.enqueue_auto_save() + elif until_checkpoint <= (3 if paused_append else 2): + for _ in range(until_checkpoint): + await session.enqueue_auto_save() + if paused_append: + store.pause_next = True + session.enqueue_auto_save() + assert await asyncio.to_thread(store.started.wait, 1) + + source = session.messages[1]["content"][0] + source["text"] = green + if first_save.startswith("manual"): + session.save(str(path)) + else: + await session.enqueue_auto_save() + saved_edit = session.store.load_snapshot(str(path), session.agent.system_prompt) + assert saved_edit["messages"][1]["content"] == _blocks(green) + base_sequence = json.loads(path.read_text())["_autosave_sequence"] + assert (base_sequence == saved_edit["_autosave_sequence"]) is (first_save != "journal") + + expected = red if revert else green + source["text"] = expected + await session.add_user_message("Continue with the current instructions.") + store.release.set() + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + delta = json.loads(journal.read_text().splitlines()[-1]) + assert delta["replace_from"] == (1 if revert else len(session.messages) - 2) + native_view = delta["meta"]["session_state"]["anthropic_history"] + assert native_view["messages"][1] == 1 + assert "source_messages" not in native_view + + shutil.copyfile(path, resumed_path) + shutil.copyfile(journal, resumed_path.with_suffix(".json.journal")) + restored = load_session(str(resumed_path), agent=_agent(), llm=client, env=environment, + auto_save_path=str(resumed_path)) + assert restored.messages == session.messages + assert restored.state.anthropic_history["messages"][1]["content"] == _blocks(expected) + assert await restored.run_loop() == "done" + assert await session.run_loop() == "done" + assert _plain_messages(peer.requests[-2]["messages"]) == _plain_messages(peer.requests[-1]["messages"]) + assert expected in json.dumps(peer.requests[-1]["messages"]) + assert not session.persistence_errors and not restored.persistence_errors + finally: + store.release.set() + if restored is not None: + await restored.aclose() + await session.aclose() + await environment.cleanup() + + @pytest.mark.parametrize("source", ["invalid", [], [{}, {}], [False]]) def test_restore_rejects_malformed_optional_source_prefix(source): message = {"role": "user", "content": "Request"} diff --git a/tests/runtime/test_exported_session_resume.py b/tests/runtime/test_exported_session_resume.py index cb657c54..0a5f527b 100644 --- a/tests/runtime/test_exported_session_resume.py +++ b/tests/runtime/test_exported_session_resume.py @@ -207,7 +207,7 @@ async def complete(self, messages, **kwargs): def autosave_tracking(session): return copy.deepcopy({name: getattr(session, name) for name in ( - "_auto_save_sequence", "_auto_save_reserved_sequence", "_auto_save_message_count", + "_auto_save_sequence", "_auto_save_reserved_sequence", "_auto_save_message_count", "_auto_save_messages", "_auto_save_rewrite_from", "_auto_save_rewrite_revision", "_auto_save_pending_checkpoints", "_auto_save_seen_result_hashes", "_next_auto_save_checkpoint", "_loop_checkpoint_results", "_loop_checkpoint_prefix", "_loop_checkpoint_step", From 227a576a7b99382c9458e5aa73823702f574171d Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 14:05:17 +0800 Subject: [PATCH 36/82] fix(llm): honor legacy completion wrapper signatures --- .../application/_session_run_completion.py | 117 ++++++++++-------- .../runtime/test_session_output_limit_http.py | 92 +++++++++++++- 2 files changed, 158 insertions(+), 51 deletions(-) diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index a6f0659c..21672275 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -59,6 +59,26 @@ logger = logging.getLogger(__name__) +def _completion_keyword_parameters(complete: Callable[..., Awaitable[Any]]) -> set[str] | None: + keywords: set[str] = set() + + def has_fixed_keywords(target: Callable[..., Any]) -> bool: + parameters = inspect.signature(target, follow_wrapped=False).parameters.values() + keywords.update( + parameter.name for parameter in parameters + if parameter.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY) + ) + return not any(parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters) + + # Follow variadic decorators through to the first fixed signature. + try: + target = inspect.unwrap(complete, stop=has_fixed_keywords) + if not hasattr(target, "__wrapped__") and not has_fixed_keywords(target): + return None + except (TypeError, ValueError): + pass + return keywords + def _selector_names_tool_choice(selector: Any) -> bool: """Return whether a structured error selector targets ``tool_choice``.""" values = selector if isinstance(selector, (list, tuple)) else [selector] @@ -627,8 +647,6 @@ async def _complete_with_choice( max_output_tokens = self._request_output_limit( messages, tools, thinking=thinking, thinking_params=params, ) - if max_output_tokens != DEFAULT_MAX_TOKENS_PER_STEP or self._completion_accepts_output_limit(): - extra["max_output_tokens"] = max_output_tokens reasoning_effort = getattr(self.agent, "reasoning_effort", None) if reasoning_effort is not None: extra["reasoning_effort"] = reasoning_effort @@ -639,6 +657,7 @@ async def _complete_with_choice( messages=messages, tools=tools, temperature=self.agent.temperature, + max_output_tokens=max_output_tokens, **extra, ) except Exception as exc: @@ -655,8 +674,6 @@ async def _complete_with_choice( max_output_tokens = self._request_output_limit( messages, tools, thinking=thinking, thinking_params=params, ) - if max_output_tokens != DEFAULT_MAX_TOKENS_PER_STEP or self._completion_accepts_output_limit(): - extra["max_output_tokens"] = max_output_tokens payload = { "reason": "prefix_binding_mismatch", "removed_blocks": removed, "retry": 1, "error_type": rejected_by, "status_code": 400, @@ -675,6 +692,7 @@ async def _complete_with_choice( messages=messages, tools=tools, temperature=self.agent.temperature, + max_output_tokens=max_output_tokens, **extra, ) @@ -763,23 +781,33 @@ def remaining_budget_tokens(self) -> int | None: return None return min(finite) - int(self.state.used_tokens) - sum(reserve() for reserve in self.budget_reservations) - def _completion_accepts_output_limit(self) -> bool: - """Include the default limit when the injected completion accepts it.""" - try: - parameters = inspect.signature(self.llm.complete).parameters.values() - except (TypeError, ValueError): - return False - return any( - parameter.kind == inspect.Parameter.VAR_KEYWORD - or ( - parameter.name == "max_output_tokens" - and parameter.kind in { - inspect.Parameter.POSITIONAL_OR_KEYWORD, - inspect.Parameter.KEYWORD_ONLY, - } - ) - for parameter in parameters - ) + def _add_completion_controls( + self, + complete: Callable[..., Awaitable[CompletionResponse]], + kwargs: dict[str, Any], + *, + max_output_tokens: int, + usage_accounting: _CompletionUsageAccounting, + revalidate: Callable[[], int], + ) -> asyncio.Event | None: + keywords = _completion_keyword_parameters(complete) + + def accepts(name: str) -> bool: + return keywords is None or name in keywords + + if max_output_tokens != DEFAULT_MAX_TOKENS_PER_STEP or accepts("max_output_tokens"): + kwargs["max_output_tokens"] = max_output_tokens + if getattr(self.llm, "supports_attempt_usage", False) and accepts("on_attempt_usage"): + kwargs["on_attempt_usage"] = usage_accounting.record_attempt + if getattr(self.llm, "supports_usage_settlement", False) and accepts("on_usage"): + kwargs["on_usage"] = usage_accounting.settle + if getattr(self.llm, "supports_request_revalidation", False) and accepts("before_request"): + kwargs["before_request"] = revalidate + if getattr(self.llm, "supports_request_admission", False) and accepts("on_admitted"): + admitted = asyncio.Event() + kwargs["on_admitted"] = admitted.set + return admitted + return None def _revalidate_request( self, messages: list[dict], tools: list[dict] | None, *, thinking: bool = False, @@ -805,25 +833,20 @@ async def _invoke_summary( ) if _summary_output_limit is not None: max_output_tokens = min(max_output_tokens, _summary_output_limit(messages)) - if max_output_tokens != DEFAULT_MAX_TOKENS_PER_STEP or self._completion_accepts_output_limit(): - kwargs["max_output_tokens"] = max_output_tokens start = time.monotonic() protected_call = self.state.wind_down_done usage_accounting = _CompletionUsageAccounting(self, protected_call=protected_call) - if getattr(self.llm, "supports_attempt_usage", False): - kwargs["on_attempt_usage"] = usage_accounting.record_attempt - admitted = asyncio.Event() if getattr(self.llm, "supports_request_admission", False) else None - if admitted is not None: - kwargs["on_admitted"] = admitted.set - if getattr(self.llm, "supports_usage_settlement", False): - kwargs["on_usage"] = usage_accounting.settle - if getattr(self.llm, "supports_request_revalidation", False): - def revalidate() -> int: - limit = self._revalidate_request( - messages, None, thinking=thinking, thinking_params=thinking_params, - ) - return min(limit, _summary_output_limit(messages)) if _summary_output_limit is not None else limit - kwargs["before_request"] = revalidate + + def revalidate() -> int: + limit = self._revalidate_request( + messages, None, thinking=thinking, thinking_params=thinking_params, + ) + return min(limit, _summary_output_limit(messages)) if _summary_output_limit is not None else limit + + admitted = self._add_completion_controls( + self.llm.complete, kwargs, max_output_tokens=max_output_tokens, + usage_accounting=usage_accounting, revalidate=revalidate, + ) abandoned = False if self.tracer: self.tracer.log_step(step_type="llm_call_started", payload={ @@ -881,7 +904,7 @@ async def complete_owned() -> CompletionResponse: self._mark_provider_task_draining(owner) raise - async def _invoke_llm(self, **kwargs: Any) -> CompletionResponse: + async def _invoke_llm(self, *, max_output_tokens: int, **kwargs: Any) -> CompletionResponse: """Bound one provider generation by ``_per_call_timeout``. ``None`` disables the ceiling. Expiry raises ``GenerationTimeoutError`` @@ -892,25 +915,21 @@ async def _invoke_llm(self, **kwargs: Any) -> CompletionResponse: self._raise_if_run_cancelled() protected_call = self.state.wind_down_done usage_accounting = _CompletionUsageAccounting(self, protected_call=protected_call) - if getattr(self.llm, "supports_attempt_usage", False): - kwargs["on_attempt_usage"] = usage_accounting.record_attempt - admitted = asyncio.Event() if getattr(self.llm, "supports_request_admission", False) else None - if admitted is not None: - kwargs["on_admitted"] = admitted.set - if getattr(self.llm, "supports_usage_settlement", False): - kwargs["on_usage"] = usage_accounting.settle - if getattr(self.llm, "supports_request_revalidation", False): - kwargs["before_request"] = lambda: self._revalidate_request( + complete = self.llm.complete + admitted = self._add_completion_controls( + complete, kwargs, max_output_tokens=max_output_tokens, usage_accounting=usage_accounting, + revalidate=lambda: self._revalidate_request( kwargs["messages"], kwargs.get("tools"), thinking=kwargs.get("thinking", False), thinking_params=kwargs.get("thinking_params"), - ) + ), + ) late_accounted = False async def complete_owned() -> CompletionResponse: nonlocal late_accounted try: response = await _complete_with_error_usage( - self, self.llm.complete, usage_accounting=usage_accounting, **kwargs, + self, complete, usage_accounting=usage_accounting, **kwargs, ) late_accounted = asyncio.current_task() in self._draining_provider_tasks self._account_completion_usage( diff --git a/tests/runtime/test_session_output_limit_http.py b/tests/runtime/test_session_output_limit_http.py index c18738d6..a3710323 100644 --- a/tests/runtime/test_session_output_limit_http.py +++ b/tests/runtime/test_session_output_limit_http.py @@ -3,17 +3,18 @@ from __future__ import annotations import json +from functools import wraps import pytest -from opencollab import OpenCollab +from opencollab import OpenCollab, RunControl from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.llm.types import LLMResponse, Usage from opencollab.bootstrap import container from opencollab.bootstrap.session_factory import build_session from opencollab.domain.agent import Agent from tests.support.environment_test_support import local_environment_factory as local_environment_factory -from tests.support.provider_sdk_http import completion_http_response, install_sdk_transport +from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport @pytest.fixture(autouse=True) @@ -176,3 +177,90 @@ async def complete(self, messages, tools=None, temperature=0.0): assert await summarizer.asummarize([{"role": "user", "content": "Keep this context"}]) == ( "Summary:\nPreserved context." ) + + +class _LegacyNativeClient(LLMClient): + @wraps(LLMClient.complete) + async def complete(self, messages, tools=None, temperature=0.0, *, max_output_tokens=None, on_admitted=None): + self.received_admission_callback = on_admitted is not None + return await super().complete( + messages, tools=tools, temperature=temperature, + max_output_tokens=max_output_tokens, on_admitted=on_admitted, + ) + + +class _SettlementWrapper(_LegacyNativeClient): + @wraps(_LegacyNativeClient.complete) + async def complete(self, *args, on_usage=None, **kwargs): + self.received_usage_callback = on_usage is not None + response = await super().complete(*args, **kwargs) + if on_usage is not None: + on_usage(response.usage) + return response + + +@pytest.mark.parametrize("failed", [False, True]) +@pytest.mark.parametrize("client_class", [_LegacyNativeClient, _SettlementWrapper]) +async def test_public_agent_charges_legacy_native_wrapper_usage_once(tmp_path, monkeypatch, failed, client_class): + requests = [] + events = [] + + async def handler(request): + requests.append(json.loads(request.content)) + response = completion_http_response(request) + if failed: + return http_response(request, 200, json=response.json() | {"choices": []}) + return response + + install_sdk_transport(monkeypatch, "openai", handler) + pool = tmp_path / "pool" + pool.mkdir() + async with client_class( + api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + request_pool=str(pool), request_capacity=1, + ) as client: + result = await OpenCollab(tmp_path).agent( + "Return ok", llm=client, tools=(), budget=20_000, trace=False, + run_control=RunControl(on_event=events.append), + ) + + assert client.received_admission_callback is True + if client_class is _SettlementWrapper: + assert client.received_usage_callback is True + assert len(requests) == 1 and requests[0]["max_tokens"] == 8192 + assert result.status == ("failed" if failed else "completed") + assert result.tokens == 4 + usage = [event.data for event in events if event.type == "usage"] + assert [event["total_tokens"] for event in usage] == [4] + assert usage[0]["error_type"] == ("IndexError" if failed else None) + + +async def test_summary_charges_legacy_native_wrapper_usage_once(tmp_path, monkeypatch, local_environment_factory): + requests = [] + + async def handler(request): + requests.append(json.loads(request.content)) + return completion_http_response(request, text="Preserved context.") + + install_sdk_transport(monkeypatch, "openai", handler) + async with _LegacyNativeClient( + api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + ) as client: + agent = Agent(name="legacy", system_prompt="system") + session = build_session( + agent=agent, llm=client, env=local_environment_factory(str(tmp_path)), max_budget_tokens=None, + ) + summarizer = container._build_summarizer( + agent, client, client, 600.0, None, completion_handler=session.runner._invoke_summary, + ) + try: + assert await summarizer.asummarize([{"role": "user", "content": "Keep this context"}]) == ( + "Summary:\nPreserved context." + ) + assert session.used_tokens == 4 + finally: + await session.aclose() + + assert len(requests) == 1 and requests[0]["max_tokens"] == 8192 From b02cdb58ce45a4d53d3c6b342a6d23f9ec428555 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 14:08:46 +0800 Subject: [PATCH 37/82] fix(workflow): preserve concurrent writes during candidate recovery --- opencollab/adapters/_candidate_adoption.py | 186 +++++++++++++++--- opencollab/adapters/_candidate_increment.py | 27 ++- .../test_candidate_adoption_recovery.py | 37 ++++ .../test_candidate_shared_mutations.py | 82 +++++++- 4 files changed, 295 insertions(+), 37 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index 435df3a1..2e84a08a 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -4,7 +4,7 @@ import posixpath import shlex -from collections.abc import AsyncIterator, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence from contextlib import AsyncExitStack, asynccontextmanager from typing import Any @@ -13,6 +13,117 @@ from opencollab.application.async_timeout import await_owned_operation from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError +_RECOVER_PATHS = r''' +present() { [ -e "$1" ] || [ -L "$1" ]; } +directory() { [ -d "$1" ] && [ ! -L "$1" ]; } +mode() { stat -c %a -- "$1" 2>/dev/null || stat -f %Lp -- "$1"; } + +same_entry() { + if [ -L "$1" ]; then + [ -L "$2" ] || return 1 + left=$(readlink "$1" && printf .) && right=$(readlink "$2" && printf .) && + [ "$left" = "$right" ] + elif [ -f "$1" ]; then + [ ! -L "$2" ] && [ -f "$2" ] || return 1 + left=$(mode "$1") && right=$(mode "$2") && [ "$left" = "$right" ] && cmp -s -- "$1" "$2" + else + ! present "$1" && ! present "$2" + fi +} + +conflict() { printf 'concurrent change at %s\n' "$1" >&2; return 2; } + +check_parents() { + if [ "$1" = "$workspace" ]; then directory "$workspace" || conflict "$workspace"; return; fi + parent=${1%/*} + while [ "$parent" != "$workspace" ] && [ -n "$parent" ]; do + directory "$parent" || { conflict "$parent"; return; } + parent=${parent%/*} + done + directory "$workspace" || conflict "$workspace" +} + +prepare_parent() { + if [ "$1" != "$workspace" ]; then check_parents "$1" || return $?; fi + if ! present "$1"; then + if directory "$5"; then conflict "$1"; return; fi + mkdir -m 700 -- "$1" && printf '700 %s\n' "$2" > "$3" + elif ! directory "$1"; then + conflict "$1" + else + observed=$(mode "$1") || return 1 + restore=$observed + if directory "$5"; then + projected=$(mode "$5") || return 1 + if [ "$observed" = "$projected" ]; then restore=$2; fi + fi + temporary=$observed + if [ "$4" = write ] && [ ! -w "$1" ] && [ -O "$1" ]; then + chmod -- u+w "$1" && temporary=$(mode "$1") || return 1 + fi + if [ "$temporary" != "$restore" ]; then printf '%s %s\n' "$temporary" "$restore" > "$3"; fi + fi +} + +reset_parent_mode() { + if [ -f "$2" ]; then + check_parents "$1" && directory "$1" || { conflict "$1"; return; } + read -r temporary observed < "$2" || return 1 + current=$(mode "$1") || return 1 + if [ "$current" = "$temporary" ] && [ "$current" != "$observed" ]; then + chmod -- "$observed" "$1" + fi + fi +} + +restore_entry() ( + old=$1 expected=$2 live=$3 + if directory "$live"; then + if ! directory "$old" && ! directory "$expected"; then + same_entry "$old" "$expected" || conflict "$live" + return + fi + result=0 + if directory "$old"; then + for child in "$old"/* "$old"/.[!.]* "$old"/..?*; do + present "$child" || continue + name=${child##*/} + after=$absent + if directory "$expected"; then after=$expected/$name; fi + restore_entry "$child" "$after" "$live/$name" || result=$? + done + fi + if directory "$expected"; then + for child in "$expected"/* "$expected"/.[!.]* "$expected"/..?*; do + present "$child" || continue + name=${child##*/} + if ! directory "$old" || ! present "$old/$name"; then + restore_entry "$absent" "$child" "$live/$name" || result=$? + fi + done + fi + [ "$result" -eq 0 ] || return "$result" + if ! directory "$old"; then + for child in "$live"/* "$live"/.[!.]* "$live"/..?*; do + present "$child" || continue + if present "$old"; then conflict "$live"; fi + return + done + rmdir -- "$live" || return 1 + if present "$old"; then cp -pPR -- "$old" "$live"; fi + fi + return + fi + if same_entry "$old" "$expected" || same_entry "$live" "$old"; then return 0; fi + if same_entry "$live" "$expected"; then + if present "$live"; then rm -- "$live" || return 1; fi + if present "$old"; then cp -pPR -- "$old" "$live"; fi + else + conflict "$live" + fi +) +''' + @asynccontextmanager async def _adoption_command( @@ -55,6 +166,7 @@ async def stage(content: str) -> str: @asynccontextmanager async def _adoption_transaction( environment: Any, workspace: str, paths: Sequence[str], *, repository: bool, + replay: Callable[[str, str | None], Awaitable[None]], ) -> AsyncIterator[None]: for path in paths: if ( @@ -102,11 +214,13 @@ async def _adoption_transaction( if not mode or len(mode) > 4 or any(digit not in "01234567" for digit in mode): raise RuntimeError("candidate ancestor directory mode is invalid") directory_modes.append((path, mode)) + git_directory = None storage = '"${TMPDIR:-/tmp}"' if repository: - storage = shlex.quote(_complete(await environment.exec_cmd( + git_directory = _complete(await environment.exec_cmd( f"git -C {shlex.quote(workspace)} rev-parse --absolute-git-dir", timeout=30, - ), "candidate recovery storage").removesuffix("\n")) + ), "candidate recovery storage").removesuffix("\n") + storage = shlex.quote(git_directory) backup = None keep_backup = False @@ -119,53 +233,54 @@ async def allocate_backup() -> None: try: await await_owned_operation(allocate_backup(), propagate_cancellation=True) assert backup is not None + staged = posixpath.join(backup, "replay") + stage_copies = [f"mkdir -m 700 -- {shlex.quote(staged)} || exit"] + stage_modes = [] copies = [] restores = [] recreate_directories = [] restore_directory_modes = [] restore_parents = {posixpath.dirname(path) or "." for path in roots} - # Git prunes empty ancestors after deleting their last leaf. Recreate - # those directories with write access until their saved leaves return. + # Git can prune empty ancestors. Make them writable until the saved files return. for index, (path, mode) in enumerate(directory_modes): - source = shlex.quote(posixpath.join(workspace, path)) + source = shlex.quote(posixpath.normpath(posixpath.join(workspace, path))) + stage_path = shlex.quote(posixpath.normpath(posixpath.join(staged, path))) + stage_copies.append(f"mkdir -p -- {stage_path} || exit") + stage_modes.append(f"chmod -- {mode} {stage_path} || exit") metadata_file = shlex.quote(posixpath.join(backup, f"directory-{index}.mode")) + temporary_mode = shlex.quote(posixpath.join(backup, f"directory-{index}.temporary")) copies.append(f"printf '%s\\0%s\\0' {source} {mode} > {metadata_file} || exit") + permission = "write" if path in restore_parents else "read" recreate_directories.append( - f"if [ ! -e {source} ] && [ ! -L {source} ]; then " - f"mkdir -m 700 -- {source} || failed=1; " - f"elif [ ! -d {source} ] || [ -L {source} ]; then failed=1; fi" + f"prepare_parent {source} {mode} {temporary_mode} {permission} {stage_path} || failed=$?" ) - if path in restore_parents: - recreate_directories.append( - f"if [ -d {source} ] && [ ! -L {source} ] && " - f"[ ! -w {source} ] && [ -O {source} ]; then " - f"chmod -- u+w {source} || failed=1; fi" - ) restore_directory_modes.append( - f"if [ -d {source} ] && [ ! -L {source} ] && " - f"{{ current=$(stat -c %a -- {source} 2>/dev/null) || " - f"current=$(stat -f %Lp -- {source}); }}; then " - f'[ "$current" = {mode} ] || chmod -- {mode} {source} || failed=1; ' - "else failed=1; fi" + f"reset_parent_mode {source} {temporary_mode} || failed=$?" ) for index, path in enumerate(roots): source = shlex.quote(posixpath.join(workspace, path)) saved = shlex.quote(posixpath.join(backup, str(index))) + expected = shlex.quote(posixpath.join(staged, path)) copies.append( f"printf '%s' {source} > {saved}.path || exit; " f"if [ -e {source} ] || [ -L {source} ]; then " f"cp -pPR -- {source} {saved} || exit; fi" ) + stage_copies.append( + f"if present {saved}; then cp -pPR -- {saved} {expected} || exit; fi" + ) restores.append( - f"if [ -e {saved} ] || [ -L {saved} ]; then " - f"if rm -rf -- {source} && mv -- {saved} {source} " - f"&& {{ [ -e {source} ] || [ -L {source} ]; }}; then :; else failed=1; fi; " - f"elif rm -rf -- {source} && [ ! -e {source} ] && [ ! -L {source} ]; " - "then :; else failed=1; fi" + f"{{ check_parents {source} && restore_entry {saved} {expected} {source}; }} || failed=$?" ) - recovery_script = "failed=0; " + "; ".join(( + replay_script = 'present() { [ -e "$1" ] || [ -L "$1" ]; }; ' + "; ".join(( + *stage_copies, *reversed(stage_modes), + )) + recovery_script = ( + f"failed=0; workspace={shlex.quote(workspace)}; " + f"absent={shlex.quote(posixpath.join(backup, 'absent'))};\n" + ) + _RECOVER_PATHS + "; ".join(( *recreate_directories, *restores, *reversed(restore_directory_modes), - )) + '; [ "$failed" -eq 0 ]' + )) + '; exit "$failed"' async with AsyncExitStack() as commands: snapshot_command = await commands.enter_async_context(_adoption_command( environment, workspace, "; ".join(copies), repository=repository, @@ -178,10 +293,16 @@ async def allocate_backup() -> None: try: yield except BaseException as failure: + async def recover() -> None: + async with _adoption_command( + environment, workspace, replay_script, repository=repository, + ) as command: + _complete(await environment.exec_cmd(command, timeout=900), "candidate recovery replay copy") + await replay(staged, git_directory) + _complete(await environment.exec_cmd(recovery_command, timeout=900), "candidate adoption recovery") + try: - _complete(await await_owned_operation(environment.exec_cmd( - recovery_command, timeout=900, - )), "candidate adoption recovery") + await await_owned_operation(recover()) except BaseException as recovery: keep_backup = True revoke = getattr(environment, "revoke", None) @@ -195,8 +316,11 @@ async def allocate_backup() -> None: finally: if backup is not None and not keep_backup: try: + quoted = shlex.quote(backup) await await_owned_operation(environment.exec_cmd( - f"rm -rf -- {shlex.quote(backup)}", timeout=120, + f"rm -rf -- {quoted} || " + f"(find {quoted} -type d ! -perm -u+w -exec chmod u+w -- {{}} + && rm -rf -- {quoted})", + timeout=120, )) except Exception: pass diff --git a/opencollab/adapters/_candidate_increment.py b/opencollab/adapters/_candidate_increment.py index 3f04f1b9..deba1acf 100644 --- a/opencollab/adapters/_candidate_increment.py +++ b/opencollab/adapters/_candidate_increment.py @@ -40,12 +40,14 @@ async def _apply_increment( workspace if docker or getattr(environment, "local_filesystem", False) else getattr(environment, "workspace", workspace) ) + # Directory copies can sit below an unrelated Git worktree. + git_prefix = "git" if repository else "GIT_DIR=/dev/null git" async with _patch_temporary( environment, workspace, patch, prefix=".candidate-adopt-", suffix=".patch", cleanup_best_effort=True, repository=repository, ) as candidate_file: output = _complete(await environment.exec_cmd( - f"git -C {shlex.quote(workspace)} apply --numstat -z -- {shlex.quote(candidate_file)}", + f"{git_prefix} -C {shlex.quote(workspace)} apply --numstat -z -- {shlex.quote(candidate_file)}", timeout=30, ), "candidate path inspection") candidate_paths = { @@ -64,7 +66,7 @@ async def _apply_increment( if docker else host_write_locks(targets, environment) ) async with locks: - git = "git -c core.filemode=true" + git = f"{git_prefix} -c core.filemode=true" guard = "" if strict and repository: git, guard = _git_command_and_config_guard() @@ -86,8 +88,29 @@ async def _apply_increment( f"{command} apply --check --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}", timeout=120, ), operation) + + async def replay(staged: str, git_directory: str | None) -> None: + async with AsyncExitStack() as metadata: + replay_git = f"cd -- {shlex.quote(workspace)} && {guard}{git}" + if physical_paths and repository: + replay_git, git_directory = await metadata.enter_async_context(_physical_git( + environment, workspace, physical_paths, strict=strict, + )) + if git_directory is not None: + replay_git += ( + f" --git-dir={shlex.quote(git_directory)} --work-tree={shlex.quote(staged)}" + f" -c safe.directory={shlex.quote(staged)}" + ) + # A failed replay still records the files Git was able to write. + result = await environment.exec_cmd( + f"{replay_git} -C {shlex.quote(staged)} apply --whitespace=nowarn --binary" + f" -- {shlex.quote(candidate_file)}", timeout=120, + ) + _complete(result, "candidate recovery replay", allowed=(0, 1, 128)) + async with _adoption_transaction( environment, workspace, sorted(candidate_paths), repository=repository, + replay=replay, ): try: _complete(await environment.exec_cmd( diff --git a/tests/workflows/test_candidate_adoption_recovery.py b/tests/workflows/test_candidate_adoption_recovery.py index eb69bc03..57be1c41 100644 --- a/tests/workflows/test_candidate_adoption_recovery.py +++ b/tests/workflows/test_candidate_adoption_recovery.py @@ -12,6 +12,7 @@ import pytest from opencollab import OpenCollab +from opencollab.adapters._candidate_increment import _apply_increment from opencollab.adapters._env_local import LocalEnvironment from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace from opencollab.adapters.llm.types import LLMResponse, Usage @@ -272,6 +273,42 @@ async def test_git_write_failure_after_successful_check_restores_partial_applica await environment.cleanup() +@pytest.mark.parametrize("inside_repository", [False, True]) +async def test_directory_copy_candidate_replays_partial_apply_outside_git(tmp_path, inside_repository): + if inside_repository: + _git(tmp_path, "init", "-q") + repo = tmp_path / "copy" + (repo / "settings").mkdir(parents=True) + (repo / "settings/retained.txt").write_text("keep\n") + (repo / "a.txt").write_text("original\n") + patch = ( + "diff --git a/a.txt b/a.txt\n--- a/a.txt\n+++ b/a.txt\n" + "@@ -1 +1 @@\n-original\n+candidate\n" + "diff --git a/settings b/settings\nnew file mode 100644\n--- /dev/null\n+++ b/settings\n" + "@@ -0,0 +1 @@\n+candidate settings\n" + ) + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + intermediate = [] + + async def inspect(command, timeout=120.0, **kwargs): + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command: + intermediate.append((repo / "a.txt").read_text()) + return result + + environment.exec_cmd = inspect + try: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await _apply_increment(environment, str(repo), patch, repository=False) + assert intermediate == ["candidate\n"] + assert (repo / "a.txt").read_text() == "original\n" + assert (repo / "settings/retained.txt").read_text() == "keep\n" + assert not environment.revoked + finally: + await environment.cleanup() + + @pytest.mark.parametrize("path", [".", ".gi\u200ct/config"]) async def test_git_rejects_metadata_aliases_before_candidate_backup(tmp_path, path): repo = _repository(tmp_path) diff --git a/tests/workflows/test_candidate_shared_mutations.py b/tests/workflows/test_candidate_shared_mutations.py index 5a92ea26..dc51e349 100644 --- a/tests/workflows/test_candidate_shared_mutations.py +++ b/tests/workflows/test_candidate_shared_mutations.py @@ -9,26 +9,31 @@ from opencollab import OpenCollab from opencollab.adapters._env_local import LocalEnvironment from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError from opencollab.tools import builtin_tools from tests.workflows.test_candidate_adoption_recovery import _flatten, _repository, _run from tests.workflows.test_workflow_candidate_workspace import _git class _PublicationBarrier(LocalEnvironment): - def __init__(self, workspace): + def __init__(self, workspace, *, after_write=False): super().__init__(str(workspace)) + self.after_write = after_write self.entered = asyncio.Event() self.release = asyncio.Event() self.publications = [] async def exec_cmd(self, command, timeout=120, **kwargs): publication = " apply --binary " in command and ".candidate-adopt-" in command - if publication: + if publication and not self.after_write: self.entered.set() await self.release.wait() result = await super().exec_cmd(command, timeout, **kwargs) if publication: self.publications.append(result.returncode) + if self.after_write: + self.entered.set() + await self.release.wait() return result @@ -42,7 +47,9 @@ async def complete(self, messages, tools=None, **kwargs): if self.calls == 1: self.requested.set() arguments = {"path": self.path} - if self.tool == "file_write": + if self.tool == "bash": + arguments = {"command": f"printf '%s\\n' 'shared replacement' > {self.path}"} + elif self.tool == "file_write": arguments.update(mode="create", content="VALUE = 100\n") else: arguments.update(mode="line_replace", start_line=1, end_line=1, new_str="VALUE = 100") @@ -51,7 +58,8 @@ async def complete(self, messages, tools=None, **kwargs): "name": self.tool, "arguments": json.dumps(arguments), }, }], usage=Usage(10, 5), finish_reason="tool_calls") - assert any("100" in str(message) for message in messages) + evidence = "Exit code: 0" if self.tool == "bash" else "100" + assert any(evidence in str(message) for message in messages) return LLMResponse(content="Shared edit complete", usage=Usage(10, 5), finish_reason="stop") @@ -137,3 +145,69 @@ async def flow(ctx, _): finally: environment.release.set() await environment.cleanup() + + +@pytest.mark.parametrize("after_write,path", [ + (True, "loader.py"), (True, "settings/ignored.cache"), (True, "settings/notes.txt"), (True, "separate.txt"), + (False, "settings/notes.txt"), +]) +async def test_sdk_shared_bash_preserves_writes_around_git_publication_failure(tmp_path, after_write, path): + repo = _repository(tmp_path) + if after_write: + with (repo / ".gitignore").open("a") as ignore: + ignore.write("settings/ignored.cache\n") + (repo / "settings/ignored.cache").write_text("runtime cache\n") + loader = (repo / "loader.py").read_bytes() + index = (repo / ".git/index").read_bytes() + environment = _PublicationBarrier(repo, after_write=after_write) + overlap = path == "loader.py" + + async def edit(child, _): + _flatten(Path(child.workspace_root)) + + async def flow(ctx, _): + candidate = await ctx.candidate_workflow(edit, {}, label="flatten") + publication = asyncio.create_task(ctx.adopt_candidate(candidate)) + try: + await asyncio.wait_for(environment.entered.wait(), 10) + assert ((repo / "loader.py").read_bytes() != loader) is after_write + writer = await asyncio.wait_for(ctx.agent_run( + "Save this file in the shared workspace", tools=list(builtin_tools("bash", headless=False)), + isolation="shared", budget=2000, system_prompt="Use the requested Bash operation", + ), 10) + assert writer.status == "completed" + assert (repo / path).read_text() == "shared replacement\n" + environment.release.set() + error = CandidateAdoptionRecoveryError if overlap else RuntimeError + detail = "concurrent change at .*/loader.py" if overlap else "Directory not empty|Is a directory" + with pytest.raises(error, match=detail): + await publication + finally: + environment.release.set() + await asyncio.gather(publication, return_exceptions=True) + + try: + result = await OpenCollab(repo, environment=environment, model="local-scripted").workflow( + flow, llm=_WriterModel("bash", path), trace=False, timeout=30, budget=5000, + concurrency=2, limit_mode="explicit", agent_profile="single2", + ) + assert result.ok, result.reason + assert environment.publications == [128] + assert (repo / path).read_text() == "shared replacement\n" + assert (repo / "settings/defaults.py").read_text() == "VALUE = 1\n" + assert (repo / "draft.txt").read_text() == "untracked user draft\n" + assert (repo / ".git/index").read_bytes() == index + backups = list((repo / ".git").glob("candidate-recovery-*")) + if overlap: + backup, = backups + for record in backup.glob("*.path"): + if Path(record.read_text()).name == "loader.py": + assert record.with_suffix("").read_bytes() == loader + break + else: + pytest.fail("recovery did not retain the original loader") + else: + assert (repo / "loader.py").read_bytes() == loader + assert not backups + finally: + await environment.cleanup() From 6c7b255273b543ebf06d3ad2bb36adcc7f755c49 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 14:17:28 +0800 Subject: [PATCH 38/82] fix(workflow): restore saved candidate files without a second copy --- opencollab/adapters/_candidate_adoption.py | 30 ++++++----- .../test_candidate_adoption_recovery.py | 53 +++++++++++++++++++ 2 files changed, 71 insertions(+), 12 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index 2e84a08a..b5cdca55 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -76,6 +76,12 @@ fi } +restore_saved() { + if ! present "$1"; then return 0; fi + if present "$2"; then conflict "$2"; return; fi + mv -- "$1" "$2" +} + restore_entry() ( old=$1 expected=$2 live=$3 if directory "$live"; then @@ -84,15 +90,6 @@ return fi result=0 - if directory "$old"; then - for child in "$old"/* "$old"/.[!.]* "$old"/..?*; do - present "$child" || continue - name=${child##*/} - after=$absent - if directory "$expected"; then after=$expected/$name; fi - restore_entry "$child" "$after" "$live/$name" || result=$? - done - fi if directory "$expected"; then for child in "$expected"/* "$expected"/.[!.]* "$expected"/..?*; do present "$child" || continue @@ -102,6 +99,15 @@ fi done fi + if directory "$old"; then + for child in "$old"/* "$old"/.[!.]* "$old"/..?*; do + present "$child" || continue + name=${child##*/} + after=$absent + if directory "$expected"; then after=$expected/$name; fi + restore_entry "$child" "$after" "$live/$name" || result=$? + done + fi [ "$result" -eq 0 ] || return "$result" if ! directory "$old"; then for child in "$live"/* "$live"/.[!.]* "$live"/..?*; do @@ -110,14 +116,14 @@ return done rmdir -- "$live" || return 1 - if present "$old"; then cp -pPR -- "$old" "$live"; fi + restore_saved "$old" "$live" fi return fi if same_entry "$old" "$expected" || same_entry "$live" "$old"; then return 0; fi if same_entry "$live" "$expected"; then if present "$live"; then rm -- "$live" || return 1; fi - if present "$old"; then cp -pPR -- "$old" "$live"; fi + restore_saved "$old" "$live" else conflict "$live" fi @@ -319,7 +325,7 @@ async def recover() -> None: quoted = shlex.quote(backup) await await_owned_operation(environment.exec_cmd( f"rm -rf -- {quoted} || " - f"(find {quoted} -type d ! -perm -u+w -exec chmod u+w -- {{}} + && rm -rf -- {quoted})", + f"(find {quoted} -type d ! -perm -u+w -exec chmod -- u+w {{}} + && rm -rf -- {quoted})", timeout=120, )) except Exception: diff --git a/tests/workflows/test_candidate_adoption_recovery.py b/tests/workflows/test_candidate_adoption_recovery.py index 57be1c41..214d4ca2 100644 --- a/tests/workflows/test_candidate_adoption_recovery.py +++ b/tests/workflows/test_candidate_adoption_recovery.py @@ -1,5 +1,6 @@ from __future__ import annotations +import errno import json import os import shlex @@ -309,6 +310,58 @@ async def inspect(command, timeout=120.0, **kwargs): await environment.cleanup() +@pytest.mark.parametrize("replay_full", [False, True]) +async def test_recovery_preserves_saved_entries_when_copies_fail(tmp_path, replay_full): + repo = _repository(tmp_path) + target = repo / "other.py" + patch = ( + "diff --git a/other.py b/other.py\n--- a/other.py\n+++ b/other.py\n" + "@@ -1 +1 @@\n-VALUE = 1\n+VALUE = 42\n" + ) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + failures = [] + + async def fault(command, timeout=120.0, **kwargs): + if command.startswith("present()") and replay_full: + failures.append("replay") + raise OSError(errno.ENOSPC, "No space left on device") + if command.startswith("failed=0;"): + command = "cp() { echo 'restore copy: No space left on device' >&2; return 1; }; " + command + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command and ".candidate-adopt-" in command: + assert result.returncode == 0, result.stderr + assert target.read_text() == "VALUE = 42\n" + if replay_full: + target.write_text("VALUE = 100\n") + failures.append("receipt") + raise OSError("publication receipt unavailable") + return result + + environment.exec_cmd = fault + try: + if replay_full: + detail = "No space left on device.*original files retained" + with pytest.raises(CandidateAdoptionRecoveryError, match=detail): + await _apply_increment(environment, str(repo), patch) + backup, = (repo / ".git").glob("candidate-recovery-*") + record, = backup.glob("*.path") + assert record.with_suffix("").read_text() == "VALUE = 1\n" + assert target.read_text() == "VALUE = 100\n" + assert failures == ["receipt", "replay"] + else: + with pytest.raises(OSError, match="publication receipt unavailable"): + await _apply_increment(environment, str(repo), patch) + assert target.read_text() == "VALUE = 1\n" + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert failures == ["receipt"] + assert environment.revoked is replay_full + assert (repo / ".git/index").read_bytes() == index + finally: + await environment.cleanup() + + @pytest.mark.parametrize("path", [".", ".gi\u200ct/config"]) async def test_git_rejects_metadata_aliases_before_candidate_backup(tmp_path, path): repo = _repository(tmp_path) From 78e27090695ed67a35512664e51a1f3ba7f3a4d5 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 14:24:12 +0800 Subject: [PATCH 39/82] fix(workflow): preserve writes made during candidate recovery --- opencollab/adapters/_candidate_adoption.py | 16 ++++++- .../test_candidate_adoption_recovery.py | 42 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index b5cdca55..a058f54f 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -79,7 +79,18 @@ restore_saved() { if ! present "$1"; then return 0; fi if present "$2"; then conflict "$2"; return; fi - mv -- "$1" "$2" + mv -n -- "$1" "$2" || return 1 + if present "$1"; then conflict "$2"; fi +} + +park_candidate_entry() { + if ! present "$1"; then return 0; fi + parked=$(mktemp -d "$recovery/observed-XXXXXXXX") || return 1 + printf '%s' "$1" > "$parked/path" && mv -- "$1" "$parked/entry" || return 1 + if ! same_entry "$parked/entry" "$2"; then + restore_saved "$parked/entry" "$1" || return $? + conflict "$1" + fi } restore_entry() ( @@ -122,7 +133,7 @@ fi if same_entry "$old" "$expected" || same_entry "$live" "$old"; then return 0; fi if same_entry "$live" "$expected"; then - if present "$live"; then rm -- "$live" || return 1; fi + park_candidate_entry "$live" "$expected" || return $? restore_saved "$old" "$live" else conflict "$live" @@ -283,6 +294,7 @@ async def allocate_backup() -> None: )) recovery_script = ( f"failed=0; workspace={shlex.quote(workspace)}; " + f"recovery={shlex.quote(backup)}; " f"absent={shlex.quote(posixpath.join(backup, 'absent'))};\n" ) + _RECOVER_PATHS + "; ".join(( *recreate_directories, *restores, *reversed(restore_directory_modes), diff --git a/tests/workflows/test_candidate_adoption_recovery.py b/tests/workflows/test_candidate_adoption_recovery.py index 214d4ca2..cc7babf4 100644 --- a/tests/workflows/test_candidate_adoption_recovery.py +++ b/tests/workflows/test_candidate_adoption_recovery.py @@ -362,6 +362,48 @@ async def fault(command, timeout=120.0, **kwargs): await environment.cleanup() +async def test_recovery_preserves_a_write_after_comparing_the_live_file(tmp_path): + repo = _repository(tmp_path) + target = repo / "other.py" + marker = tmp_path / "compared" + patch = ( + "diff --git a/other.py b/other.py\n--- a/other.py\n+++ b/other.py\n" + "@@ -1 +1 @@\n-VALUE = 1\n+VALUE = 42\n" + ) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + + async def after_comparison(command, timeout=120.0, **kwargs): + if command.startswith("failed=0;"): + command = ( + 'cmp() { command cmp "$@" || return; ' + f'if [ "$3" = {shlex.quote(str(target))} ] && [ ! -e {shlex.quote(str(marker))} ]; then ' + f"printf 'VALUE = 100\\n' > {shlex.quote(str(target))} && touch {shlex.quote(str(marker))}; " + "fi; }; " + command + ) + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command and ".candidate-adopt-" in command: + assert result.returncode == 0, result.stderr + raise OSError("publication receipt unavailable") + return result + + environment.exec_cmd = after_comparison + try: + with pytest.raises((OSError, CandidateAdoptionRecoveryError)) as failed: + await _apply_increment(environment, str(repo), patch) + assert marker.is_file() + assert target.read_text() == "VALUE = 100\n" + assert isinstance(failed.value, CandidateAdoptionRecoveryError) + assert "concurrent change at " + str(target) in str(failed.value) + backup, = (repo / ".git").glob("candidate-recovery-*") + record, = backup.glob("*.path") + assert record.with_suffix("").read_text() == "VALUE = 1\n" + assert (repo / ".git/index").read_bytes() == index + finally: + await environment.cleanup() + + @pytest.mark.parametrize("path", [".", ".gi\u200ct/config"]) async def test_git_rejects_metadata_aliases_before_candidate_backup(tmp_path, path): repo = _repository(tmp_path) From d2c66c4855efbc5ee43d4cdd0a1588c455bed031 Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 14:42:07 +0800 Subject: [PATCH 40/82] fix(session): prepare snapshots without querying model transports --- opencollab/application/session.py | 9 +++----- .../test_anthropic_preserved_history.py | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/opencollab/application/session.py b/opencollab/application/session.py index 1afb3805..2a1adad7 100644 --- a/opencollab/application/session.py +++ b/opencollab/application/session.py @@ -757,7 +757,7 @@ def _restore_auto_save_tracking(self, snapshot: dict[str, Any]) -> None: self._auto_save_messages = ( copy.deepcopy(stored_messages) if isinstance(stored_messages, list) and self._auto_save_path - and isinstance(self.store, JournalSnapshotStorePort) and self._tracks_native_history() + and isinstance(self.store, JournalSnapshotStorePort) and self.state.anthropic_history is not None else None ) self._auto_save_rewrite_from = None @@ -942,7 +942,7 @@ def save(self, path: str) -> None: # skip its sequence and its completion cannot rewind our cursor. with self._auto_save_lock: messages, meta = self._snapshot_for_save() - tracked_messages = messages if self._tracks_native_history() else None + tracked_messages = messages if self.state.anthropic_history is not None else None sequence = self._reserve_auto_save_sequence() rewrite_revision = self._auto_save_rewrite_revision self._auto_save_pending_checkpoints.add(sequence) @@ -1057,9 +1057,6 @@ def _reserve_auto_save_sequence(self) -> int: ) + 1 return self._auto_save_reserved_sequence - def _tracks_native_history(self) -> bool: - return self.state.anthropic_history is not None or self.runner._requires_thinking_prefix() - def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: preceding_write_pending = self._auto_save_reserved_sequence > self._auto_save_sequence sequence = self._reserve_auto_save_sequence() @@ -1075,7 +1072,7 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: rewrite_revision = self._auto_save_rewrite_revision if self._auto_save_rewrite_from is not None: replace_from = min(replace_from, self._auto_save_rewrite_from) - track_messages = self._tracks_native_history() + track_messages = self.state.anthropic_history is not None stored_messages = self._auto_save_messages or [] if track_messages: # Native history references resolve against the last committed transcript. diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index 11c1501a..9f0f1056 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -598,6 +598,29 @@ async def test_native_history_delta_uses_the_last_saved_transcript(monkeypatch, await environment.cleanup() +async def test_autosave_without_native_history_uses_only_session_state(tmp_path, monkeypatch): + path = tmp_path / "session.json" + environment = LocalEnvironment(str(tmp_path)) + async with _client() as client: + session = build_session(agent=_agent(), llm=client, env=environment, auto_save_path=str(path)) + + def unavailable(): + raise RuntimeError("model metadata unavailable during snapshot") + + monkeypatch.setattr(session.runner, "_requires_thinking_prefix", unavailable) + try: + await session.add_user_message("Save this prompt before the first request.") + await session.enqueue_auto_save() + assert not session.persistence_errors + session.save(str(path)) + snapshot = session.store.load_snapshot(str(path), session.agent.system_prompt) + assert snapshot["messages"][-1]["content"] == "Save this prompt before the first request." + assert snapshot["session_state"]["anthropic_history"] is None + finally: + await session.aclose() + await environment.cleanup() + + @pytest.mark.parametrize("source", ["invalid", [], [{}, {}], [False]]) def test_restore_rejects_malformed_optional_source_prefix(source): message = {"role": "user", "content": "Request"} From 006bbe976ce2e909be991b350e0e3cdd4d13459a Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 17:08:36 +0800 Subject: [PATCH 41/82] fix(llm): honor explicit revalidation in decorated clients --- opencollab/adapters/llm/client.py | 10 ++- .../application/_session_run_completion.py | 22 +++++-- .../runtime/test_session_output_limit_http.py | 65 +++++++++++++++++++ 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index dbc51451..b8ba69e7 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -70,6 +70,13 @@ def _locked_record() -> None: except Exception: return + +class _InheritedRequestRevalidation: + # A class or instance can shadow this default with an explicit boolean flag. + def __get__(self, instance: object, owner: type) -> bool: + return True + + class LLMClient: """Client for OpenAI-compatible endpoints and native Anthropic requests. @@ -77,6 +84,8 @@ class LLMClient: OpenAI-format history per request. """ + supports_request_revalidation = _InheritedRequestRevalidation() + def __init__( self, model: str = "gpt-4o", @@ -123,7 +132,6 @@ def __init__( self.request_pool = request_pool self.request_capacity = request_capacity self.supports_request_admission = request_pool is not None - self.supports_request_revalidation = True self.supports_attempt_usage = True self.supports_usage_settlement = True self.model = model diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index 21672275..9cc4ab9e 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -59,25 +59,30 @@ logger = logging.getLogger(__name__) -def _completion_keyword_parameters(complete: Callable[..., Awaitable[Any]]) -> set[str] | None: +def _completion_keyword_parameters(complete: Callable[..., Awaitable[Any]]) -> tuple[set[str] | None, bool]: keywords: set[str] = set() + outer_accepts_kwargs = False def has_fixed_keywords(target: Callable[..., Any]) -> bool: + nonlocal outer_accepts_kwargs parameters = inspect.signature(target, follow_wrapped=False).parameters.values() keywords.update( parameter.name for parameter in parameters if parameter.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY) ) - return not any(parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters) + accepts_kwargs = any(parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters) + if target is complete: + outer_accepts_kwargs = accepts_kwargs + return not accepts_kwargs # Follow variadic decorators through to the first fixed signature. try: target = inspect.unwrap(complete, stop=has_fixed_keywords) if not hasattr(target, "__wrapped__") and not has_fixed_keywords(target): - return None + return None, outer_accepts_kwargs except (TypeError, ValueError): pass - return keywords + return keywords, outer_accepts_kwargs def _selector_names_tool_choice(selector: Any) -> bool: """Return whether a structured error selector targets ``tool_choice``.""" @@ -790,7 +795,7 @@ def _add_completion_controls( usage_accounting: _CompletionUsageAccounting, revalidate: Callable[[], int], ) -> asyncio.Event | None: - keywords = _completion_keyword_parameters(complete) + keywords, outer_accepts_kwargs = _completion_keyword_parameters(complete) def accepts(name: str) -> bool: return keywords is None or name in keywords @@ -801,7 +806,12 @@ def accepts(name: str) -> bool: kwargs["on_attempt_usage"] = usage_accounting.record_attempt if getattr(self.llm, "supports_usage_settlement", False) and accepts("on_usage"): kwargs["on_usage"] = usage_accounting.settle - if getattr(self.llm, "supports_request_revalidation", False) and accepts("before_request"): + # A declared flag lets middleware consume the callback before forwarding to a fixed signature. + revalidation = "supports_request_revalidation" + if getattr(self.llm, revalidation, False) and ( + accepts("before_request") + or (outer_accepts_kwargs and inspect.getattr_static(self.llm, revalidation, False) is True) + ): kwargs["before_request"] = revalidate if getattr(self.llm, "supports_request_admission", False) and accepts("on_admitted"): admitted = asyncio.Event() diff --git a/tests/runtime/test_session_output_limit_http.py b/tests/runtime/test_session_output_limit_http.py index a3710323..97a03af1 100644 --- a/tests/runtime/test_session_output_limit_http.py +++ b/tests/runtime/test_session_output_limit_http.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import json from functools import wraps @@ -10,6 +11,7 @@ from opencollab import OpenCollab, RunControl from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.application._session_run_shared import _TokenBudgetStop from opencollab.bootstrap import container from opencollab.bootstrap.session_factory import build_session from opencollab.domain.agent import Agent @@ -199,6 +201,69 @@ async def complete(self, *args, on_usage=None, **kwargs): return response +class _RevalidatingWrapper(_LegacyNativeClient): + @wraps(_LegacyNativeClient.complete) + async def complete(self, *args, **kwargs): + self.arrived.set() + await self.release.wait() + before_request = kwargs.pop("before_request", None) + if before_request is not None: + kwargs["max_output_tokens"] = min(kwargs["max_output_tokens"], before_request()) + return await super().complete(*args, **kwargs) + + +class _ClassOptInWrapper(_RevalidatingWrapper): + supports_request_revalidation = True + + +@pytest.mark.parametrize("purpose", ["completion", "summary"]) +@pytest.mark.parametrize("declare_on_class", [False, True], ids=["instance-opt-in", "class-opt-in"]) +async def test_decorated_client_revalidates_budget_before_sending( + tmp_path, monkeypatch, local_environment_factory, purpose, declare_on_class, +): + requests = [] + + async def handler(request): + requests.append(json.loads(request.content)) + return completion_http_response(request, text="Preserved context.") + + install_sdk_transport(monkeypatch, "openai", handler) + client_class = _ClassOptInWrapper if declare_on_class else _RevalidatingWrapper + async with client_class( + api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + ) as client: + if not declare_on_class: + client.supports_request_revalidation = True + client.arrived, client.release = asyncio.Event(), asyncio.Event() + session = build_session( + agent=Agent(name="queued", system_prompt="system", max_tokens_per_step=128), + llm=client, env=local_environment_factory(str(tmp_path)), max_budget_tokens=20_000, + ) + await session.add_user_message("Keep this context") + operation = ( + session._context_summarizer.asummarize(session.messages) + if purpose == "summary" else session.run_loop() + ) + task = asyncio.create_task(operation) + try: + await asyncio.wait_for(client.arrived.wait(), 5) + session.max_budget_tokens = 1 + client.release.set() + if purpose == "summary": + with pytest.raises(_TokenBudgetStop): + await asyncio.wait_for(task, 5) + else: + assert await asyncio.wait_for(task, 5) == "" + assert "budget exhausted before model call" in session.state.terminal_reason + assert requests == [] + assert session.used_tokens == 0 + finally: + client.release.set() + await asyncio.gather(task, return_exceptions=True) + await session.aclose() + + @pytest.mark.parametrize("failed", [False, True]) @pytest.mark.parametrize("client_class", [_LegacyNativeClient, _SettlementWrapper]) async def test_public_agent_charges_legacy_native_wrapper_usage_once(tmp_path, monkeypatch, failed, client_class): From 5e838493f942174c728e04b7a4b4d41dbea60ddb Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 17:17:47 +0800 Subject: [PATCH 42/82] fix(workflow): preserve Git attributes during candidate recovery --- opencollab/adapters/_candidate_adoption.py | 59 ++++++- opencollab/adapters/_candidate_increment.py | 29 +++- .../test_candidate_adoption_recovery.py | 148 ++++++++++++++++++ 3 files changed, 224 insertions(+), 12 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index a058f54f..c9f0a09b 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -43,6 +43,21 @@ directory "$workspace" || conflict "$workspace" } +check_candidate_descendants() { + prefix=${1#"$workspace_prefix"}; shift + for path; do + case "$path" in + "$prefix"/*) + target=$workspace_prefix$path + if check_parents "$target" 2>/dev/null && present "$target"; then + conflict "$target" + return + fi + ;; + esac + done +} + prepare_parent() { if [ "$1" != "$workspace" ]; then check_parents "$1" || return $?; fi if ! present "$1"; then @@ -95,9 +110,15 @@ restore_entry() ( old=$1 expected=$2 live=$3 + if same_entry "$old" "$expected"; then + # An unchanged replay entry cannot explain a changed patch target. + if candidate_path "$live" && ! same_entry "$old" "$live"; then conflict "$live"; return; fi + if directory "$live"; then candidate_descendants "$live"; return; fi + return 0 + fi if directory "$live"; then if ! directory "$old" && ! directory "$expected"; then - same_entry "$old" "$expected" || conflict "$live" + conflict "$live" return fi result=0 @@ -131,7 +152,7 @@ fi return fi - if same_entry "$old" "$expected" || same_entry "$live" "$old"; then return 0; fi + if same_entry "$live" "$old"; then return 0; fi if same_entry "$live" "$expected"; then park_candidate_entry "$live" "$expected" || return $? restore_saved "$old" "$live" @@ -183,7 +204,8 @@ async def stage(content: str) -> str: @asynccontextmanager async def _adoption_transaction( environment: Any, workspace: str, paths: Sequence[str], *, repository: bool, - replay: Callable[[str, str | None], Awaitable[None]], + replay: Callable[[str, str | None, str | None], Awaitable[None]], + attributes_file: str | None = None, ) -> AsyncIterator[None]: for path in paths: if ( @@ -258,6 +280,15 @@ async def allocate_backup() -> None: recreate_directories = [] restore_directory_modes = [] restore_parents = {posixpath.dirname(path) or "." for path in roots} + saved_attributes_file = None + if attributes_file is not None: + saved_attributes_file = posixpath.join(backup, "configured-attributes") + source_attributes = shlex.quote(attributes_file) + copies.append( + f"if [ -f {source_attributes} ] && [ -r {source_attributes} ]; then " + f"cat -- {source_attributes} > {shlex.quote(saved_attributes_file)} || exit; fi" + ) + saved_roots = set(roots) # Git can prune empty ancestors. Make them writable until the saved files return. for index, (path, mode) in enumerate(directory_modes): source = shlex.quote(posixpath.normpath(posixpath.join(workspace, path))) @@ -267,6 +298,21 @@ async def allocate_backup() -> None: metadata_file = shlex.quote(posixpath.join(backup, f"directory-{index}.mode")) temporary_mode = shlex.quote(posixpath.join(backup, f"directory-{index}.temporary")) copies.append(f"printf '%s\\0%s\\0' {source} {mode} > {metadata_file} || exit") + # Attribute files inside affected roots are already in the file snapshot. + attributes_path = posixpath.normpath(posixpath.join(path, ".gitattributes")) + if repository and attributes_path not in saved_roots: + source_attributes = shlex.quote(posixpath.join(workspace, attributes_path)) + saved_attributes = shlex.quote(posixpath.join(backup, f"directory-{index}.attributes")) + staged_attributes = shlex.quote(posixpath.join(staged, attributes_path)) + copies.append( + f"if [ -f {source_attributes} ] && [ ! -L {source_attributes} ] " + f"&& [ -r {source_attributes} ]; then " + f"cat -- {source_attributes} > {saved_attributes} || exit; fi" + ) + stage_copies.append( + f"if [ -f {saved_attributes} ]; then " + f"cat -- {saved_attributes} > {staged_attributes} || exit; fi" + ) permission = "write" if path in restore_parents else "read" recreate_directories.append( f"prepare_parent {source} {mode} {temporary_mode} {permission} {stage_path} || failed=$?" @@ -292,10 +338,15 @@ async def allocate_backup() -> None: replay_script = 'present() { [ -e "$1" ] || [ -L "$1" ]; }; ' + "; ".join(( *stage_copies, *reversed(stage_modes), )) + targets = [shlex.quote(path) for path in paths] + target_test = f'case "${{1#"$workspace_prefix"}}" in {"|".join(targets)}) return 0;; esac; ' if targets else "" recovery_script = ( f"failed=0; workspace={shlex.quote(workspace)}; " f"recovery={shlex.quote(backup)}; " f"absent={shlex.quote(posixpath.join(backup, 'absent'))};\n" + 'workspace_prefix="${workspace%/}/";\n' + f"candidate_path() {{ {target_test}return 1; }}\n" + f"candidate_descendants() {{ check_candidate_descendants \"$1\" {' '.join(targets)}; }}\n" ) + _RECOVER_PATHS + "; ".join(( *recreate_directories, *restores, *reversed(restore_directory_modes), )) + '; exit "$failed"' @@ -316,7 +367,7 @@ async def recover() -> None: environment, workspace, replay_script, repository=repository, ) as command: _complete(await environment.exec_cmd(command, timeout=900), "candidate recovery replay copy") - await replay(staged, git_directory) + await replay(staged, git_directory, saved_attributes_file) _complete(await environment.exec_cmd(recovery_command, timeout=900), "candidate adoption recovery") try: diff --git a/opencollab/adapters/_candidate_increment.py b/opencollab/adapters/_candidate_increment.py index deba1acf..b2dd2401 100644 --- a/opencollab/adapters/_candidate_increment.py +++ b/opencollab/adapters/_candidate_increment.py @@ -84,12 +84,18 @@ async def _apply_increment( operation = "physical candidate adoption" # Let Git reject unsafe path aliases before a recursive backup can # encounter repository metadata. Publication can still fail later. - _complete(await environment.exec_cmd( - f"{command} apply --check --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}", - timeout=120, - ), operation) + check = f"{command} apply --check --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}" + configured_rules = repository and not strict and not physical_paths + if configured_rules: + check += f' && ({git} config --path --get core.attributesFile || [ "$?" -eq 1 ])' + checked = _complete(await environment.exec_cmd(check, timeout=120), operation) + attributes_file = None + if configured_rules and (configured := checked.removesuffix("\n")): + attributes_file = posixpath.join(workspace, configured) - async def replay(staged: str, git_directory: str | None) -> None: + async def replay( + staged: str, git_directory: str | None, saved_attributes_file: str | None, + ) -> None: async with AsyncExitStack() as metadata: replay_git = f"cd -- {shlex.quote(workspace)} && {guard}{git}" if physical_paths and repository: @@ -101,16 +107,23 @@ async def replay(staged: str, git_directory: str | None) -> None: f" --git-dir={shlex.quote(git_directory)} --work-tree={shlex.quote(staged)}" f" -c safe.directory={shlex.quote(staged)}" ) + if saved_attributes_file is not None: + replay_git += f" -c core.attributesFile={shlex.quote(saved_attributes_file)}" + replay_git += f" -C {shlex.quote(staged)}" + _complete(await environment.exec_cmd( + f"{replay_git} apply --whitespace=nowarn --check --binary" + f" -- {shlex.quote(candidate_file)}", timeout=120, + ), "candidate recovery replay check") # A failed replay still records the files Git was able to write. result = await environment.exec_cmd( - f"{replay_git} -C {shlex.quote(staged)} apply --whitespace=nowarn --binary" + f"{replay_git} apply --whitespace=nowarn --binary" f" -- {shlex.quote(candidate_file)}", timeout=120, ) - _complete(result, "candidate recovery replay", allowed=(0, 1, 128)) + _complete(result, "candidate recovery replay", allowed=(0, 128)) async with _adoption_transaction( environment, workspace, sorted(candidate_paths), repository=repository, - replay=replay, + replay=replay, attributes_file=attributes_file, ): try: _complete(await environment.exec_cmd( diff --git a/tests/workflows/test_candidate_adoption_recovery.py b/tests/workflows/test_candidate_adoption_recovery.py index cc7babf4..f9d818ee 100644 --- a/tests/workflows/test_candidate_adoption_recovery.py +++ b/tests/workflows/test_candidate_adoption_recovery.py @@ -14,10 +14,12 @@ from opencollab import OpenCollab from opencollab.adapters._candidate_increment import _apply_increment +from opencollab.adapters._env_base import ExecResult from opencollab.adapters._env_local import LocalEnvironment from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace from opencollab.adapters.llm.types import LLMResponse, Usage from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError +from opencollab.workflows import CandidateRun from tests.workflows.test_workflow_candidate_workspace import _git @@ -274,6 +276,152 @@ async def test_git_write_failure_after_successful_check_restores_partial_applica await environment.cleanup() +@pytest.mark.parametrize(("rules", "blocked"), [ + ("tracked", True), + ("unstaged", True), + ("nested", True), + ("info", True), + ("configured", True), + ("tracked", False), +]) +async def test_public_sdk_replays_effective_crlf_attributes(tmp_path, rules, blocked): + repo = tmp_path + _git(repo, "init", "-q") + _git(repo, "config", "core.autocrlf", "false") + (repo / "nested").mkdir() + target = repo / "nested/a.txt" + target.write_bytes(b"original\n") + attributes = { + "nested": repo / "nested/.gitattributes", + "info": repo / ".git/info/attributes", + "configured": repo / "local-attributes", + }.get(rules, repo / ".gitattributes") + if rules == "configured": + _git(repo, "config", "core.attributesFile", attributes.name) + if rules in {"tracked", "unstaged"}: + eol = "lf" if rules == "unstaged" else "crlf" + attributes.write_text(f"*.txt text eol={eol}\n") + _git(repo, "add", ".gitattributes") + _git(repo, "add", "nested/a.txt") + _git(repo, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare CRLF recovery") + if rules != "tracked": + attributes.write_text("*.txt text eol=crlf\n") + target.write_bytes(b"original\r\n") + patch = ( + "diff --git a/nested/a.txt b/nested/a.txt\n--- a/nested/a.txt\n+++ b/nested/a.txt\n" + "@@ -1 +1 @@\n-original\n+candidate\n" + ) + if blocked: + (repo / "settings").mkdir() + (repo / "settings/cache.txt").write_text("preserve cache\n") + patch += ( + "diff --git a/settings b/settings\nnew file mode 100644\n--- /dev/null\n+++ b/settings\n" + "@@ -0,0 +1 @@\n+new settings\n" + ) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + publications = [] + replays = [] + change_rules = rules in {"nested", "configured"} + + async def observe(command, timeout=120.0, **kwargs): + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command: + publications.append((result.returncode, target.read_bytes())) + if change_rules: + attributes.write_text("*.txt text eol=lf\n") + if "candidate-recovery-" in command and " apply " in command: + replays.append(result.returncode) + return result + + environment.exec_cmd = observe + + async def flow(ctx, inputs): + candidate = CandidateRun("text update", "ready", patch, (), ()) + await ctx.adopt_candidate(candidate) + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow(flow, trace=False) + assert publications == [(128 if blocked else 0, b"candidate\r\n")] + if blocked: + assert type(result.error) is RuntimeError + assert "Directory not empty" in result.reason + assert replays == [0, 128] + assert target.read_bytes() == b"original\r\n" + assert (repo / "settings/cache.txt").read_bytes() == b"preserve cache\n" + else: + assert result.ok, result.reason + assert target.read_bytes() == b"candidate\r\n" + assert replays == [] + assert attributes.read_text() == f"*.txt text eol={'lf' if change_rules else 'crlf'}\n" + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not environment.revoked + finally: + await environment.cleanup() + + +@pytest.mark.parametrize(("replay_status", "target_kind"), [ + (1, "existing"), (128, "existing"), (128, "new"), (128, "sibling"), +]) +async def test_unverified_candidate_replay_preserves_live_entries(tmp_path, replay_status, target_kind): + repo = _repository(tmp_path) + if target_kind == "existing": + target = repo / "other.py" + patch = ( + "diff --git a/other.py b/other.py\n--- a/other.py\n+++ b/other.py\n" + "@@ -1 +1 @@\n-VALUE = 1\n+VALUE = 42\n" + ) + else: + target = repo / "new/inner/file.py" + patch = ( + "diff --git a/new/inner/file.py b/new/inner/file.py\nnew file mode 100644\n" + "--- /dev/null\n+++ b/new/inner/file.py\n@@ -0,0 +1 @@\n+VALUE = 42\n" + ) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + + async def fault(command, timeout=120.0, **kwargs): + if " apply --whitespace=nowarn --binary " in command: + return ExecResult(replay_status, "", "replay interrupted before writing") + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command: + assert result.returncode == 0, result.stderr + if target_kind == "sibling": + target.unlink() + target.with_name("notes.txt").write_text("unrelated notes\n") + raise OSError("publication receipt unavailable") + return result + + environment.exec_cmd = fault + try: + if target_kind == "sibling": + with pytest.raises(OSError, match="publication receipt unavailable"): + await _apply_increment(environment, str(repo), patch) + assert target.with_name("notes.txt").read_text() == "unrelated notes\n" + assert not target.exists() + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not environment.revoked + else: + with pytest.raises(CandidateAdoptionRecoveryError, match="original files retained"): + await _apply_increment(environment, str(repo), patch) + assert target.read_text() == "VALUE = 42\n" + backup, = (repo / ".git").glob("candidate-recovery-*") + record, = backup.glob("*.path") + if target_kind == "existing": + assert record.with_suffix("").read_bytes() == b"VALUE = 1\n" + else: + assert Path(record.read_text()) == repo / "new" + assert not record.with_suffix("").exists() + assert environment.revoked + assert (repo / ".git/index").read_bytes() == index + finally: + await environment.cleanup() + + @pytest.mark.parametrize("inside_repository", [False, True]) async def test_directory_copy_candidate_replays_partial_apply_outside_git(tmp_path, inside_repository): if inside_repository: From bdb097ab8450850640d7cde1dffdf6042b4eccfd Mon Sep 17 00:00:00 2001 From: Kai Date: Wed, 18 Nov 2026 17:25:00 +0800 Subject: [PATCH 43/82] fix(session): persist rebuilt native history without duplicating the journal --- opencollab/adapters/storage.py | 1 + opencollab/application/session.py | 58 ++++++++----- tests/agents/test_session_persistence.py | 13 +++ .../test_anthropic_preserved_history.py | 87 +++++++++++++++++++ 4 files changed, 135 insertions(+), 24 deletions(-) diff --git a/opencollab/adapters/storage.py b/opencollab/adapters/storage.py index e2f60f41..3e42eb3e 100644 --- a/opencollab/adapters/storage.py +++ b/opencollab/adapters/storage.py @@ -501,6 +501,7 @@ def checkpoint_snapshot( or sequence < 0 ): raise ValueError("autosave sequence must be a non-negative integer") + self._validate_messages(messages) self._ensure_parent(path) obj = { **meta, diff --git a/opencollab/application/session.py b/opencollab/application/session.py index 2a1adad7..6b5fe2fb 100644 --- a/opencollab/application/session.py +++ b/opencollab/application/session.py @@ -942,7 +942,8 @@ def save(self, path: str) -> None: # skip its sequence and its completion cannot rewind our cursor. with self._auto_save_lock: messages, meta = self._snapshot_for_save() - tracked_messages = messages if self.state.anthropic_history is not None else None + track_messages = self._auto_save_messages is not None or self.state.anthropic_history is not None + tracked_messages = messages if track_messages else None sequence = self._reserve_auto_save_sequence() rewrite_revision = self._auto_save_rewrite_revision self._auto_save_pending_checkpoints.add(sequence) @@ -1072,7 +1073,8 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: rewrite_revision = self._auto_save_rewrite_revision if self._auto_save_rewrite_from is not None: replace_from = min(replace_from, self._auto_save_rewrite_from) - track_messages = self.state.anthropic_history is not None + has_native_history = self.state.anthropic_history is not None + track_messages = self._auto_save_messages is not None or has_native_history stored_messages = self._auto_save_messages or [] if track_messages: # Native history references resolve against the last committed transcript. @@ -1083,6 +1085,10 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: if current_messages[index] != stored_messages[index]: replace_from = index break + # A full native rewrite can exceed the journal's remaining capacity. + checkpoint_only = has_native_history and replace_from == 0 and ( + preceding_write_pending or self._auto_save_message_count > 0 + ) messages, meta = self._snapshot_for_save(replace_from) tracked_messages = [*stored_messages[:replace_from], *messages] if track_messages else None current_seen_hashes = set(self.state.turn.seen_result_hashes) @@ -1095,33 +1101,35 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: else current_seen_hashes - self._auto_save_seen_result_hashes ) checkpoint_messages = None - if sequence >= self._next_auto_save_checkpoint: + if checkpoint_only or sequence >= self._next_auto_save_checkpoint: checkpoint_messages = tracked_messages if checkpoint_messages is None: checkpoint_messages = copy.deepcopy(self.state.enriched_messages()) + def advance_cursor() -> None: + self._auto_save_sequence = sequence + self._auto_save_message_count = message_count + self._auto_save_messages = tracked_messages + if self._auto_save_rewrite_revision == rewrite_revision: + self._auto_save_rewrite_from = None + self._auto_save_seen_result_hashes = current_seen_hashes + def persist() -> None: - self.store.append_snapshot_delta( - path, - sequence=sequence, - replace_from=replace_from, - messages=messages, - meta=meta, - seen_result_hashes_reset=seen_hashes_reset, - seen_result_hashes_added=seen_hashes_added, - ) - with self._auto_save_lock: - if sequence <= self._auto_save_sequence: - return - # The journal append is already fsync'd. Advance the absolute - # cursor before optional compaction so a failed base rewrite - # cannot cause the durable delta to be skipped on the next save. - self._auto_save_sequence = sequence - self._auto_save_message_count = message_count - self._auto_save_messages = tracked_messages - if self._auto_save_rewrite_revision == rewrite_revision: - self._auto_save_rewrite_from = None - self._auto_save_seen_result_hashes = current_seen_hashes + if not checkpoint_only: + self.store.append_snapshot_delta( + path, + sequence=sequence, + replace_from=replace_from, + messages=messages, + meta=meta, + seen_result_hashes_reset=seen_hashes_reset, + seen_result_hashes_added=seen_hashes_added, + ) + with self._auto_save_lock: + if sequence <= self._auto_save_sequence: + return + # The fsync'd append survives a failed base rewrite. + advance_cursor() if checkpoint_messages is not None: with self._auto_save_checkpoint_lock: with self._auto_save_lock: @@ -1134,6 +1142,8 @@ def persist() -> None: sequence=sequence, ) with self._auto_save_lock: + if checkpoint_only: + advance_cursor() self._next_auto_save_checkpoint = 1 << sequence.bit_length() def persist_incrementally() -> None: diff --git a/tests/agents/test_session_persistence.py b/tests/agents/test_session_persistence.py index 4fda8ddb..27fc4f76 100644 --- a/tests/agents/test_session_persistence.py +++ b/tests/agents/test_session_persistence.py @@ -157,6 +157,19 @@ def test_store_replay_ignores_journal_records_covered_by_atomic_base(tmp_path) - "step 3", ] +def test_invalid_checkpoint_messages_preserve_the_durable_journal(tmp_path) -> None: + path = str(tmp_path / "session.json") + store = SessionStore() + messages = [{"role": "user", "content": "last durable prompt"}] + store.append_snapshot_delta(path, sequence=1, replace_from=0, messages=messages, meta={}) + + with pytest.raises(ValueError, match="missing 'content'"): + store.checkpoint_snapshot(path, [{"role": "user"}], meta={}, sequence=2) + + restored = store.load_snapshot(path, "fallback") + assert restored["_autosave_sequence"] == 1 + assert restored["messages"] == messages + def test_store_rejects_complete_invalid_journal_record(tmp_path) -> None: path = tmp_path / "session.json" store = SessionStore() diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index 9f0f1056..03f34069 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -10,6 +10,7 @@ import pytest from opencollab import OpenCollab +from opencollab.adapters import storage as storage_module from opencollab.adapters.env import LocalEnvironment from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.tools.fs import FileReadTool, FileWriteTool @@ -598,6 +599,92 @@ async def test_native_history_delta_uses_the_last_saved_transcript(monkeypatch, await environment.cleanup() +@pytest.mark.parametrize("previous_save", ["journal", "reopen-journal", "manual-covers-append"]) +async def test_autosave_native_view_after_large_transcript_replacement(monkeypatch, tmp_path, previous_save): + monkeypatch.setattr(storage_module, "MAX_SESSION_SNAPSHOT_BYTES", 256 * 1024) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + archived = "A" * (140 * 1024) + path = tmp_path / "session.json" + journal = path.with_suffix(".json.journal") + held_request, resume = asyncio.Event(), asyncio.Event() + peer = _BoundThinkingProvider(reads=0, thinking=False) + store = PausedAppendStore() + + class ArchiveShaper: + def shape(self, messages): + return [ + {**message, "content": "[Archived context]"} if message.get("content") == archived else message + for message in messages + ] + + async def provider(request): + response = peer(request) + if len(peer.requests) == 3: + held_request.set() + await resume.wait() + return response + + install_sdk_transport(monkeypatch, "anthropic", provider) + model = "claude-sonnet-5-5" + async with _client(model) as client: + environment = LocalEnvironment(str(tmp_path)) + options = { + "agent": _agent(thinking=False, model=model), "llm": client, "env": environment, + "auto_save_path": str(path), "shaper": ArchiveShaper(), "store": store, + } + session = build_session(**options) + running = None + try: + await session.add_user_message("Warm up the conversation") + assert await session.run_loop() == "done" + assert session.state.anthropic_history is not None + while session._next_auto_save_checkpoint - session._auto_save_sequence <= 3: + await session.enqueue_auto_save() + session.messages = [*session.messages, {"role": "user", "content": archived}] + store.pause_next = previous_save == "manual-covers-append" + await session.add_user_message("Read source.txt before answering") + if previous_save == "manual-covers-append": + assert await asyncio.to_thread(store.started.wait, 1) + session.save(str(path)) + store.release.set() + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + before_bytes = journal.stat().st_size + before = session.store.load_snapshot(str(path), session.agent.system_prompt) + assert before_bytes > len(archived) + assert before["session_state"]["anthropic_history"] is None + assert before["messages"][-2]["content"] == archived + + if previous_save == "reopen-journal": + await session.aclose() + session = load_session(str(path), **options) + peer.reads = 2 + running = asyncio.create_task(session.run_loop()) + await asyncio.wait_for(held_request.wait(), 5) + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + assert not running.done() + assert "file evidence" in json.dumps(peer.requests[-1]["messages"]) + assert not session.persistence_errors + + saved = session.store.load_snapshot(str(path), session.agent.system_prompt) + assert saved["_autosave_sequence"] > before["_autosave_sequence"] + assert any("file evidence" in message["content"] for message in saved["messages"] + if message["role"] == "tool") + history = saved["session_state"]["anthropic_history"] + assert "source_messages" not in history + assert any(type(message) is int for message in history["messages"]) + assert journal.stat().st_size < before_bytes + 8 * 1024 + + resume.set() + assert await running == "done" + finally: + resume.set() + store.release.set() + if running is not None: + await asyncio.gather(running, return_exceptions=True) + await session.aclose() + await environment.cleanup() + + async def test_autosave_without_native_history_uses_only_session_state(tmp_path, monkeypatch): path = tmp_path / "session.json" environment = LocalEnvironment(str(tmp_path)) From cfb6e5c6127fcac809b82ff6abbfe5bc3c394b45 Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 10:36:05 +0800 Subject: [PATCH 44/82] fix(llm): honor revalidation declarations stored as descriptors --- opencollab/adapters/llm/client.py | 9 +- .../application/_session_run_completion.py | 14 ++- opencollab/application/ports.py | 9 ++ .../runtime/test_session_output_limit_http.py | 98 +++++++++++++++++-- 4 files changed, 111 insertions(+), 19 deletions(-) diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index b8ba69e7..d49ac4a3 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -40,6 +40,7 @@ from opencollab.adapters.llm.types import LLMResponse, model_context_window from opencollab.adapters.llm.usage_ledger import record_api_usage from opencollab.application.async_timeout import await_owned_operation +from opencollab.application.ports import _NATIVE_REQUEST_REVALIDATION from opencollab.domain.token_estimation import estimate_request_tokens # Native data-residency endpoints share the OpenAI model parameter rules. @@ -71,12 +72,6 @@ def _locked_record() -> None: return -class _InheritedRequestRevalidation: - # A class or instance can shadow this default with an explicit boolean flag. - def __get__(self, instance: object, owner: type) -> bool: - return True - - class LLMClient: """Client for OpenAI-compatible endpoints and native Anthropic requests. @@ -84,7 +79,7 @@ class LLMClient: OpenAI-format history per request. """ - supports_request_revalidation = _InheritedRequestRevalidation() + supports_request_revalidation = _NATIVE_REQUEST_REVALIDATION def __init__( self, diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index 9cc4ab9e..11977636 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -36,6 +36,7 @@ from opencollab.application._tool_loop_execution import _apply_completed_prefix_progress from opencollab.application.async_timeout import CallerTimeoutError, abandon_on_timeout from opencollab.application.ports import ( + _NATIVE_REQUEST_REVALIDATION, CompletionResponse, ContextTokenEstimatorPort, RequestOutputRequirementPort, @@ -806,11 +807,16 @@ def accepts(name: str) -> bool: kwargs["on_attempt_usage"] = usage_accounting.record_attempt if getattr(self.llm, "supports_usage_settlement", False) and accepts("on_usage"): kwargs["on_usage"] = usage_accounting.settle - # A declared flag lets middleware consume the callback before forwarding to a fixed signature. - revalidation = "supports_request_revalidation" - if getattr(self.llm, revalidation, False) and ( + revalidation = getattr(self.llm, "supports_request_revalidation", False) + # Explicit middleware opt-ins can consume the callback before forwarding. + if revalidation and ( accepts("before_request") - or (outer_accepts_kwargs and inspect.getattr_static(self.llm, revalidation, False) is True) + or ( + outer_accepts_kwargs and revalidation is True + and inspect.getattr_static( + self.llm, "supports_request_revalidation", _NATIVE_REQUEST_REVALIDATION, + ) is not _NATIVE_REQUEST_REVALIDATION + ) ): kwargs["before_request"] = revalidate if getattr(self.llm, "supports_request_admission", False) and accepts("on_admitted"): diff --git a/opencollab/application/ports.py b/opencollab/application/ports.py index 13e8234c..fbad138c 100644 --- a/opencollab/application/ports.py +++ b/opencollab/application/ports.py @@ -470,6 +470,15 @@ def transport_timing(self) -> dict[str, Any] | None: """ ... +class _NativeRequestRevalidation: + def __get__(self, instance: object, owner: type) -> bool: + return True + + +# Native clients support the callback through their completion signature. +_NATIVE_REQUEST_REVALIDATION = _NativeRequestRevalidation() + + class LLMPort(Protocol): """LLM client surface used by the session run loop and compaction.""" diff --git a/tests/runtime/test_session_output_limit_http.py b/tests/runtime/test_session_output_limit_http.py index 97a03af1..6e7423eb 100644 --- a/tests/runtime/test_session_output_limit_http.py +++ b/tests/runtime/test_session_output_limit_http.py @@ -202,10 +202,14 @@ async def complete(self, *args, on_usage=None, **kwargs): class _RevalidatingWrapper(_LegacyNativeClient): + wait_for_prompt = None + @wraps(_LegacyNativeClient.complete) async def complete(self, *args, **kwargs): - self.arrived.set() - await self.release.wait() + messages = args[0] if args else kwargs["messages"] + if self.wait_for_prompt is None or messages[-1]["content"] == self.wait_for_prompt: + self.arrived.set() + await self.release.wait() before_request = kwargs.pop("before_request", None) if before_request is not None: kwargs["max_output_tokens"] = min(kwargs["max_output_tokens"], before_request()) @@ -216,10 +220,24 @@ class _ClassOptInWrapper(_RevalidatingWrapper): supports_request_revalidation = True +class _SlotOptInWrapper(_RevalidatingWrapper): + __slots__ = ("supports_request_revalidation",) + + +class _PropertyOptInWrapper(_RevalidatingWrapper): + @property + def supports_request_revalidation(self): + return True + + @pytest.mark.parametrize("purpose", ["completion", "summary"]) -@pytest.mark.parametrize("declare_on_class", [False, True], ids=["instance-opt-in", "class-opt-in"]) +@pytest.mark.parametrize( + "client_class", + [_RevalidatingWrapper, _ClassOptInWrapper, _SlotOptInWrapper, _PropertyOptInWrapper], + ids=["instance", "class", "slot", "property"], +) async def test_decorated_client_revalidates_budget_before_sending( - tmp_path, monkeypatch, local_environment_factory, purpose, declare_on_class, + tmp_path, monkeypatch, local_environment_factory, purpose, client_class, ): requests = [] @@ -228,12 +246,11 @@ async def handler(request): return completion_http_response(request, text="Preserved context.") install_sdk_transport(monkeypatch, "openai", handler) - client_class = _ClassOptInWrapper if declare_on_class else _RevalidatingWrapper async with client_class( api_key="controlled-test", # pragma: allowlist secret base_url="https://controlled.invalid/v1", max_retries=0, ) as client: - if not declare_on_class: + if client_class in {_RevalidatingWrapper, _SlotOptInWrapper}: client.supports_request_revalidation = True client.arrived, client.release = asyncio.Event(), asyncio.Event() session = build_session( @@ -264,8 +281,72 @@ async def handler(request): await session.aclose() +async def test_decorated_slot_client_revalidates_shared_workflow_spend(tmp_path, monkeypatch): + requests = [] + + async def handler(request): + body = json.loads(request.content) + prompt = body["messages"][-1]["content"] + requests.append((prompt, body["max_tokens"])) + response = completion_http_response(request) + if prompt == "spender": + usage = {"prompt_tokens": 1997, "completion_tokens": 2, "total_tokens": 1999} + return http_response(request, 200, json=response.json() | {"usage": usage}) + return response + + install_sdk_transport(monkeypatch, "openai", handler) + async with _SlotOptInWrapper( + api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + ) as client: + client.supports_request_revalidation = True + client.wait_for_prompt = "waiter" + client.arrived, client.release = asyncio.Event(), asyncio.Event() + + async def flow(ctx, _inputs): + waiter = asyncio.create_task(ctx.agent_run("waiter", tools=[], system_prompt="System.", budget=1000)) + try: + await asyncio.wait_for(client.arrived.wait(), 5) + spender = await ctx.agent_run("spender", tools=[], system_prompt="System.", budget=1000) + assert spender.tokens == ctx.tokens_spent() == 1999 + client.release.set() + return await asyncio.wait_for(waiter, 5) + finally: + client.release.set() + await asyncio.gather(waiter, return_exceptions=True) + + result = await OpenCollab(tmp_path, config={"max_output_tokens": 128}).workflow( + flow, llm=client, budget=2000, trace=False, limit_mode="explicit", concurrency=2, + ) + + assert result.status == "completed" + assert result.output.status == "stopped" + assert "budget exhausted before model call" in result.output.reason + assert result.output.tokens == 0 + assert result.tokens == 1999 + assert requests == [("spender", 128)] + + +class _ForwardingWrapper(_LegacyNativeClient): + @wraps(_LegacyNativeClient.complete) + async def complete(self, *args, **kwargs): + return await super().complete(*args, **kwargs) + + +class _ForwardingProxy: + def __init__(self, client): + self.client = client + + def __getattr__(self, name): + return getattr(self.client, name) + + @wraps(_ForwardingWrapper.complete) + async def complete(self, *args, **kwargs): + return await self.client.complete(*args, **kwargs) + + @pytest.mark.parametrize("failed", [False, True]) -@pytest.mark.parametrize("client_class", [_LegacyNativeClient, _SettlementWrapper]) +@pytest.mark.parametrize("client_class", [_LegacyNativeClient, _SettlementWrapper, _ForwardingWrapper]) async def test_public_agent_charges_legacy_native_wrapper_usage_once(tmp_path, monkeypatch, failed, client_class): requests = [] events = [] @@ -285,8 +366,9 @@ async def handler(request): base_url="https://controlled.invalid/v1", max_retries=0, request_pool=str(pool), request_capacity=1, ) as client: + llm = _ForwardingProxy(client) if client_class is _ForwardingWrapper else client result = await OpenCollab(tmp_path).agent( - "Return ok", llm=client, tools=(), budget=20_000, trace=False, + "Return ok", llm=llm, tools=(), budget=20_000, trace=False, run_control=RunControl(on_event=events.append), ) From c5537783c6cd9f8969fdabfe7141aaeab76b8f8f Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 10:43:35 +0800 Subject: [PATCH 45/82] fix(session): preserve journal fallback when native checkpoints fail --- opencollab/adapters/storage.py | 149 ++++++++++++------ opencollab/application/ports.py | 16 ++ opencollab/application/session.py | 40 +++-- tests/agents/test_session_persistence.py | 43 +++++ .../test_anthropic_preserved_history.py | 107 +++++++++++++ 5 files changed, 301 insertions(+), 54 deletions(-) diff --git a/opencollab/adapters/storage.py b/opencollab/adapters/storage.py index 3e42eb3e..a3a097d0 100644 --- a/opencollab/adapters/storage.py +++ b/opencollab/adapters/storage.py @@ -20,6 +20,7 @@ write_regular_bytes_atomic, write_regular_file_atomic, ) +from opencollab.application.ports import SnapshotCapacityError MAX_SESSION_SNAPSHOT_BYTES = 64 * 1024 * 1024 _JSON_WRITE_CHUNK_CHARS = 64 * 1024 @@ -127,7 +128,7 @@ def write(self, text: str) -> int: payload = text[offset : offset + _JSON_WRITE_CHUNK_CHARS].encode("utf-8") next_size = self._bytes_written + len(payload) if next_size > MAX_SESSION_SNAPSHOT_BYTES: - raise ValueError( + raise SnapshotCapacityError( "session snapshot exceeds " f"{MAX_SESSION_SNAPSHOT_BYTES} UTF-8 bytes while writing: " f"{self._path}" @@ -183,7 +184,7 @@ def load_snapshot(self, path: str, system_prompt: str) -> dict[str, Any]: ) def _load_snapshot_unlocked( - self, path: str, system_prompt: str, *, read_journal: bool = True, + self, path: str, fallback_system_prompt: str | None, *, read_journal: bool = True, ) -> dict[str, Any]: """Load a locked snapshot or a standalone read-only export.""" try: @@ -282,7 +283,8 @@ def _load_snapshot_unlocked( raise ValueError( "Invalid session snapshot: expected at least one message" ) - messages = [{"role": "system", "content": system_prompt}] + if fallback_system_prompt is not None: + messages = [{"role": "system", "content": fallback_system_prompt}] snapshot["messages"] = messages return snapshot @@ -446,6 +448,63 @@ def append_snapshot_delta( seen_result_hashes_added: list[str] | None = None, ) -> None: """Durably append one absolute, idempotent autosave delta.""" + record = self._snapshot_delta_record( + sequence=sequence, + replace_from=replace_from, + messages=messages, + meta=meta, + seen_result_hashes_reset=seen_result_hashes_reset, + seen_result_hashes_added=seen_result_hashes_added, + ) + self._append_journal_record(path, record) + + def append_snapshot_replacement( + self, + path: str, + messages: list[dict[str, Any]], + *, + meta: dict[str, Any], + sequence: int, + ) -> bool: + state = meta.get("session_state") + seen_result_hashes = state.get("seen_result_hashes", []) if isinstance(state, dict) else [] + record = self._snapshot_delta_record( + sequence=sequence, + replace_from=0, + messages=messages, + meta=meta, + seen_result_hashes_reset=True, + seen_result_hashes_added=seen_result_hashes, + ) + with _journal_operation_lock(path): + try: + persisted = self._load_snapshot_unlocked(path, fallback_system_prompt=None) + except FileNotFoundError: + pass + else: + if sequence <= self._snapshot_sequence(persisted): + return False + stored_messages = persisted["messages"] + replace_from = min(len(stored_messages), len(messages)) + for index in range(replace_from): + if stored_messages[index] != messages[index]: + replace_from = index + break + record["replace_from"] = replace_from + record["messages"] = messages[replace_from:] + self._append_journal_record_unlocked(path, record) + return True + + def _snapshot_delta_record( + self, + *, + sequence: int, + replace_from: int, + messages: list[dict[str, Any]], + meta: dict[str, Any], + seen_result_hashes_reset: bool, + seen_result_hashes_added: list[str] | None, + ) -> dict[str, Any]: if ( isinstance(sequence, bool) or not isinstance(sequence, int) @@ -474,7 +533,7 @@ def append_snapshot_delta( journal_state = dict(raw_state) journal_state.pop("seen_result_hashes", None) journal_meta["session_state"] = journal_state - record = { + return { "journal_version": _AUTOSAVE_JOURNAL_VERSION, "sequence": sequence, "replace_from": replace_from, @@ -484,7 +543,6 @@ def append_snapshot_delta( "seen_result_hashes_reset": seen_result_hashes_reset, "seen_result_hashes_added": seen_added, } - self._append_journal_record(path, record) def checkpoint_snapshot( self, @@ -565,47 +623,50 @@ def _encode_journal_record(record: dict[str, Any]) -> str: def _append_journal_record(self, path: str, record: dict[str, Any]) -> None: with _journal_operation_lock(path): - journal_path = self._journal_path(path) - payload = self._encode_journal_record(record) - payload_size = len(payload.encode("utf-8")) - if payload_size > MAX_SESSION_SNAPSHOT_BYTES: - raise ValueError( - "autosave journal record exceeds " - f"{MAX_SESSION_SNAPSHOT_BYTES} UTF-8 bytes while writing: " - f"{journal_path}" + self._append_journal_record_unlocked(path, record) + + def _append_journal_record_unlocked(self, path: str, record: dict[str, Any]) -> None: + journal_path = self._journal_path(path) + payload = self._encode_journal_record(record) + payload_size = len(payload.encode("utf-8")) + if payload_size > MAX_SESSION_SNAPSHOT_BYTES: + raise ValueError( + "autosave journal record exceeds " + f"{MAX_SESSION_SNAPSHOT_BYTES} UTF-8 bytes while writing: " + f"{journal_path}" + ) + append_start: int | None = None + try: + with open_regular_text_append(journal_path, readable=True) as handle: + current_size = os.fstat(handle.fileno()).st_size + complete_size = self._complete_journal_size( + handle.fileno(), + current_size, ) - append_start: int | None = None - try: - with open_regular_text_append(journal_path, readable=True) as handle: - current_size = os.fstat(handle.fileno()).st_size - complete_size = self._complete_journal_size( - handle.fileno(), - current_size, + if complete_size < current_size: + os.ftruncate(handle.fileno(), complete_size) + current_size = complete_size + if current_size + payload_size > MAX_SESSION_SNAPSHOT_BYTES: + raise ValueError( + "autosave journal exceeds " + f"{MAX_SESSION_SNAPSHOT_BYTES} UTF-8 bytes while writing: " + f"{journal_path}" ) - if complete_size < current_size: - os.ftruncate(handle.fileno(), complete_size) - current_size = complete_size - if current_size + payload_size > MAX_SESSION_SNAPSHOT_BYTES: - raise ValueError( - "autosave journal exceeds " - f"{MAX_SESSION_SNAPSHOT_BYTES} UTF-8 bytes while writing: " - f"{journal_path}" - ) - append_start = current_size - write_locked_text(handle, payload) - handle.flush() - os.fsync(handle.fileno()) - except BaseException as append_error: - if append_start is not None: - try: - # Close the append handle before truncating so a buffered - # close cannot write the failed record back afterward. - with open_regular_text_append(journal_path, readable=True) as handle: - os.ftruncate(handle.fileno(), append_start) - os.fsync(handle.fileno()) - except BaseException as rollback_error: - raise append_error from rollback_error - raise + append_start = current_size + write_locked_text(handle, payload) + handle.flush() + os.fsync(handle.fileno()) + except BaseException as append_error: + if append_start is not None: + try: + # Close the append handle before truncating so a buffered + # close cannot write the failed record back afterward. + with open_regular_text_append(journal_path, readable=True) as handle: + os.ftruncate(handle.fileno(), append_start) + os.fsync(handle.fileno()) + except BaseException as rollback_error: + raise append_error from rollback_error + raise @staticmethod def _complete_journal_size(fd: int, size: int) -> int: diff --git a/opencollab/application/ports.py b/opencollab/application/ports.py index fbad138c..ef6b7023 100644 --- a/opencollab/application/ports.py +++ b/opencollab/application/ports.py @@ -614,6 +614,22 @@ def checkpoint_snapshot( ) -> None: ... +class SnapshotCapacityError(ValueError): + """An encoded snapshot exceeds the store's write limit.""" + +@runtime_checkable +class SnapshotReplacementStorePort(Protocol): + def append_snapshot_replacement( + self, + path: str, + messages: list[dict[str, Any]], + *, + meta: dict[str, Any], + sequence: int, + ) -> bool: + """Return whether the replacement was appended at a new durable cursor.""" + ... + class TracePort(Protocol): """Trajectory recorder surface.""" diff --git a/opencollab/application/session.py b/opencollab/application/session.py index 6b5fe2fb..a62a5f9b 100644 --- a/opencollab/application/session.py +++ b/opencollab/application/session.py @@ -20,6 +20,8 @@ PermissionPort, SafetyPolicyPort, SessionStorePort, + SnapshotCapacityError, + SnapshotReplacementStorePort, SnapshotStorePort, TracePort, ) @@ -1086,7 +1088,7 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: replace_from = index break # A full native rewrite can exceed the journal's remaining capacity. - checkpoint_only = has_native_history and replace_from == 0 and ( + checkpoint_first = has_native_history and replace_from == 0 and ( preceding_write_pending or self._auto_save_message_count > 0 ) messages, meta = self._snapshot_for_save(replace_from) @@ -1101,7 +1103,7 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: else current_seen_hashes - self._auto_save_seen_result_hashes ) checkpoint_messages = None - if checkpoint_only or sequence >= self._next_auto_save_checkpoint: + if checkpoint_first or sequence >= self._next_auto_save_checkpoint: checkpoint_messages = tracked_messages if checkpoint_messages is None: checkpoint_messages = copy.deepcopy(self.state.enriched_messages()) @@ -1115,7 +1117,7 @@ def advance_cursor() -> None: self._auto_save_seen_result_hashes = current_seen_hashes def persist() -> None: - if not checkpoint_only: + if not checkpoint_first: self.store.append_snapshot_delta( path, sequence=sequence, @@ -1135,14 +1137,32 @@ def persist() -> None: with self._auto_save_lock: if sequence < self._auto_save_sequence: return - self.store.checkpoint_snapshot( - path, - checkpoint_messages, - meta=meta, - sequence=sequence, - ) + try: + self.store.checkpoint_snapshot( + path, + checkpoint_messages, + meta=meta, + sequence=sequence, + ) + except (OSError, SnapshotCapacityError) as checkpoint_error: + if not checkpoint_first or not isinstance(self.store, SnapshotReplacementStorePort): + raise + try: + saved = self.store.append_snapshot_replacement( + path, + checkpoint_messages, + meta=meta, + sequence=sequence, + ) + except Exception as journal_error: + raise checkpoint_error from journal_error + if not saved: + raise + with self._auto_save_lock: + advance_cursor() + return with self._auto_save_lock: - if checkpoint_only: + if checkpoint_first: advance_cursor() self._next_auto_save_checkpoint = 1 << sequence.bit_length() diff --git a/tests/agents/test_session_persistence.py b/tests/agents/test_session_persistence.py index 27fc4f76..57feb16d 100644 --- a/tests/agents/test_session_persistence.py +++ b/tests/agents/test_session_persistence.py @@ -170,6 +170,49 @@ def test_invalid_checkpoint_messages_preserve_the_durable_journal(tmp_path) -> N assert restored["_autosave_sequence"] == 1 assert restored["messages"] == messages +def test_journal_replacement_uses_the_durable_prefix_and_requires_a_new_sequence(tmp_path, monkeypatch) -> None: + path = tmp_path / "replacement.json" + journal = path.with_suffix(".json.journal") + store = SessionStore() + base = [{"role": "system", "content": "prompt"}] + archive = {"role": "user", "content": "€" * 1024} + replacement = [*base, archive, {"role": "assistant", "content": "latest"}] + store.checkpoint_snapshot(str(path), base, meta={}, sequence=1) + store.append_snapshot_delta( + str(path), sequence=2, replace_from=1, messages=[archive], meta={}, + seen_result_hashes_added=["prior"], + ) + before = journal.read_bytes() + monkeypatch.setattr(storage_module, "MAX_SESSION_SNAPSHOT_BYTES", len(before) + 512) + + meta = {"session_state": {"seen_result_hashes": ["latest"]}} + for sequence in (1, 2): + assert not store.append_snapshot_replacement(str(path), replacement, meta=meta, sequence=sequence) + assert journal.read_bytes() == before + + edited_archive = {"role": "user", "content": "雪" * 1024} + with pytest.raises(ValueError, match="autosave journal exceeds"): + store.append_snapshot_replacement(str(path), [*base, edited_archive], meta=meta, sequence=3) + assert journal.read_bytes() == before + + assert store.append_snapshot_replacement(str(path), replacement, meta=meta, sequence=3) + saved = SessionStore().load_snapshot(str(path), "prompt") + assert saved == { + "_autosave_sequence": 3, + "messages": replacement, + "session_state": {"seen_result_hashes": ["latest"]}, + } + assert json.loads(journal.read_bytes().splitlines()[-1])["replace_from"] == 2 + +def test_journal_replacement_uses_real_messages_from_blank_legacy_base(tmp_path) -> None: + path = tmp_path / "empty.json" + path.touch() + messages = [{"role": "system", "content": ""}, {"role": "user", "content": "persist me"}] + store = SessionStore() + + assert store.append_snapshot_replacement(str(path), messages, meta={}, sequence=1) + assert store.load_messages(str(path), "") == messages + def test_store_rejects_complete_invalid_journal_record(tmp_path) -> None: path = tmp_path / "session.json" store = SessionStore() diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index 03f34069..ac81c4d2 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -685,6 +685,113 @@ async def provider(request): await environment.cleanup() +@pytest.mark.parametrize("failure", ["capacity", "before-publish", "after-publish"]) +async def test_native_rewrite_recovers_tool_result_when_checkpoint_fails(monkeypatch, tmp_path, failure): + limit = 128 * 1024 + monkeypatch.setattr(storage_module, "MAX_SESSION_SNAPSHOT_BYTES", limit) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + size = (limit - 14_000) // 2 if failure == "capacity" else 1_000 + archive = _blocks("é" * size) + _blocks("x") * 256 + path = tmp_path / "fallback.json" + journal = path.with_suffix(".json.journal") + held_request, resume = asyncio.Event(), asyncio.Event() + peer = _BoundThinkingProvider(reads=0, thinking=False) + + class ArchiveShaper: + def shape(self, messages): + return [ + {**message, "content": "[Archived context]"} if message.get("content") == archive else message + for message in messages + ] + + async def provider(request): + response = peer(request) + if len(peer.requests) == 3: + held_request.set() + await resume.wait() + return response + + install_sdk_transport(monkeypatch, "anthropic", provider) + model = "claude-sonnet-5-5" + environment = LocalEnvironment(str(tmp_path)) + async with _client(model) as client: + agent = _agent(thinking=False, model=model) + session = build_session(agent=agent, llm=client, env=environment, + auto_save_path=str(path), shaper=ArchiveShaper()) + running = restored = None + try: + await session.add_user_message("Warm up the conversation") + assert await session.run_loop() == "done" + await session.add_user_message("Read source.txt before answering") + await asyncio.gather(*session.pending_cleanup_tasks) + session.save(str(path)) + base = path.read_bytes() + sequence = json.loads(base)["_autosave_sequence"] + 1 + session.messages = [*session.messages, {"role": "user", "content": archive}] + assert session.state.anthropic_history is None + + checkpoint_error = OSError(f"controlled checkpoint failure {failure}") + original_write = session.store._atomic_json_write + original_compact = storage_module.write_regular_bytes_atomic + + def write_snapshot(target, document): + if failure == "before-publish" and document["_autosave_sequence"] == sequence: + raise checkpoint_error + original_write(target, document) + + def compact_journal(target, payload, **options): + if failure == "after-publish": + raise checkpoint_error + original_compact(target, payload, **options) + + peer.reads = 2 + with monkeypatch.context() as fault: + fault.setattr(session.store, "_atomic_json_write", write_snapshot) + fault.setattr(storage_module, "write_regular_bytes_atomic", compact_journal) + running = asyncio.create_task(session.run_loop()) + await asyncio.wait_for(held_request.wait(), 5) + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + + assert not running.done() + saved = session.store.load_snapshot(str(path), agent.system_prompt) + assert saved["_autosave_sequence"] == sequence + assert any(message["content"] == archive for message in saved["messages"]) + restored = load_session(str(path), agent=agent, llm=client, env=environment) + assert restored.state.anthropic_history is not None + assert [row["content"] for row in restored.messages if row["role"] == "tool"] == [ + row["content"] for row in session.messages if row["role"] == "tool" + ] + assert sum(row["role"] == "tool" for row in restored.messages) == 1 + if failure == "after-publish": + assert session.persistence_errors == (checkpoint_error,) + assert json.loads(path.read_bytes())["_autosave_sequence"] == sequence + assert journal.read_bytes() == b"" + else: + assert not session.persistence_errors + record = json.loads(journal.read_bytes()) + assert record["sequence"] == sequence + assert record["replace_from"] > 0 + assert path.read_bytes() == base + if failure == "capacity": + assert len(journal.read_bytes()) < limit + assert len(json.dumps(saved, ensure_ascii=False, indent=2).encode("utf-8")) > limit + + resume.set() + if failure == "capacity": + with pytest.raises(ValueError, match="session snapshot exceeds"): + await running + else: + assert await running == "done" + finally: + resume.set() + if running is not None: + await asyncio.gather(running, return_exceptions=True) + if restored is not None: + await restored.aclose() + await session.aclose() + await environment.cleanup() + + async def test_autosave_without_native_history_uses_only_session_state(tmp_path, monkeypatch): path = tmp_path / "session.json" environment = LocalEnvironment(str(tmp_path)) From 0bd50256d2ead132556b1d4cab83b351fa0014d1 Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 10:52:06 +0800 Subject: [PATCH 46/82] fix(session): fit native checkpoints using compact encoding --- opencollab/adapters/storage.py | 12 ++++++++--- .../test_anthropic_preserved_history.py | 20 ++++++++++++++----- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/opencollab/adapters/storage.py b/opencollab/adapters/storage.py index a3a097d0..66a83820 100644 --- a/opencollab/adapters/storage.py +++ b/opencollab/adapters/storage.py @@ -599,7 +599,10 @@ def checkpoint_snapshot( "refusing checkpoint sequence " f"{sequence}; it conflicts with persisted journal state" ) - self._atomic_json_write(path, obj) + try: + self._atomic_json_write(path, obj) + except SnapshotCapacityError: + self._atomic_json_write(path, obj, indent=None) write_regular_bytes_atomic( self._journal_path(path), b"", @@ -804,11 +807,14 @@ def _read_snapshot_text(path: str) -> str: ).decode("utf-8") @staticmethod - def _atomic_json_write(path: str, value: Any) -> None: + def _atomic_json_write(path: str, value: Any, *, indent: int | None = 2) -> None: """Durably replace one JSON file without exposing a partial snapshot.""" def serialize(handle: BinaryIO) -> None: writer = _BoundedUTF8Writer(handle, path=path) - json.dump(value, writer, ensure_ascii=False, indent=2) + if indent is None: + json.dump(value, writer, ensure_ascii=False, separators=(",", ":")) + else: + json.dump(value, writer, ensure_ascii=False, indent=indent) write_regular_file_atomic( path, diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index ac81c4d2..1898c24b 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -685,12 +685,12 @@ async def provider(request): await environment.cleanup() -@pytest.mark.parametrize("failure", ["capacity", "before-publish", "after-publish"]) +@pytest.mark.parametrize("failure", ["capacity", "format", "before-publish", "after-publish"]) async def test_native_rewrite_recovers_tool_result_when_checkpoint_fails(monkeypatch, tmp_path, failure): limit = 128 * 1024 monkeypatch.setattr(storage_module, "MAX_SESSION_SNAPSHOT_BYTES", limit) monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") - size = (limit - 14_000) // 2 if failure == "capacity" else 1_000 + size = (limit - 14_000) // 2 if failure in {"capacity", "format"} else 1_000 archive = _blocks("é" * size) + _blocks("x") * 256 path = tmp_path / "fallback.json" journal = path.with_suffix(".json.journal") @@ -720,6 +720,10 @@ async def provider(request): auto_save_path=str(path), shaper=ArchiveShaper()) running = restored = None try: + if failure == "capacity": + session.messages = [*session.messages, { + "role": "user", "content": _blocks("[Archived prefix] " + "p" * 20_000), + }] await session.add_user_message("Warm up the conversation") assert await session.run_loop() == "done" await session.add_user_message("Read source.txt before answering") @@ -734,10 +738,10 @@ async def provider(request): original_write = session.store._atomic_json_write original_compact = storage_module.write_regular_bytes_atomic - def write_snapshot(target, document): + def write_snapshot(target, document, **options): if failure == "before-publish" and document["_autosave_sequence"] == sequence: raise checkpoint_error - original_write(target, document) + original_write(target, document, **options) def compact_journal(target, payload, **options): if failure == "after-publish": @@ -766,6 +770,12 @@ def compact_journal(target, payload, **options): assert session.persistence_errors == (checkpoint_error,) assert json.loads(path.read_bytes())["_autosave_sequence"] == sequence assert journal.read_bytes() == b"" + elif failure == "format": + assert not session.persistence_errors + assert len(json.dumps(saved, ensure_ascii=False, indent=2).encode("utf-8")) > limit + assert path.stat().st_size < limit + assert json.loads(path.read_bytes())["_autosave_sequence"] == sequence + assert journal.read_bytes() == b"" else: assert not session.persistence_errors record = json.loads(journal.read_bytes()) @@ -774,7 +784,7 @@ def compact_journal(target, payload, **options): assert path.read_bytes() == base if failure == "capacity": assert len(journal.read_bytes()) < limit - assert len(json.dumps(saved, ensure_ascii=False, indent=2).encode("utf-8")) > limit + assert len(json.dumps(saved, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) > limit resume.set() if failure == "capacity": From c2e6147e84ba6a059a46013fecacdc52e4a45760 Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 10:52:06 +0800 Subject: [PATCH 47/82] fix(workflow): retain recovery data for overlapping managed commands --- opencollab/adapters/_candidate_adoption.py | 98 +++++++++++--- opencollab/adapters/_env_base.py | 23 ++++ opencollab/adapters/hooks.py | 4 +- opencollab/adapters/tools/bash.py | 4 +- opencollab/bootstrap/authored_workflow.py | 9 +- .../bootstrap/authored_workflow_bridge.py | 19 +-- .../runtime/test_authored_workflow_bridge.py | 128 ++++++++++++++++++ .../test_candidate_adoption_recovery.py | 80 +++++++++++ .../test_candidate_shared_mutations.py | 112 ++++++++++++++- 9 files changed, 444 insertions(+), 33 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index c9f0a09b..346d8138 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -13,9 +13,8 @@ from opencollab.application.async_timeout import await_owned_operation from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError -_RECOVER_PATHS = r''' +_COMPARE_ENTRIES = r''' present() { [ -e "$1" ] || [ -L "$1" ]; } -directory() { [ -d "$1" ] && [ ! -L "$1" ]; } mode() { stat -c %a -- "$1" 2>/dev/null || stat -f %Lp -- "$1"; } same_entry() { @@ -32,6 +31,11 @@ } conflict() { printf 'concurrent change at %s\n' "$1" >&2; return 2; } +''' + +_RECOVER_PATHS = _COMPARE_ENTRIES + r''' +directory() { [ -d "$1" ] && [ ! -L "$1" ]; } +device() { stat -c %d -- "$1" 2>/dev/null || stat -f %d -- "$1"; } check_parents() { if [ "$1" = "$workspace" ]; then directory "$workspace" || conflict "$workspace"; return; fi @@ -100,6 +104,13 @@ park_candidate_entry() { if ! present "$1"; then return 0; fi + if [ -f "$1" ] && [ ! -L "$1" ]; then + source_device=$(device "$1") && recovery_device=$(device "$recovery") || return 1 + if [ "$source_device" != "$recovery_device" ]; then + printf 'cannot park %s across filesystems\n' "$1" >&2 + return 2 + fi + fi parked=$(mktemp -d "$recovery/observed-XXXXXXXX") || return 1 printf '%s' "$1" > "$parked/path" && mv -- "$1" "$parked/entry" || return 1 if ! same_entry "$parked/entry" "$2"; then @@ -162,6 +173,15 @@ ) ''' +_RECHECK_PARKED = _COMPARE_ENTRIES + r''' +for parked in "$recovery"/observed-*; do + [ -d "$parked" ] || continue + path=$(cat -- "$parked/path" && printf .) || exit + path=${path%.} + same_entry "$parked/entry" "$staged/${path#"$workspace_prefix"}" || { conflict "$path"; exit; } +done +''' + @asynccontextmanager async def _adoption_command( @@ -256,10 +276,13 @@ async def _adoption_transaction( git_directory = None storage = '"${TMPDIR:-/tmp}"' if repository: - git_directory = _complete(await environment.exec_cmd( - f"git -C {shlex.quote(workspace)} rev-parse --absolute-git-dir", timeout=30, - ), "candidate recovery storage").removesuffix("\n") - storage = shlex.quote(git_directory) + git = f"git -C {shlex.quote(workspace)}" + layout = _complete(await environment.exec_cmd( + f"{git} rev-parse --absolute-git-dir && printf '\\0' && {git} rev-parse --git-common-dir", timeout=30, + ), "candidate recovery storage") + git_directory, common_directory = (path.removesuffix("\n") for path in layout.split("\0")) + # Git removes the private metadata along with a linked worktree. + storage = shlex.quote(posixpath.join(workspace, common_directory)) backup = None keep_backup = False @@ -269,6 +292,22 @@ async def allocate_backup() -> None: f'mktemp -d {storage}/candidate-recovery-XXXXXXXX', timeout=30, ), "candidate recovery storage").removesuffix("\n") + async def remove_backup(path: str, *, verify: bool = False) -> None: + quoted = shlex.quote(path) + removal = ( + f"rm -rf -- {quoted} || " + f"(find {quoted} -type d ! -perm -u+w -exec chmod -- u+w {{}} + && rm -rf -- {quoted})" + ) + if verify: + removal = ( + f"recovery={quoted}; staged={shlex.quote(posixpath.join(path, 'replay'))}; " + f"workspace_prefix={shlex.quote(workspace.rstrip('/') + '/')};\n" + + _RECHECK_PARKED + removal + ) + result = await environment.exec_cmd(removal, timeout=120) + if verify: + _complete(result, "candidate recovery cleanup") + try: await await_owned_operation(allocate_backup(), propagate_cancellation=True) assert backup is not None @@ -349,7 +388,12 @@ async def allocate_backup() -> None: f"candidate_descendants() {{ check_candidate_descendants \"$1\" {' '.join(targets)}; }}\n" ) + _RECOVER_PATHS + "; ".join(( *recreate_directories, *restores, *reversed(restore_directory_modes), - )) + '; exit "$failed"' + )) + r'''; +if [ "$failed" -ne 0 ]; then exit "$failed"; fi +for parked in "$recovery"/observed-*/entry; do + if [ -f "$parked" ] && [ ! -L "$parked" ]; then printf 'parked\n'; break; fi +done +''' async with AsyncExitStack() as commands: snapshot_command = await commands.enter_async_context(_adoption_command( environment, workspace, "; ".join(copies), repository=repository, @@ -362,16 +406,26 @@ async def allocate_backup() -> None: try: yield except BaseException as failure: - async def recover() -> None: + async def recover() -> tuple[bool, bool]: async with _adoption_command( environment, workspace, replay_script, repository=repository, ) as command: _complete(await environment.exec_cmd(command, timeout=900), "candidate recovery replay copy") await replay(staged, git_directory, saved_attributes_file) - _complete(await environment.exec_cmd(recovery_command, timeout=900), "candidate adoption recovery") + + shell_commands = getattr(environment, "_shell_commands", None) + active = shell_commands.active if shell_commands is not None else 0 + last_started = shell_commands.last_started if shell_commands is not None else None + receipt = _complete(await environment.exec_cmd( + recovery_command, timeout=900, + ), "candidate adoption recovery") + shell_overlapped = shell_commands is not None and ( + active > 0 or last_started is not shell_commands.last_started + ) + return "parked" in receipt.splitlines(), shell_overlapped try: - await await_owned_operation(recover()) + parked, shell_overlapped = await await_owned_operation(recover()) except BaseException as recovery: keep_backup = True revoke = getattr(environment, "revoke", None) @@ -381,15 +435,27 @@ async def recover() -> None: f"candidate adoption failed ({failure}); recovery failed ({recovery}); " f"original files retained in {backup}" ) from recovery + if parked: + keep_backup = True + if shell_overlapped: + # A command can still own a detached inode after its original path is restored. + raise CandidateAdoptionRecoveryError( + f"candidate adoption failed ({failure}); a concurrent shell command may have written " + f"to parked files retained in {backup}" + ) from failure + assert backup is not None + try: + await await_owned_operation(remove_backup(backup, verify=True)) + except BaseException as cleanup: + raise CandidateAdoptionRecoveryError( + f"candidate adoption failed ({failure}); cleanup failed ({cleanup}); " + f"recovery location {backup}" + ) from cleanup + backup = None raise finally: if backup is not None and not keep_backup: try: - quoted = shlex.quote(backup) - await await_owned_operation(environment.exec_cmd( - f"rm -rf -- {quoted} || " - f"(find {quoted} -type d ! -perm -u+w -exec chmod -- u+w {{}} + && rm -rf -- {quoted})", - timeout=120, - )) + await await_owned_operation(remove_backup(backup)) except Exception: pass diff --git a/opencollab/adapters/_env_base.py b/opencollab/adapters/_env_base.py index d36ab371..cc6c28e8 100644 --- a/opencollab/adapters/_env_base.py +++ b/opencollab/adapters/_env_base.py @@ -2,6 +2,8 @@ from __future__ import annotations +from collections.abc import Iterator +from contextlib import AbstractContextManager, contextmanager, nullcontext from dataclasses import dataclass, field from opencollab.adapters._safe_file_common import _normalize_newlines, _split_lines @@ -28,6 +30,26 @@ class TextFileRange: has_more: bool chars_truncated: bool = False +class _ShellCommands: + def __init__(self) -> None: + self.active = 0 + self.last_started: object | None = None + + @contextmanager + def running(self) -> Iterator[None]: + self.active += 1 + self.last_started = object() + try: + yield + finally: + self.active -= 1 + + +def _track_shell_command(environment: object) -> AbstractContextManager[None]: + commands = getattr(environment, "_shell_commands", None) + return commands.running() if commands is not None else nullcontext() + + class Environment: """Abstract execution environment used by tools and workflows.""" @@ -39,6 +61,7 @@ class Environment: def __init__(self) -> None: self._aborted = False + self._shell_commands = _ShellCommands() @property def revoked(self) -> bool: diff --git a/opencollab/adapters/hooks.py b/opencollab/adapters/hooks.py index 4db25095..09032782 100644 --- a/opencollab/adapters/hooks.py +++ b/opencollab/adapters/hooks.py @@ -11,6 +11,7 @@ from collections.abc import Awaitable, Callable from typing import Any +from opencollab.adapters._env_base import _track_shell_command from opencollab.adapters._env_process import ProcessCleanupError, run_process from opencollab.application.ports import HookPort from opencollab.domain.hooks import TOOL_SCOPED_EVENTS, HookOutcome, HookSpec, match_hooks @@ -94,7 +95,8 @@ async def _run_command(self, spec: HookSpec, payload: dict[str, Any]) -> None: f"printf '%s' {shlex.quote(stdin_bytes.decode())} | " f"{assignments} sh -c {shlex.quote(spec.command)}" ) - result = await execution_environment.exec_cmd(command, timeout=timeout) + with _track_shell_command(execution_environment): + result = await execution_environment.exec_cmd(command, timeout=timeout) if getattr(result, "timed_out", False): logger.warning("hook command timed out after %.2fs: %s", timeout, spec.command) return diff --git a/opencollab/adapters/tools/bash.py b/opencollab/adapters/tools/bash.py index 44305288..63ef8799 100644 --- a/opencollab/adapters/tools/bash.py +++ b/opencollab/adapters/tools/bash.py @@ -4,6 +4,7 @@ from typing import Any +from opencollab.adapters._env_base import _track_shell_command from opencollab.adapters.tools._output import require_positive_int, truncate from opencollab.adapters.tools.base import Tool from opencollab.application.tool_execution import ToolRuntime @@ -116,7 +117,8 @@ async def execute_with_runtime( if safety_policy: await safety_policy.check_cmd_interactive(cmd, runtime.confirm_fn()) - result = await env.exec_cmd(cmd, timeout=timeout) + with _track_shell_command(env): + result = await env.exec_cmd(cmd, timeout=timeout) observations = getattr(runtime, "observations", None) if observations is not None: observations.record_execution( diff --git a/opencollab/bootstrap/authored_workflow.py b/opencollab/bootstrap/authored_workflow.py index fae8e606..730c6a07 100644 --- a/opencollab/bootstrap/authored_workflow.py +++ b/opencollab/bootstrap/authored_workflow.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, get_args from opencollab.adapters._docker_snapshot import _copy_artifact_to_host +from opencollab.adapters._env_base import ExecResult, _track_shell_command from opencollab.adapters._env_process import ProcessCleanupError from opencollab.adapters.session_environments import SessionEnvironmentPool from opencollab.adapters.tools._paths import checked_path @@ -345,13 +346,15 @@ def _account_usage(self, child: _Child, tokens: Any, *, reported: bool = True) - child.tokens += delta async def _execute(self, child: _Child, command: str, request_bytes: bytes) -> None: + async def run() -> ExecResult: + with _track_shell_command(child.environment): + return await child.environment.exec_cmd(command, timeout=math.inf, input_bytes=request_bytes) + execution = None logs: dict[str, Any] = {} status = "running" try: - execution = asyncio.create_task(child.environment.exec_cmd( - command, timeout=math.inf, input_bytes=request_bytes, - )) + execution = asyncio.create_task(run()) child.execution = execution if child.stop_requested: execution.cancel() diff --git a/opencollab/bootstrap/authored_workflow_bridge.py b/opencollab/bootstrap/authored_workflow_bridge.py index 5743ff70..412ebbe4 100644 --- a/opencollab/bootstrap/authored_workflow_bridge.py +++ b/opencollab/bootstrap/authored_workflow_bridge.py @@ -8,13 +8,13 @@ import posixpath import time from collections.abc import AsyncIterator, Sequence -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, nullcontext from contextvars import ContextVar from dataclasses import asdict from pathlib import Path from typing import Any, Callable -from opencollab.adapters._env_base import Environment, ExecResult, TextFileRange +from opencollab.adapters._env_base import Environment, ExecResult, TextFileRange, _track_shell_command from opencollab.adapters.llm import errors as llm_errors from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.llm.types import LLMResponse, Usage, to_plain_data @@ -371,13 +371,14 @@ def admitted() -> None: if "input_bytes" in arguments: arguments["input_bytes"] = base64.b64decode(arguments["input_bytes"], validate=True) edit_id = request.get("edit") - if edit_id is None: - result = await getattr(environment, action)(**arguments) - else: - edit = self._edits.get(edit_id) - if edit is None or edit.environment is not environment: - raise ValueError("unknown file edit lock for this environment") - result = await edit.call(action, arguments) + with _track_shell_command(environment) if action == "exec_cmd" else nullcontext(): + if edit_id is None: + result = await getattr(environment, action)(**arguments) + else: + edit = self._edits.get(edit_id) + if edit is None or edit.environment is not environment: + raise ValueError("unknown file edit lock for this environment") + result = await edit.call(action, arguments) if isinstance(result, (ExecResult, TextFileRange)): result = asdict(result) if result.get("stdout_bytes") is not None: diff --git a/tests/runtime/test_authored_workflow_bridge.py b/tests/runtime/test_authored_workflow_bridge.py index 32be958f..3782ec39 100644 --- a/tests/runtime/test_authored_workflow_bridge.py +++ b/tests/runtime/test_authored_workflow_bridge.py @@ -1,18 +1,24 @@ from __future__ import annotations import asyncio +import json import os +import shlex +import sys from dataclasses import asdict from types import SimpleNamespace import pytest +from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace from opencollab.adapters.env import DockerEnvironment, LocalEnvironment +from opencollab.adapters.hooks import ShellHookRunner from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.llm.errors import TransientProviderError from opencollab.adapters.llm.request_admission import request_slot from opencollab.adapters.llm.types import LLMResponse, Usage from opencollab.adapters.session_environments import SessionEnvironmentPool +from opencollab.adapters.tools.bash import BashTool from opencollab.adapters.tools.fs import FileWriteTool from opencollab.application._session_run_shared import _TeamBudgetStop, _TokenBudgetStop from opencollab.application.ports import ( @@ -22,6 +28,7 @@ ) from opencollab.application.run_control import _RunCancelled from opencollab.application.tool_execution import ToolRuntime +from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError from opencollab.bootstrap.authored_workflow_bridge import ( AuthoredWorkflowBridge, BridgeClient, @@ -31,6 +38,9 @@ ) from opencollab.bootstrap.session_factory import build_session from opencollab.domain.agent import Agent +from opencollab.domain.hooks import HookSpec +from tests.interfaces.test_member_authored_workflow import _launch, _manager +from tests.workflows.test_workflow_candidate_workspace import _git @pytest.mark.parametrize("error", [ @@ -48,6 +58,124 @@ def test_bridge_restores_usage_on_control_errors(error): assert vars(restored) == {**vars(error), "usage": usage} +@pytest.mark.parametrize("writer_kind", ["bridge", "authored-process", "tool-hook"]) +async def test_managed_commands_keep_parked_candidate_bytes(tmp_path, monkeypatch, writer_kind): + monkeypatch.chdir(tmp_path) + workspace = tmp_path / "workspace" + workspace.mkdir() + _git(workspace, "init", "-q") + (workspace / "record.txt").write_text("original\n") + _git(workspace, "add", "record.txt") + _git(workspace, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare bridged writer") + (workspace / "settings").mkdir() + (workspace / "settings/notes.txt").write_text("Existing settings\n") + patch = ( + "diff --git a/record.txt b/record.txt\n--- a/record.txt\n+++ b/record.txt\n" + "@@ -1 +1 @@\n-original\n+candidate\n" + "diff --git a/settings b/settings\nnew file mode 100644\n--- /dev/null\n+++ b/settings\n" + "@@ -0,0 +1 @@\n+replacement\n" + ) + (workspace / "authored.py").write_text( + "import asyncio, os\n" + "from pathlib import Path\n" + "async def main(ctx, args):\n" + " with open('record.txt', 'a') as stream:\n" + " Path('writer.ready').touch()\n" + " while not Path('writer.release').is_file(): await asyncio.sleep(0.01)\n" + " stream.write('authored append\\n')\n" + " stream.flush()\n" + " os.fsync(stream.fileno())\n" + " Path('writer.flushed').touch()\n" + " return 'saved'\n" + "if __name__ == '__main__': asyncio.run(main(None, {}))\n" + ) + index = (workspace / ".git/index").read_bytes() + host = LocalEnvironment(str(workspace)) + mailbox = LocalEnvironment(str(tmp_path)) + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + bridge = AuthoredWorkflowBridge( + mailbox, str(artifacts), SessionEnvironmentPool(str(tmp_path), isolation="shared", base_environment=mailbox), + {}, ToolRuntime(mailbox, None, None), allow_unisolated_shell=True, + ) + bridge._environments[1] = host + proxy = BridgeEnvironment(BridgeClient(artifacts, bridge.options()), { + "handle": 1, "workspace": str(workspace), "process_isolated": False, + }) + parent, manager, runtime = _manager(host) if writer_kind == "authored-process" else (None, None, None) + published, permit_recovery, stop = asyncio.Event(), asyncio.Event(), asyncio.Event() + native_exec = host.exec_cmd + + async def pause_failed_apply(cmd, *args, **kwargs): + result = await native_exec(cmd, *args, **kwargs) + if " apply --binary " in cmd and ".candidate-adopt-" in cmd: + assert result.returncode == 128 + published.set() + await permit_recovery.wait() + return result + + monkeypatch.setattr(host, "exec_cmd", pause_failed_apply) + + async def serve(): + while not stop.is_set(): + await bridge.poll() + await asyncio.sleep(0.005) + + async def writer_ready(): + while not (workspace / "writer.ready").is_file(): + if writer.done(): + writer.result() + raise AssertionError("writer finished before opening the file") + await asyncio.sleep(0.005) + + async def run_writer(): + command = shlex.join((sys.executable, "authored.py")) + if writer_kind == "bridge": + outcome = await BashTool(require_process_isolation=False).execute_with_runtime( + {"command": command}, ToolRuntime(proxy, None, None), + ) + assert "Exit code: 0" in outcome + elif writer_kind == "authored-process": + started = await _launch(manager, runtime, isolation="shared") + outcome = json.loads(await manager.execute_with_runtime({"action": "wait", "id": started["id"]}, runtime)) + assert outcome["status"] == "completed" + assert outcome["result"]["output"] == "saved" + else: + hook = HookSpec(event="PreToolUse", action_type="command", command=command) + runner = ShellHookRunner((hook,), scheduler=SimpleNamespace(agent_environment=lambda _: host)) + await runner.fire("PreToolUse", {"hook_event_name": "PreToolUse", "aid": 0, "tool": "bash"}) + assert (workspace / "writer.flushed").is_file() + + server = asyncio.create_task(serve()) + publication = asyncio.create_task(EnvCandidateWorkspace(host).adopt(patch)) + writer = None + try: + await asyncio.wait_for(published.wait(), 10) + writer = asyncio.create_task(run_writer()) + await asyncio.wait_for(writer_ready(), 10) + permit_recovery.set() + with pytest.raises(CandidateAdoptionRecoveryError, match="parked files retained"): + await asyncio.wait_for(publication, 10) + (workspace / "writer.release").touch() + await asyncio.wait_for(writer, 10) + saved, = (workspace / ".git").glob("candidate-recovery-*/observed-*/entry") + assert saved.read_text() == "candidate\nauthored append\n" + assert (workspace / "record.txt").read_text() == "original\n" + assert (workspace / ".git/index").read_bytes() == index + finally: + permit_recovery.set() + (workspace / "writer.release").touch() + await asyncio.gather(publication, *([writer] if writer else []), return_exceptions=True) + stop.set() + await server + await bridge.aclose() + if parent is not None: + await parent.aclose() + await mailbox.cleanup() + await host.cleanup() + + @pytest.mark.parametrize("docker", [False, True]) async def test_bridge_retains_native_whole_edit_lock(tmp_path, monkeypatch, docker): image = os.environ.get("OPENCOLLAB_DOCKER_TEST_IMAGE") diff --git a/tests/workflows/test_candidate_adoption_recovery.py b/tests/workflows/test_candidate_adoption_recovery.py index f9d818ee..370c072a 100644 --- a/tests/workflows/test_candidate_adoption_recovery.py +++ b/tests/workflows/test_candidate_adoption_recovery.py @@ -88,6 +88,25 @@ def _flatten(folder): (folder / "draft.txt").write_text("candidate draft\n") +def _partial_publication(tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q") + (repo / "record.txt").write_text("original\n") + _git(repo, "add", "record.txt") + _git(repo, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare partial publication") + (repo / "settings").mkdir() + (repo / "settings/notes.txt").write_text("existing settings\n") + patch = ( + "diff --git a/record.txt b/record.txt\n--- a/record.txt\n+++ b/record.txt\n" + "@@ -1 +1 @@\n-original\n+candidate\n" + "diff --git a/settings b/settings\nnew file mode 100644\n--- /dev/null\n+++ b/settings\n" + "@@ -0,0 +1 @@\n+replacement settings\n" + ) + return repo, CandidateRun("record update", None, patch, (), ()) + + @pytest.mark.parametrize("physical", [False, True]) @pytest.mark.parametrize("cache", [False, True]) async def test_public_sdk_candidate_recovers_directory_replacement(tmp_path, physical, cache): @@ -552,6 +571,67 @@ async def after_comparison(command, timeout=120.0, **kwargs): await environment.cleanup() +@pytest.mark.parametrize("linked", [False, True]) +async def test_recovery_compares_parked_bytes_before_removing_the_backup(tmp_path, linked): + source, candidate = _partial_publication(tmp_path) + repo = source + if linked: + repo = tmp_path / "linked" + _git(source, "worktree", "add", "--detach", str(repo)) + (repo / "settings").mkdir() + (repo / "settings/notes.txt").write_text("existing settings\n") + target = repo / "record.txt" + original = target.read_bytes() + index = Path(_git(repo, "rev-parse", "--absolute-git-dir").removesuffix("\n")) / "index" + index_before = index.read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + writer = None + appended = b"# independent write\n" + published = None + + async def observe(command, timeout=120.0, **kwargs): + nonlocal writer, published + if command.startswith("recovery="): + assert writer is not None + assert target.read_bytes() == original + writer.write(appended) + writer.flush() + os.fsync(writer.fileno()) + writer.close() + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command and ".candidate-adopt-" in command: + assert result.returncode == 128, result.stderr + published = target.read_bytes() + assert published != original + writer = target.open("ab") + return result + + environment.exec_cmd = observe + + async def flow(ctx, _): + await ctx.adopt_candidate(candidate) + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow(flow, trace=False) + assert isinstance(result.error, CandidateAdoptionRecoveryError) + assert f"concurrent change at {target}" in result.reason + backup, = (source / ".git").glob("candidate-recovery-*") + assert str(backup) in result.reason + record, = (record for record in backup.glob("observed-*/path") if record.read_text() == str(target)) + assert record.with_name("entry").read_bytes() == published + appended + assert target.read_bytes() == original + assert index.read_bytes() == index_before + finally: + if writer is not None: + writer.close() + await environment.cleanup() + if linked: + _git(source, "worktree", "remove", "--force", str(repo)) + assert not index.exists() + assert record.with_name("entry").read_bytes() == published + appended + + @pytest.mark.parametrize("path", [".", ".gi\u200ct/config"]) async def test_git_rejects_metadata_aliases_before_candidate_backup(tmp_path, path): repo = _repository(tmp_path) diff --git a/tests/workflows/test_candidate_shared_mutations.py b/tests/workflows/test_candidate_shared_mutations.py index dc51e349..1d853431 100644 --- a/tests/workflows/test_candidate_shared_mutations.py +++ b/tests/workflows/test_candidate_shared_mutations.py @@ -2,6 +2,8 @@ import asyncio import json +import shlex +import sys from pathlib import Path import pytest @@ -11,7 +13,7 @@ from opencollab.adapters.llm.types import LLMResponse, Usage from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError from opencollab.tools import builtin_tools -from tests.workflows.test_candidate_adoption_recovery import _flatten, _repository, _run +from tests.workflows.test_candidate_adoption_recovery import _flatten, _partial_publication, _repository, _run from tests.workflows.test_workflow_candidate_workspace import _git @@ -38,8 +40,9 @@ async def exec_cmd(self, command, timeout=120, **kwargs): class _WriterModel: - def __init__(self, tool, path): + def __init__(self, tool, path, *, command=None): self.tool, self.path, self.calls = tool, path, 0 + self.command = command self.requested = asyncio.Event() async def complete(self, messages, tools=None, **kwargs): @@ -48,7 +51,7 @@ async def complete(self, messages, tools=None, **kwargs): self.requested.set() arguments = {"path": self.path} if self.tool == "bash": - arguments = {"command": f"printf '%s\\n' 'shared replacement' > {self.path}"} + arguments = {"command": self.command or f"printf '%s\\n' 'shared replacement' > {self.path}"} elif self.tool == "file_write": arguments.update(mode="create", content="VALUE = 100\n") else: @@ -211,3 +214,106 @@ async def flow(ctx, _): assert not backups finally: await environment.cleanup() + + +class _ParkingBarrier(_PublicationBarrier): + def __init__(self, workspace): + super().__init__(workspace, after_write=True) + self.parking = asyncio.Event() + self.allow_parking = asyncio.Event() + self.parked = asyncio.Event() + self.return_recovery = asyncio.Event() + + async def exec_cmd(self, command, timeout=120, **kwargs): + recovery = command.startswith("failed=0;") + if recovery: + self.parking.set() + await self.allow_parking.wait() + result = await super().exec_cmd(command, timeout, **kwargs) + if recovery: + self.parked.set() + await self.return_recovery.wait() + return result + + +@pytest.mark.parametrize("finished_inside_recovery", [False, True]) +async def test_sdk_shared_bash_retains_writes_through_an_open_file(tmp_path, finished_inside_recovery): + repo, candidate = _partial_publication(tmp_path) + target = repo / "record.txt" + original = target.read_bytes() + index = (repo / ".git/index").read_bytes() + environment = _ParkingBarrier(repo) + ready = repo / "writer.ready" + release = repo / "writer.release" + appended = b"# shared shell bytes\n" + command = shlex.join((sys.executable, "-c", f""" +import os +import time +from pathlib import Path + +with Path({target.name!r}).open("ab") as stream: + Path({ready.name!r}).touch() + deadline = time.monotonic() + 10 + while not Path({release.name!r}).exists(): + if time.monotonic() >= deadline: + raise TimeoutError("waiting for recovery") + time.sleep(0.01) + stream.write({appended!r}) + stream.flush() + os.fsync(stream.fileno()) +""")) + + async def wait_for_writer(): + while not ready.exists(): + await asyncio.sleep(0.01) + + async def flow(ctx, _): + publication = asyncio.create_task(ctx.adopt_candidate(candidate)) + writer = None + try: + await asyncio.wait_for(environment.entered.wait(), 10) + published = target.read_bytes() + assert published != original + if finished_inside_recovery: + environment.release.set() + await asyncio.wait_for(environment.parking.wait(), 10) + writer = asyncio.create_task(ctx.agent_run( + "Save the shared Bash result", tools=list(builtin_tools("bash", headless=False)), + isolation="shared", budget=2000, system_prompt="Run the shell command", + )) + await asyncio.wait_for(wait_for_writer(), 10) + environment.release.set() + environment.allow_parking.set() + await asyncio.wait_for(environment.parked.wait(), 10) + if finished_inside_recovery: + release.touch() + assert (await asyncio.wait_for(writer, 10)).status == "completed" + environment.return_recovery.set() + with pytest.raises(CandidateAdoptionRecoveryError, match="concurrent shell command.*retained"): + await asyncio.wait_for(asyncio.shield(publication), 10) + release.touch() + assert (await asyncio.wait_for(writer, 10)).status == "completed" + return published + finally: + environment.release.set() + environment.allow_parking.set() + environment.return_recovery.set() + release.touch() + await asyncio.gather(publication, *([writer] if writer else []), return_exceptions=True) + + try: + result = await OpenCollab(repo, environment=environment, model="local-scripted").workflow( + flow, llm=_WriterModel("bash", target.name, command=command), trace=False, timeout=40, + budget=5000, concurrency=2, limit_mode="explicit", agent_profile="single2", + ) + assert result.ok, result.reason + assert environment.publications == [128] + assert target.read_bytes() == original + assert (repo / "settings/notes.txt").read_text() == "existing settings\n" + assert (repo / ".git/index").read_bytes() == index + retained, = (repo / ".git").glob("candidate-recovery-*") + record, = (record for record in retained.glob("observed-*/path") if record.read_text() == str(target)) + assert record.with_name("entry").read_bytes() == result.output + appended + assert not environment.revoked + finally: + await environment.cleanup() From be259681343c2ce75af68336aaa02c49a119d5f4 Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 11:22:37 +0800 Subject: [PATCH 48/82] test: express UTF-8 fixture values with Unicode escapes --- tests/agents/test_session_persistence.py | 4 ++-- tests/runtime/test_anthropic_preserved_history.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/agents/test_session_persistence.py b/tests/agents/test_session_persistence.py index 57feb16d..413bb7c3 100644 --- a/tests/agents/test_session_persistence.py +++ b/tests/agents/test_session_persistence.py @@ -175,7 +175,7 @@ def test_journal_replacement_uses_the_durable_prefix_and_requires_a_new_sequence journal = path.with_suffix(".json.journal") store = SessionStore() base = [{"role": "system", "content": "prompt"}] - archive = {"role": "user", "content": "€" * 1024} + archive = {"role": "user", "content": "\u20ac" * 1024} replacement = [*base, archive, {"role": "assistant", "content": "latest"}] store.checkpoint_snapshot(str(path), base, meta={}, sequence=1) store.append_snapshot_delta( @@ -190,7 +190,7 @@ def test_journal_replacement_uses_the_durable_prefix_and_requires_a_new_sequence assert not store.append_snapshot_replacement(str(path), replacement, meta=meta, sequence=sequence) assert journal.read_bytes() == before - edited_archive = {"role": "user", "content": "雪" * 1024} + edited_archive = {"role": "user", "content": "\u96ea" * 1024} with pytest.raises(ValueError, match="autosave journal exceeds"): store.append_snapshot_replacement(str(path), [*base, edited_archive], meta=meta, sequence=3) assert journal.read_bytes() == before diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index 1898c24b..fc7450e6 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -691,7 +691,7 @@ async def test_native_rewrite_recovers_tool_result_when_checkpoint_fails(monkeyp monkeypatch.setattr(storage_module, "MAX_SESSION_SNAPSHOT_BYTES", limit) monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") size = (limit - 14_000) // 2 if failure in {"capacity", "format"} else 1_000 - archive = _blocks("é" * size) + _blocks("x") * 256 + archive = _blocks("\u00e9" * size) + _blocks("x") * 256 path = tmp_path / "fallback.json" journal = path.with_suffix(".json.journal") held_request, resume = asyncio.Event(), asyncio.Event() From 1ba940c76a4d24564d13950277f9a9df72a1d21b Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 12:29:36 +0800 Subject: [PATCH 49/82] fix(workflow): preserve detached files during candidate rollback Keep the original and parked inodes reachable after rollback so independent writers can finish writing through held descriptors. Report the retained data path and discard replay scratch after a verified rollback. Exercise late writes across linked worktree removal and preserve ordinary fallback when a failed publication detached only original files. --- docs/duo.md | 6 + docs/duo/README.zh-CN.md | 2 + opencollab/adapters/_candidate_adoption.py | 137 ++++++++--- .../test_candidate_adoption_command_size.py | 6 +- ...test_candidate_adoption_external_writer.py | 225 ++++++++++++++++++ .../test_candidate_adoption_recovery.py | 52 +++- .../test_candidate_mutation_backends.py | 8 +- .../test_candidate_shared_mutations.py | 45 +++- 8 files changed, 437 insertions(+), 44 deletions(-) create mode 100644 tests/workflows/test_candidate_adoption_external_writer.py diff --git a/docs/duo.md b/docs/duo.md index df36840a..f3b8def5 100644 --- a/docs/duo.md +++ b/docs/duo.md @@ -204,6 +204,12 @@ Git candidates start with the source's current tracked files and untracked files that aren't ignored. Their delivered patches describe the subsequent candidate edits. Adoption applies that increment to the current source files and preserves independent user edits and the source index. A conflicting patch leaves the current source in place. +If a partially applied patch is rolled back, OpenCollab retains detached files that +another process may still be writing to. The error or warning gives the +`candidate-recovery-*` directory, usually in the common Git directory and beside +the worktree when Git metadata is on another filesystem. The adjacent `path` or +`.path` record identifies each file's original location. Close the writers before +recovering their data and removing the recovery directory. Changes to ignore rules preserve files copied from the source and files added to the candidate index or commits. Newly created ignored files join delivery when explicitly added with `git add --force`. diff --git a/docs/duo/README.zh-CN.md b/docs/duo/README.zh-CN.md index 213f294a..c030ee90 100644 --- a/docs/duo/README.zh-CN.md +++ b/docs/duo/README.zh-CN.md @@ -89,6 +89,8 @@ Duo 通过现有候选工作区接口完成隔离、变更捕获与采用。默 Git 候选从源目录当前受跟踪文件和未被忽略的未跟踪文件开始,交付补丁表示候选随后产生的增量。采用时将增量应用到当前源文件,保留独立用户修改和源索引,冲突会保留当前源目录。修改忽略规则后,已从源目录复制的文件和候选中已加入索引或提交的文件仍进入交付。新建的忽略文件经 `git add --force` 加入后进入交付。 +补丁部分应用后回滚时,OpenCollab 会保留其他进程可能仍在写入的文件。错误或警告会给出 `candidate-recovery-*` 目录。它通常在 Git 公共目录下,Git 元数据位于另一文件系统时则放在工作树旁。相邻的 `path` 或 `.path` 记录标明文件原来的位置。等待写入进程关闭文件后,再恢复其中的数据和清理该目录。 + 候选运行期间源目录发生变化,运行时会报告错误并保留候选工作树。错误包含初始 Git tree,工作树中的 `refs/worktree/opencollab-source` 继续引用它,可以通过 `git show :` 恢复文件。后端提供源版本时,`CandidateRun.source_revision` 记录源 `HEAD`,采用前再次核对。源 diff 相同而 `HEAD` 已变化时,采用也会被拒绝。 本地 Git 候选初始化源目录中已有的干净子模块,包含选定工作区内的嵌套依赖。尚未初始化的可选子模块保持空目录。源子模块的变化会在候选获取前报告。候选修改子模块文件、提交或 gitlink 时,`CandidateCaptureError` 会保留完整工作树供恢复。仓库补丁后端交付父仓库的改动,子模块修改需要对应的交付后端。 diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index 346d8138..b5ed08d1 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import posixpath import shlex from collections.abc import AsyncIterator, Awaitable, Callable, Sequence @@ -11,8 +12,11 @@ from opencollab.adapters._env_base import ENV_FILE_WRITE_LIMIT_BYTES from opencollab.adapters._git_patch_bytes import _complete, _patch_temporary from opencollab.application.async_timeout import await_owned_operation +from opencollab.application.exception_notes import add_exception_note from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError +logger = logging.getLogger(__name__) + _COMPARE_ENTRIES = r''' present() { [ -e "$1" ] || [ -L "$1" ]; } mode() { stat -c %a -- "$1" 2>/dev/null || stat -f %Lp -- "$1"; } @@ -33,9 +37,8 @@ conflict() { printf 'concurrent change at %s\n' "$1" >&2; return 2; } ''' -_RECOVER_PATHS = _COMPARE_ENTRIES + r''' +_CHECK_PARENTS = r''' directory() { [ -d "$1" ] && [ ! -L "$1" ]; } -device() { stat -c %d -- "$1" 2>/dev/null || stat -f %d -- "$1"; } check_parents() { if [ "$1" = "$workspace" ]; then directory "$workspace" || conflict "$workspace"; return; fi @@ -46,6 +49,44 @@ done directory "$workspace" || conflict "$workspace" } +''' + +_HOLD_ORIGINALS = _COMPARE_ENTRIES + _CHECK_PARENTS + r''' +hold_originals() { + index=0 + mkdir -m 700 -- "$recovery/original-inodes" || return 1 + for path; do + source=$workspace_prefix$path + if check_parents "$source" 2>/dev/null && [ -f "$source" ] && [ ! -L "$source" ]; then + entry=$recovery/original-inodes/$index + ln -P -- "$source" "$entry" && printf '%s' "$source" > "$entry.path" || return 1 + index=$((index + 1)) + fi + done +} +''' + +_DEVICE = r''' +device() { stat -c %d -- "$1" 2>/dev/null || stat -f %d -- "$1"; } +''' + +_RECOVERY_PARENT = _DEVICE + r''' +recovery_parent() { + worktree_device=$(device .) && storage_device=$(device "$1/") || return 1 + if [ "$worktree_device" = "$storage_device" ]; then + printf '%s\n' "$1" + return + fi + cd -P -- .. && parent_device=$(device .) || return 1 + if [ "$worktree_device" != "$parent_device" ]; then + printf 'no recovery directory outside the worktree on its filesystem\n' >&2 + return 1 + fi + pwd -P +} +''' + +_RECOVER_PATHS = _COMPARE_ENTRIES + _CHECK_PARENTS + _DEVICE + r''' check_candidate_descendants() { prefix=${1#"$workspace_prefix"}; shift @@ -274,15 +315,22 @@ async def _adoption_transaction( raise RuntimeError("candidate ancestor directory mode is invalid") directory_modes.append((path, mode)) git_directory = None - storage = '"${TMPDIR:-/tmp}"' + directory_script = f"cd -- {shlex.quote(workspace)} || exit\n" + _RECOVERY_PARENT + if repository: + # Git removes private metadata with a linked worktree. A worktree on + # another volume needs a recovery directory beside it on that volume. + directory_script += r''' +git rev-parse --absolute-git-dir && printf '\0' || exit +common=$(git rev-parse --git-common-dir && printf .) || exit +recovery_parent "${common%??}" +''' + else: + directory_script += 'recovery_parent "${TMPDIR:-/tmp}"' + recovery_parent = _complete(await environment.exec_cmd(directory_script, timeout=30), "candidate recovery storage") if repository: - git = f"git -C {shlex.quote(workspace)}" - layout = _complete(await environment.exec_cmd( - f"{git} rev-parse --absolute-git-dir && printf '\\0' && {git} rev-parse --git-common-dir", timeout=30, - ), "candidate recovery storage") - git_directory, common_directory = (path.removesuffix("\n") for path in layout.split("\0")) - # Git removes the private metadata along with a linked worktree. - storage = shlex.quote(posixpath.join(workspace, common_directory)) + git_directory, recovery_parent = recovery_parent.split("\0") + git_directory = git_directory.removesuffix("\n") + storage = shlex.quote(posixpath.join(workspace, recovery_parent.removesuffix("\n"))) backup = None keep_backup = False @@ -292,20 +340,27 @@ async def allocate_backup() -> None: f'mktemp -d {storage}/candidate-recovery-XXXXXXXX', timeout=30, ), "candidate recovery storage").removesuffix("\n") - async def remove_backup(path: str, *, verify: bool = False) -> None: + async def remove_backup(path: str, *, retain_detached: bool = False) -> None: quoted = shlex.quote(path) - removal = ( - f"rm -rf -- {quoted} || " - f"(find {quoted} -type d ! -perm -u+w -exec chmod -- u+w {{}} + && rm -rf -- {quoted})" + remove_entry = ( + 'remove_entry() { rm -rf -- "$1" || ' + '(find "$1" -type d ! -perm -u+w -exec chmod -- u+w {} + && rm -rf -- "$1"); }\n' ) - if verify: + if retain_detached: removal = ( f"recovery={quoted}; staged={shlex.quote(posixpath.join(path, 'replay'))}; " f"workspace_prefix={shlex.quote(workspace.rstrip('/') + '/')};\n" - + _RECHECK_PARKED + removal + + _RECHECK_PARKED + remove_entry + r''' +for entry in "$recovery"/*; do + case "${entry##*/}" in original-inodes|observed-*) continue;; esac + remove_entry "$entry" || exit +done +''' ) + else: + removal = remove_entry + f"remove_entry {quoted}" result = await environment.exec_cmd(removal, timeout=120) - if verify: + if retain_detached: _complete(result, "candidate recovery cleanup") try: @@ -378,25 +433,49 @@ async def remove_backup(path: str, *, verify: bool = False) -> None: *stage_copies, *reversed(stage_modes), )) targets = [shlex.quote(path) for path in paths] + # Git can unlink the original before installing the candidate. + workspace_vars = ( + f"workspace={shlex.quote(workspace)}; recovery={shlex.quote(backup)};\n" + 'workspace_prefix="${workspace%/}/";\n' + ) + snapshot_script = ( + workspace_vars + _HOLD_ORIGINALS + f"hold_originals {' '.join(targets)} || exit; " + + "; ".join(copies) + ) target_test = f'case "${{1#"$workspace_prefix"}}" in {"|".join(targets)}) return 0;; esac; ' if targets else "" recovery_script = ( - f"failed=0; workspace={shlex.quote(workspace)}; " - f"recovery={shlex.quote(backup)}; " + f"failed=0; {workspace_vars}" f"absent={shlex.quote(posixpath.join(backup, 'absent'))};\n" - 'workspace_prefix="${workspace%/}/";\n' f"candidate_path() {{ {target_test}return 1; }}\n" f"candidate_descendants() {{ check_candidate_descendants \"$1\" {' '.join(targets)}; }}\n" ) + _RECOVER_PATHS + "; ".join(( *recreate_directories, *restores, *reversed(restore_directory_modes), )) + r'''; if [ "$failed" -ne 0 ]; then exit "$failed"; fi +detached=0 +for record in "$recovery"/original-inodes/*.path; do + [ -f "$record" ] || continue + path=$(cat -- "$record" && printf .) || exit + path=${path%.} + entry=${record%.path} + if check_parents "$path" 2>/dev/null && [ ! -L "$path" ] && [ "$path" -ef "$entry" ]; then + rm -- "$record" "$entry" || exit + else + detached=1 + fi +done for parked in "$recovery"/observed-*/entry; do if [ -f "$parked" ] && [ ! -L "$parked" ]; then printf 'parked\n'; break; fi done +if [ "$detached" -eq 1 ]; then + printf 'original-detached\n' +else + rmdir -- "$recovery/original-inodes" || exit +fi ''' async with AsyncExitStack() as commands: snapshot_command = await commands.enter_async_context(_adoption_command( - environment, workspace, "; ".join(copies), repository=repository, + environment, workspace, snapshot_script, repository=repository, )) # Store long recovery commands before publishing any source changes. recovery_command = await commands.enter_async_context(_adoption_command( @@ -422,10 +501,12 @@ async def recover() -> tuple[bool, bool]: shell_overlapped = shell_commands is not None and ( active > 0 or last_started is not shell_commands.last_started ) - return "parked" in receipt.splitlines(), shell_overlapped + entries = receipt.splitlines() + parked = "parked" in entries + return parked or "original-detached" in entries, parked and shell_overlapped try: - parked, shell_overlapped = await await_owned_operation(recover()) + detached, parked_during_shell = await await_owned_operation(recover()) except BaseException as recovery: keep_backup = True revoke = getattr(environment, "revoke", None) @@ -435,9 +516,9 @@ async def recover() -> tuple[bool, bool]: f"candidate adoption failed ({failure}); recovery failed ({recovery}); " f"original files retained in {backup}" ) from recovery - if parked: + if detached: keep_backup = True - if shell_overlapped: + if parked_during_shell: # A command can still own a detached inode after its original path is restored. raise CandidateAdoptionRecoveryError( f"candidate adoption failed ({failure}); a concurrent shell command may have written " @@ -445,13 +526,15 @@ async def recover() -> tuple[bool, bool]: ) from failure assert backup is not None try: - await await_owned_operation(remove_backup(backup, verify=True)) + await await_owned_operation(remove_backup(backup, retain_detached=True)) except BaseException as cleanup: raise CandidateAdoptionRecoveryError( f"candidate adoption failed ({failure}); cleanup failed ({cleanup}); " f"recovery location {backup}" ) from cleanup - backup = None + detail = f"Detached candidate files retained in {backup}" + add_exception_note(failure, detail) + logger.warning("%s", detail) raise finally: if backup is not None and not keep_backup: diff --git a/tests/workflows/test_candidate_adoption_command_size.py b/tests/workflows/test_candidate_adoption_command_size.py index af21e3ed..9ef32951 100644 --- a/tests/workflows/test_candidate_adoption_command_size.py +++ b/tests/workflows/test_candidate_adoption_command_size.py @@ -7,6 +7,7 @@ from opencollab import OpenCollab from opencollab.adapters._env_local import LocalEnvironment +from tests.workflows.test_candidate_adoption_recovery import _detached_files from tests.workflows.test_workflow_candidate_workspace import _git @@ -67,10 +68,13 @@ async def flow(ctx, inputs): if blocked: assert (repo / "settings/value.txt").read_text() == "original settings\n" assert (repo / "settings/session.cache").read_text() == "runtime cache\n" + originals, _ = _detached_files(repo / ".git") + assert originals[repo / "settings/value.txt"] == b"original settings\n" + assert all(originals[repo / "src" / name] == b"VALUE = 1\n" for name in names) else: assert (repo / "settings").read_text() == "new settings\n" + assert not list((repo / ".git").glob("candidate-recovery-*")) assert (repo / ".git/index").read_bytes() == index - assert not list((repo / ".git").glob("candidate-recovery-*")) assert not list((repo / ".git").rglob(".candidate-command-*.sh")) finally: await environment.cleanup() diff --git a/tests/workflows/test_candidate_adoption_external_writer.py b/tests/workflows/test_candidate_adoption_external_writer.py new file mode 100644 index 00000000..2b552c29 --- /dev/null +++ b/tests/workflows/test_candidate_adoption_external_writer.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import asyncio +import os +import shlex +import sys +import tempfile +from dataclasses import replace +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.tools.bash import BashTool +from opencollab.application.tool_execution import ToolRuntime +from tests.workflows.test_candidate_adoption_recovery import _partial_publication +from tests.workflows.test_workflow_candidate_workspace import _git + +_WRITER = """ +import os +import sys + +with open(sys.argv[1], "ab") as stream: + opened = os.fstat(stream.fileno()) + print(opened.st_dev, opened.st_ino, flush=True) + data = sys.stdin.buffer.read() + if not data: + raise RuntimeError("writer exited before release") + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) +print("written") +""" + + +def _identity(path: Path) -> tuple[int, int]: + status = path.stat() + return status.st_dev, status.st_ino + + +@pytest.fixture +def worktree_parent(tmp_path, request): + if not request.param: + yield tmp_path + return + shared_memory = Path("/dev/shm") + if not shared_memory.is_dir() or not os.access(shared_memory, os.W_OK): + pytest.skip("requires writable shared memory on a separate filesystem") + if shared_memory.stat().st_dev == tmp_path.stat().st_dev: + pytest.skip("requires Git metadata and worktree on different filesystems") + with tempfile.TemporaryDirectory(prefix="oc-worktree-", dir=shared_memory) as path: + yield Path(path) + + +@pytest.mark.parametrize("opened_before_apply,linked,worktree_parent", [ + pytest.param(True, False, False, id="original-inode"), + pytest.param(False, True, False, id="candidate-inode-after-worktree-removal"), + pytest.param(False, True, True, id="separate-worktree-filesystem"), +], indirect=["worktree_parent"]) +async def test_candidate_rollback_retains_inode_for_external_writer( + tmp_path, monkeypatch, caplog, opened_before_apply, linked, worktree_parent, +): + source, candidate = _partial_publication(tmp_path) + repo = source + if linked: + repo = worktree_parent / "linked" + _git(source, "worktree", "add", "--detach", str(repo)) + (repo / "settings").mkdir() + (repo / "settings/notes.txt").write_text("existing settings\n") + (repo / "user-note.txt").write_text("staged user note\n") + _git(repo, "add", "--", "user-note.txt") + target = repo / "record.txt" + original = target.read_bytes() + original_inode = _identity(target) + published = b"candidate\n" + appended = b"external write after rollback\n" + index = Path(_git(repo, "rev-parse", "--absolute-git-dir").removesuffix("\n")) / "index" + index_before = index.read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + writer = None + writer_inode = None + + async def start_writer(expected: bytes) -> None: + nonlocal writer, writer_inode + assert writer is None + assert target.read_bytes() == expected + writer = await asyncio.create_subprocess_exec( + sys.executable, "-c", _WRITER, str(target), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + assert writer.stdout is not None + ready = await asyncio.wait_for(writer.stdout.readline(), 10) + writer_inode = _identity(target) + assert ready.split() == [str(value).encode() for value in writer_inode] + + async def observe(command, timeout=120.0, **kwargs): + if " apply --binary " not in command or ".candidate-adopt-" not in command: + return await execute(command, timeout, **kwargs) + if opened_before_apply: + await start_writer(original) + outcome = await execute(command, timeout, **kwargs) + assert outcome.returncode == 128, outcome.stderr + assert target.read_bytes() == published + assert _identity(target) != original_inode + if not opened_before_apply: + await start_writer(published) + return outcome + + monkeypatch.setattr(environment, "exec_cmd", observe) + + async def flow(ctx, _): + await ctx.adopt_candidate(candidate) + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow( + flow, trace=False, timeout=30, + ) + assert type(result.error) is RuntimeError, result.reason + assert "candidate adoption failed" in result.reason + assert "settings" in result.reason + assert target.read_bytes() == original + assert (repo / "settings/notes.txt").read_bytes() == b"existing settings\n" + assert index.read_bytes() == index_before + assert writer is not None, result.reason + assert writer_inode is not None + + if linked: + _git(source, "worktree", "remove", "--force", str(repo)) + assert not repo.exists() + assert not index.exists() + assert writer.returncode is None + stdout, stderr = await asyncio.wait_for(writer.communicate(appended), 10) + assert writer.returncode == 0, stderr + assert stdout == b"written\n" + assert (source / "record.txt").read_bytes() == original + + storage = source / ".git" if worktree_parent == tmp_path else worktree_parent + backups = list(storage.glob("candidate-recovery-*")) + assert backups, "candidate rollback left no recovery storage for the open descriptor" + backup, = backups + assert all(path.name == "original-inodes" or path.name.startswith("observed-") for path in backup.iterdir()) + if opened_before_apply: + path_record, = (path for path in backup.glob("original-inodes/*.path") if path.read_text() == str(target)) + retained = path_record.with_suffix("") + initial = original + else: + path_record, = (path for path in backup.glob("observed-*/path") if path.read_text() == str(target)) + retained = path_record.with_name("entry") + initial = published + assert _identity(retained) == writer_inode + assert retained.read_bytes() == initial + appended + assert any(str(backup) in message for message in caplog.messages) + assert any(str(backup) in note for note in getattr(result.error, "__notes__", ())) + finally: + if writer is not None and writer.returncode is None: + writer.kill() + await writer.communicate() + await environment.cleanup() + + +async def test_detached_original_only_preserves_publication_error_during_shell_execution(tmp_path, monkeypatch): + repo, candidate = _partial_publication(tmp_path) + _, replacement = candidate.diff.split("diff --git a/settings", 1) + candidate = replace(candidate, diff=( + "diff --git a/record.txt b/record.txt\ndeleted file mode 100644\n" + "--- a/record.txt\n+++ /dev/null\n@@ -1 +0,0 @@\n-original\n" + "diff --git a/settings" + replacement + )) + target = repo / "record.txt" + original_inode = _identity(target) + index = repo / ".git/index" + index_before = index.read_bytes() + ready = tmp_path / "shell-ready" + release = tmp_path / "shell-release" + command = ( + f"touch {shlex.quote(str(ready))} && " + f"while [ ! -f {shlex.quote(str(release))} ]; do sleep 0.01; done" + ) + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + + async def observe(command, timeout=120.0, **kwargs): + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command and ".candidate-adopt-" in command: + assert result.returncode == 128, result.stderr + assert not target.exists() + return result + + monkeypatch.setattr(environment, "exec_cmd", observe) + + async def wait_for_shell(): + while not ready.exists(): + await asyncio.sleep(0.01) + + async def flow(ctx, _): + shell = asyncio.create_task(BashTool().execute_with_runtime( + {"command": command, "timeout": 20}, ToolRuntime(environment, None, None), + )) + try: + await asyncio.wait_for(wait_for_shell(), 10) + assert not shell.done() + await ctx.adopt_candidate(candidate) + finally: + release.touch() + assert await shell == "Exit code: 0" + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow( + flow, trace=False, timeout=30, + ) + assert type(result.error) is RuntimeError, result.reason + assert "candidate adoption failed" in result.reason + assert target.read_bytes() == b"original\n" + assert index.read_bytes() == index_before + backup, = (repo / ".git").glob("candidate-recovery-*") + record, = (backup / "original-inodes").glob("*.path") + assert record.read_text() == str(target) + assert _identity(record.with_suffix("")) == original_inode + assert not list(backup.glob("observed-*")) + finally: + await environment.cleanup() diff --git a/tests/workflows/test_candidate_adoption_recovery.py b/tests/workflows/test_candidate_adoption_recovery.py index 370c072a..dfaf2a3b 100644 --- a/tests/workflows/test_candidate_adoption_recovery.py +++ b/tests/workflows/test_candidate_adoption_recovery.py @@ -43,6 +43,21 @@ def _contents(root): return result +def _detached_files(storage: Path) -> tuple[dict[Path, bytes], dict[Path, bytes]]: + backup, = storage.glob("candidate-recovery-*") + assert not (backup / "replay").exists() + originals = { + Path(record.read_text()): record.with_suffix("").read_bytes() + for record in backup.glob("original-inodes/*.path") + } + parked = { + Path(record.read_text()): record.with_name("entry").read_bytes() + for record in backup.glob("observed-*/path") + if not record.with_name("entry").is_symlink() + } + return originals, parked + + def _repository(tmp_path, *, physical=False, cache=False): repo = tmp_path / "repo" repo.mkdir() @@ -134,7 +149,12 @@ async def flow(ctx, inputs): assert result.ok, result.reason assert _run(repo) == ("1\n" if cache else "42\n") assert (repo / ".git/index").read_bytes() == index - assert not list((repo / ".git").glob("candidate-recovery-*")) + if cache: + originals, parked = _detached_files(repo / ".git") + assert originals[repo / "settings/defaults.py"] == before["settings/defaults.py"][2] + assert parked[repo / "draft.txt"] == b"candidate draft\n" + else: + assert not list((repo / ".git").glob("candidate-recovery-*")) assert _git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1 @@ -184,7 +204,11 @@ async def flow(ctx, inputs): assert result.output == "recovered" assert _run(repo) == ("1\n" if cache else "42\n") assert (repo / ".git/index").read_bytes() == index - assert not list((repo / ".git").glob("candidate-recovery-*")) + if cache: + originals, _ = _detached_files(repo / ".git") + assert originals[repo / "old/nested/helper.py"] == before["old/nested/helper.py"][2] + else: + assert not list((repo / ".git").glob("candidate-recovery-*")) finally: await environment.cleanup() @@ -252,7 +276,8 @@ async def flow(ctx, inputs): result = await OpenCollab(repo, environment=environment).workflow(flow, trace=False) assert result.ok, result.reason assert (repo / ".git/index").read_bytes() == index - assert not list((repo / ".git").glob("candidate-recovery-*")) + originals, _ = _detached_files(repo / ".git") + assert originals[repo / "settings/defaults.py"] == before["settings/defaults.py"][2] chmod_paths = [Path(value.decode()) for value in log.read_bytes().split(b"\0")[2::3] if value] assert stat.S_IMODE(repo.stat().st_mode) == root_mode assert (repo / "old/nested" in chmod_paths) is (change != "readonly-root") @@ -287,7 +312,9 @@ async def test_git_write_failure_after_successful_check_restores_partial_applica await EnvCandidateWorkspace(environment).adopt(patch) assert _contents(repo) == before assert (repo / ".git/index").read_bytes() == index - assert not list((repo / ".git").glob("candidate-recovery-*")) + originals, parked = _detached_files(repo / ".git") + assert originals[repo / "other.py"] == b"VALUE = 1\n" + assert parked[repo / "other.py"] == b"VALUE = 42\n" assert not environment.revoked assert (await environment.exec_cmd("printf usable")).stdout == "usable" finally: @@ -370,13 +397,16 @@ async def flow(ctx, inputs): assert replays == [0, 128] assert target.read_bytes() == b"original\r\n" assert (repo / "settings/cache.txt").read_bytes() == b"preserve cache\n" + originals, parked = _detached_files(repo / ".git") + assert originals[target] == b"original\r\n" + assert parked[target] == b"candidate\r\n" else: assert result.ok, result.reason assert target.read_bytes() == b"candidate\r\n" assert replays == [] + assert not list((repo / ".git").glob("candidate-recovery-*")) assert attributes.read_text() == f"*.txt text eol={'lf' if change_rules else 'crlf'}\n" assert (repo / ".git/index").read_bytes() == index - assert not list((repo / ".git").glob("candidate-recovery-*")) assert not environment.revoked finally: await environment.cleanup() @@ -442,7 +472,8 @@ async def fault(command, timeout=120.0, **kwargs): @pytest.mark.parametrize("inside_repository", [False, True]) -async def test_directory_copy_candidate_replays_partial_apply_outside_git(tmp_path, inside_repository): +async def test_directory_copy_candidate_replays_partial_apply_outside_git(tmp_path, monkeypatch, inside_repository): + monkeypatch.setenv("TMPDIR", str(tmp_path)) if inside_repository: _git(tmp_path, "init", "-q") repo = tmp_path / "copy" @@ -472,6 +503,9 @@ async def inspect(command, timeout=120.0, **kwargs): assert intermediate == ["candidate\n"] assert (repo / "a.txt").read_text() == "original\n" assert (repo / "settings/retained.txt").read_text() == "keep\n" + originals, parked = _detached_files(tmp_path) + assert originals[repo / "a.txt"] == b"original\n" + assert parked[repo / "a.txt"] == b"candidate\n" assert not environment.revoked finally: await environment.cleanup() @@ -521,7 +555,9 @@ async def fault(command, timeout=120.0, **kwargs): with pytest.raises(OSError, match="publication receipt unavailable"): await _apply_increment(environment, str(repo), patch) assert target.read_text() == "VALUE = 1\n" - assert not list((repo / ".git").glob("candidate-recovery-*")) + originals, parked = _detached_files(repo / ".git") + assert originals[target] == b"VALUE = 1\n" + assert parked[target] == b"VALUE = 42\n" assert failures == ["receipt"] assert environment.revoked is replay_full assert (repo / ".git/index").read_bytes() == index @@ -572,7 +608,7 @@ async def after_comparison(command, timeout=120.0, **kwargs): @pytest.mark.parametrize("linked", [False, True]) -async def test_recovery_compares_parked_bytes_before_removing_the_backup(tmp_path, linked): +async def test_recovery_compares_parked_bytes_before_pruning_replay_files(tmp_path, linked): source, candidate = _partial_publication(tmp_path) repo = source if linked: diff --git a/tests/workflows/test_candidate_mutation_backends.py b/tests/workflows/test_candidate_mutation_backends.py index 25b4acf3..099cbd35 100644 --- a/tests/workflows/test_candidate_mutation_backends.py +++ b/tests/workflows/test_candidate_mutation_backends.py @@ -12,7 +12,7 @@ from opencollab.application.tool_execution import ToolRuntime from opencollab.bootstrap.authored_workflow_bridge import AuthoredWorkflowBridge, BridgeClient, BridgeEnvironment from tests.support.docker_native_edits_support import LocalDockerTransport, docker_environment -from tests.workflows.test_candidate_adoption_recovery import _flatten, _repository, _run +from tests.workflows.test_candidate_adoption_recovery import _detached_files, _flatten, _repository, _run from tests.workflows.test_workflow_candidate_workspace import _git @@ -111,6 +111,7 @@ async def serve(): async def test_cancelled_publication_recovers_before_releasing_shared_writer(tmp_path, monkeypatch): repo = _repository(tmp_path) target = repo / "settings/defaults.py" + index = (repo / ".git/index").read_bytes() target.write_text("VALUE = 42\n") patch = _git(repo, "diff", "--binary", "--", "settings/defaults.py") target.write_text("VALUE = 1\n") @@ -143,7 +144,10 @@ async def pause_publication(command, timeout=120, **kwargs): await publication assert "Created/wrote" in await asyncio.wait_for(writer, 10) assert _run(repo) == "100\n" - assert not list((repo / ".git").glob("candidate-recovery-*")) + assert (repo / ".git/index").read_bytes() == index + assert _detached_files(repo / ".git") == ( + {target: b"VALUE = 1\n"}, {target: b"VALUE = 42\n"}, + ) finally: if not publication.done(): publication.cancel() diff --git a/tests/workflows/test_candidate_shared_mutations.py b/tests/workflows/test_candidate_shared_mutations.py index 1d853431..a21ebb95 100644 --- a/tests/workflows/test_candidate_shared_mutations.py +++ b/tests/workflows/test_candidate_shared_mutations.py @@ -13,7 +13,13 @@ from opencollab.adapters.llm.types import LLMResponse, Usage from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError from opencollab.tools import builtin_tools -from tests.workflows.test_candidate_adoption_recovery import _flatten, _partial_publication, _repository, _run +from tests.workflows.test_candidate_adoption_recovery import ( + _detached_files, + _flatten, + _partial_publication, + _repository, + _run, +) from tests.workflows.test_workflow_candidate_workspace import _git @@ -66,6 +72,24 @@ async def complete(self, messages, tools=None, **kwargs): return LLMResponse(content="Shared edit complete", usage=Usage(10, 5), finish_reason="stop") +def _assert_flatten_recovery(repo: Path, loader: bytes) -> None: + assert (repo / "loader.py").read_bytes() == loader + assert (repo / "draft.txt").read_bytes() == b"untracked user draft\n" + assert _detached_files(repo / ".git") == ( + { + repo / "loader.py": loader, + repo / "draft.txt": b"untracked user draft\n", + repo / "settings/defaults.py": b"VALUE = 1\n", + }, + { + repo / "loader.py": ( + b'from pathlib import Path\ndef value():\n return int(Path("settings").read_text())\n' + ), + repo / "draft.txt": b"candidate draft\n", + }, + ) + + @pytest.mark.parametrize("tool", ["file_write", "apply_patch"]) @pytest.mark.parametrize("relation,failure", [ ("target", False), ("target", True), ("descendant", True), @@ -85,6 +109,7 @@ async def test_sdk_shared_agent_owns_mutations_through_candidate_publication( environment = _PublicationBarrier(repo) path = "other.py" if relation == "unrelated" else "settings/defaults.py" model = _WriterModel(tool, path) + loader = (repo / "loader.py").read_bytes() index = (repo / ".git/index").read_bytes() async def edit(child, _): @@ -143,7 +168,17 @@ async def flow(ctx, _): assert result.output == "retained" assert _run(repo) == ("100\n" if relation != "unrelated" else "1\n" if failure else "42\n") assert (repo / ".git/index").read_bytes() == index - assert not list((repo / ".git").glob("candidate-recovery-*")) + if not failure: + assert not list((repo / ".git").glob("candidate-recovery-*")) + elif relation == "descendant": + _assert_flatten_recovery(repo, loader) + else: + blocked = repo / "blocked/entry.py" + assert blocked.read_bytes() == b"VALUE = 1\n" + assert _detached_files(repo / ".git") == ( + {blocked: b"VALUE = 1\n", repo / "settings/defaults.py": b"VALUE = 1\n"}, + {}, + ) assert not environment.revoked finally: environment.release.set() @@ -200,9 +235,8 @@ async def flow(ctx, _): assert (repo / "settings/defaults.py").read_text() == "VALUE = 1\n" assert (repo / "draft.txt").read_text() == "untracked user draft\n" assert (repo / ".git/index").read_bytes() == index - backups = list((repo / ".git").glob("candidate-recovery-*")) if overlap: - backup, = backups + backup, = (repo / ".git").glob("candidate-recovery-*") for record in backup.glob("*.path"): if Path(record.read_text()).name == "loader.py": assert record.with_suffix("").read_bytes() == loader @@ -210,8 +244,7 @@ async def flow(ctx, _): else: pytest.fail("recovery did not retain the original loader") else: - assert (repo / "loader.py").read_bytes() == loader - assert not backups + _assert_flatten_recovery(repo, loader) finally: await environment.cleanup() From 7c6d792ceb27ab9123d4f311468a99dd2241ca7e Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 12:41:06 +0800 Subject: [PATCH 50/82] fix(workflow): retain rollback files without hard-link support Prepare same-filesystem replacements before moving originals. On a preparation failure, restore only interrupted moves instead of replaying a patch that never touched the worktree. --- opencollab/adapters/_candidate_adoption.py | 146 ++++++++++++------ ...test_candidate_adoption_external_writer.py | 93 +++++++++-- .../test_candidate_docker_alias_mutations.py | 5 +- 3 files changed, 186 insertions(+), 58 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index b5ed08d1..5aecd97a 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -28,7 +28,12 @@ [ "$left" = "$right" ] elif [ -f "$1" ]; then [ ! -L "$2" ] && [ -f "$2" ] || return 1 - left=$(mode "$1") && right=$(mode "$2") && [ "$left" = "$right" ] && cmp -s -- "$1" "$2" + left=$(mode "$1") && right=$(mode "$2") && [ "$left" = "$right" ] || return 1 + if command -v cmp >/dev/null 2>&1; then + cmp -s -- "$1" "$2" + else + git diff --no-index --quiet --no-ext-diff --no-textconv -- "$1" "$2" + fi else ! present "$1" && ! present "$2" fi @@ -51,23 +56,92 @@ } ''' -_HOLD_ORIGINALS = _COMPARE_ENTRIES + _CHECK_PARENTS + r''' +_DEVICE = r''' +device() { stat -c %d -- "$1" 2>/dev/null || stat -f %d -- "$1"; } +''' + +_HOLD_ORIGINALS = _COMPARE_ENTRIES + _CHECK_PARENTS + _DEVICE + r''' hold_originals() { index=0 + needs_activation=0 mkdir -m 700 -- "$recovery/original-inodes" || return 1 for path; do source=$workspace_prefix$path if check_parents "$source" 2>/dev/null && [ -f "$source" ] && [ ! -L "$source" ]; then entry=$recovery/original-inodes/$index - ln -P -- "$source" "$entry" && printf '%s' "$source" > "$entry.path" || return 1 + if ! ln -P -- "$source" "$entry" 2>/dev/null; then + source_device=$(device "$source") && recovery_device=$(device "$recovery") || return 1 + if [ "$source_device" != "$recovery_device" ]; then + printf 'cannot preserve %s across filesystems\n' "$source" >&2 + return 1 + fi + mkdir -p -m 700 -- "$recovery/replacements" && + cp -pP -- "$source" "$recovery/replacements/$index" || return 1 + needs_activation=1 + fi + printf '%s' "$source" > "$entry.path" || return 1 index=$((index + 1)) fi done + if [ "$needs_activation" -eq 1 ]; then printf 'activate-originals\n'; fi } ''' -_DEVICE = r''' -device() { stat -c %d -- "$1" 2>/dev/null || stat -f %d -- "$1"; } +_RESTORE_SAVED = r''' +restore_saved() { + if ! present "$1"; then return 0; fi + if present "$2"; then conflict "$2"; return; fi + mv -n -- "$1" "$2" || return 1 + if present "$1"; then conflict "$2"; fi +} +''' + +_ACTIVATE_ORIGINALS = _COMPARE_ENTRIES + _CHECK_PARENTS + _RESTORE_SAVED + r''' +for replacement in "$recovery"/replacements/*; do + [ -f "$replacement" ] || continue + entry=$recovery/original-inodes/${replacement##*/} + live=$(cat -- "$entry.path" && printf .) || exit + live=${live%.} + check_parents "$live" && same_entry "$replacement" "$live" || { conflict "$live"; exit; } + restore_saved "$live" "$entry" || exit + if ! same_entry "$replacement" "$entry"; then + restore_saved "$entry" "$live" || exit + conflict "$live" + exit + fi + restore_saved "$replacement" "$live" || exit +done +''' + +_UNDO_INCOMPLETE_ACTIVATION = _COMPARE_ENTRIES + _CHECK_PARENTS + _RESTORE_SAVED + r''' +for replacement in "$recovery"/replacements/*; do + entry=$recovery/original-inodes/${replacement##*/} + [ -f "$replacement" ] && [ -f "$entry" ] || continue + live=$(cat -- "$entry.path" && printf .) || exit + live=${live%.} + check_parents "$live" && restore_saved "$entry" "$live" || exit +done +''' + +_DETACHED_FILES = r''' +detached=0 +for record in "$recovery"/original-inodes/*.path; do + [ -f "$record" ] || continue + path=$(cat -- "$record" && printf .) || exit + path=${path%.} + entry=${record%.path} + if ! present "$entry"; then + rm -- "$record" || exit + elif check_parents "$path" 2>/dev/null && [ ! -L "$path" ] && [ "$path" -ef "$entry" ]; then + rm -- "$record" "$entry" || exit + else + detached=1 + fi +done +for parked in "$recovery"/observed-*/entry; do + if [ -f "$parked" ] && [ ! -L "$parked" ]; then printf 'parked\n'; break; fi +done +if [ "$detached" -eq 1 ]; then printf 'original-detached\n'; fi ''' _RECOVERY_PARENT = _DEVICE + r''' @@ -86,7 +160,7 @@ } ''' -_RECOVER_PATHS = _COMPARE_ENTRIES + _CHECK_PARENTS + _DEVICE + r''' +_RECOVER_PATHS = _COMPARE_ENTRIES + _CHECK_PARENTS + _DEVICE + _RESTORE_SAVED + r''' check_candidate_descendants() { prefix=${1#"$workspace_prefix"}; shift @@ -136,13 +210,6 @@ fi } -restore_saved() { - if ! present "$1"; then return 0; fi - if present "$2"; then conflict "$2"; return; fi - mv -n -- "$1" "$2" || return 1 - if present "$1"; then conflict "$2"; fi -} - park_candidate_entry() { if ! present "$1"; then return 0; fi if [ -f "$1" ] && [ ! -L "$1" ]; then @@ -450,29 +517,7 @@ async def remove_backup(path: str, *, retain_detached: bool = False) -> None: f"candidate_descendants() {{ check_candidate_descendants \"$1\" {' '.join(targets)}; }}\n" ) + _RECOVER_PATHS + "; ".join(( *recreate_directories, *restores, *reversed(restore_directory_modes), - )) + r'''; -if [ "$failed" -ne 0 ]; then exit "$failed"; fi -detached=0 -for record in "$recovery"/original-inodes/*.path; do - [ -f "$record" ] || continue - path=$(cat -- "$record" && printf .) || exit - path=${path%.} - entry=${record%.path} - if check_parents "$path" 2>/dev/null && [ ! -L "$path" ] && [ "$path" -ef "$entry" ]; then - rm -- "$record" "$entry" || exit - else - detached=1 - fi -done -for parked in "$recovery"/observed-*/entry; do - if [ -f "$parked" ] && [ ! -L "$parked" ]; then printf 'parked\n'; break; fi -done -if [ "$detached" -eq 1 ]; then - printf 'original-detached\n' -else - rmdir -- "$recovery/original-inodes" || exit -fi -''' + )) + ';\nif [ "$failed" -ne 0 ]; then exit "$failed"; fi\n' + _DETACHED_FILES async with AsyncExitStack() as commands: snapshot_command = await commands.enter_async_context(_adoption_command( environment, workspace, snapshot_script, repository=repository, @@ -481,22 +526,37 @@ async def remove_backup(path: str, *, retain_detached: bool = False) -> None: recovery_command = await commands.enter_async_context(_adoption_command( environment, workspace, recovery_script, repository=repository, )) - _complete(await environment.exec_cmd(snapshot_command, timeout=900), "candidate recovery snapshot") + snapshot = _complete( + await environment.exec_cmd(snapshot_command, timeout=900), "candidate recovery snapshot", + ) + publication_started = False try: + if "activate-originals" in snapshot.splitlines(): + # Filesystems without hard links keep the original by + # moving it aside and giving Git an equivalent working copy. + _complete(await environment.exec_cmd( + workspace_vars + _ACTIVATE_ORIGINALS, timeout=900, + ), "candidate recovery preparation") + publication_started = True yield except BaseException as failure: async def recover() -> tuple[bool, bool]: - async with _adoption_command( - environment, workspace, replay_script, repository=repository, - ) as command: - _complete(await environment.exec_cmd(command, timeout=900), "candidate recovery replay copy") - await replay(staged, git_directory, saved_attributes_file) + command = recovery_command + if publication_started: + async with _adoption_command( + environment, workspace, replay_script, repository=repository, + ) as copy_command: + _complete(await environment.exec_cmd(copy_command, timeout=900), + "candidate recovery replay copy") + await replay(staged, git_directory, saved_attributes_file) + else: + command = workspace_vars + _UNDO_INCOMPLETE_ACTIVATION + _DETACHED_FILES shell_commands = getattr(environment, "_shell_commands", None) active = shell_commands.active if shell_commands is not None else 0 last_started = shell_commands.last_started if shell_commands is not None else None receipt = _complete(await environment.exec_cmd( - recovery_command, timeout=900, + command, timeout=900, ), "candidate adoption recovery") shell_overlapped = shell_commands is not None and ( active > 0 or last_started is not shell_commands.last_started diff --git a/tests/workflows/test_candidate_adoption_external_writer.py b/tests/workflows/test_candidate_adoption_external_writer.py index 2b552c29..12e0cc38 100644 --- a/tests/workflows/test_candidate_adoption_external_writer.py +++ b/tests/workflows/test_candidate_adoption_external_writer.py @@ -39,6 +39,15 @@ def _identity(path: Path) -> tuple[int, int]: return status.st_dev, status.st_ino +def _delete_original(candidate): + _, replacement = candidate.diff.split("diff --git a/settings", 1) + return replace(candidate, diff=( + "diff --git a/record.txt b/record.txt\ndeleted file mode 100644\n" + "--- a/record.txt\n+++ /dev/null\n@@ -1 +0,0 @@\n-original\n" + "diff --git a/settings" + replacement + )) + + @pytest.fixture def worktree_parent(tmp_path, request): if not request.param: @@ -53,13 +62,14 @@ def worktree_parent(tmp_path, request): yield Path(path) -@pytest.mark.parametrize("opened_before_apply,linked,worktree_parent", [ - pytest.param(True, False, False, id="original-inode"), - pytest.param(False, True, False, id="candidate-inode-after-worktree-removal"), - pytest.param(False, True, True, id="separate-worktree-filesystem"), +@pytest.mark.parametrize("opened_before_apply,linked,worktree_parent,hard_links", [ + pytest.param(True, False, False, True, id="original-inode"), + pytest.param(False, True, False, True, id="candidate-inode-after-worktree-removal"), + pytest.param(False, True, True, True, id="separate-worktree-filesystem"), + pytest.param(True, True, False, False, id="original-inode-without-hard-links"), ], indirect=["worktree_parent"]) async def test_candidate_rollback_retains_inode_for_external_writer( - tmp_path, monkeypatch, caplog, opened_before_apply, linked, worktree_parent, + tmp_path, monkeypatch, caplog, opened_before_apply, linked, worktree_parent, hard_links, ): source, candidate = _partial_publication(tmp_path) repo = source @@ -98,10 +108,10 @@ async def start_writer(expected: bytes) -> None: assert ready.split() == [str(value).encode() for value in writer_inode] async def observe(command, timeout=120.0, **kwargs): + if not hard_links and "hold_originals()" in command: + command = "ln() { return 1; }; " + command if " apply --binary " not in command or ".candidate-adopt-" not in command: return await execute(command, timeout, **kwargs) - if opened_before_apply: - await start_writer(original) outcome = await execute(command, timeout, **kwargs) assert outcome.returncode == 128, outcome.stderr assert target.read_bytes() == published @@ -116,6 +126,8 @@ async def flow(ctx, _): await ctx.adopt_candidate(candidate) try: + if opened_before_apply: + await start_writer(original) result = await OpenCollab(repo, environment=environment, model="offline").workflow( flow, trace=False, timeout=30, ) @@ -162,14 +174,69 @@ async def flow(ctx, _): await environment.cleanup() +@pytest.mark.parametrize("external_delete", [False, True], ids=["interrupted-move", "external-delete"]) +async def test_failed_preparation_recovers_only_owned_moves(tmp_path, monkeypatch, external_delete): + repo, candidate = _partial_publication(tmp_path) + if external_delete: + candidate = _delete_original(candidate) + target = repo / "record.txt" + identity = _identity(target) + index = repo / ".git/index" + index_before = index.read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + reached_activation = False + published = False + + async def interrupt(command, timeout=120.0, **kwargs): + nonlocal reached_activation, published + if "hold_originals()" in command: + command = "ln() { return 1; }; " + command + if 'restore_saved "$replacement" "$live"' in command: + reached_activation = True + if external_delete: + writer = await asyncio.create_subprocess_exec( + sys.executable, "-c", "import os, sys; os.unlink(sys.argv[1])", str(target), + ) + assert await asyncio.wait_for(writer.wait(), 10) == 0 + else: + command = r''' +mv() { + case "$3" in + */replacements/*) printf 'replacement move interrupted\n' >&2; return 1;; + *) command mv "$@";; + esac +} +''' + command + if " apply --binary " in command and ".candidate-adopt-" in command: + published = True + return await execute(command, timeout, **kwargs) + + monkeypatch.setattr(environment, "exec_cmd", interrupt) + + async def flow(ctx, _): + await ctx.adopt_candidate(candidate) + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow(flow, trace=False) + assert reached_activation and not published + assert type(result.error) is RuntimeError + if external_delete: + assert "concurrent change" in result.reason + assert not target.exists() + else: + assert "replacement move interrupted" in result.reason + assert _identity(target) == identity + assert target.read_bytes() == b"original\n" + assert index.read_bytes() == index_before + assert not list((repo / ".git").glob("candidate-recovery-*")) + finally: + await environment.cleanup() + + async def test_detached_original_only_preserves_publication_error_during_shell_execution(tmp_path, monkeypatch): repo, candidate = _partial_publication(tmp_path) - _, replacement = candidate.diff.split("diff --git a/settings", 1) - candidate = replace(candidate, diff=( - "diff --git a/record.txt b/record.txt\ndeleted file mode 100644\n" - "--- a/record.txt\n+++ /dev/null\n@@ -1 +0,0 @@\n-original\n" - "diff --git a/settings" + replacement - )) + candidate = _delete_original(candidate) target = repo / "record.txt" original_inode = _identity(target) index = repo / ".git/index" diff --git a/tests/workflows/test_candidate_docker_alias_mutations.py b/tests/workflows/test_candidate_docker_alias_mutations.py index c12374d2..3912c39f 100644 --- a/tests/workflows/test_candidate_docker_alias_mutations.py +++ b/tests/workflows/test_candidate_docker_alias_mutations.py @@ -16,7 +16,7 @@ from opencollab.tools import builtin_tools from tests.support.docker_native_edits_support import LocalDockerTransport, RemoteDockerTransport, docker_environment from tests.workflows.test_candidate_adoption_recovery import _flatten, _repository, _run -from tests.workflows.test_candidate_shared_mutations import _WriterModel +from tests.workflows.test_candidate_shared_mutations import _assert_flatten_recovery, _WriterModel from tests.workflows.test_workflow_candidate_workspace import _git @@ -52,6 +52,7 @@ async def test_sdk_remote_alias_mutations_survive_directory_recovery(tmp_path, m _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: retain workspace directory alias") for cached in ("settings/runtime.cache", "unrelated.cache"): (repo / cached).write_text("VALUE = 1\n") + loader = (repo / "loader.py").read_bytes() index = (repo / ".git/index").read_bytes() environment, transport = _remote_environment(repo) entered, release = asyncio.Event(), asyncio.Event() @@ -121,7 +122,7 @@ async def flow(ctx, _): assert result.output == "retained" assert _run(repo) == "1\n" assert (repo / ".git/index").read_bytes() == index - assert not list((repo / ".git").glob("candidate-recovery-*")) + _assert_flatten_recovery(repo, loader) assert not environment._temporary_files assert not environment.revoked finally: From 81428b89855a90dc442fc54632fb2c17337d89e8 Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 12:41:06 +0800 Subject: [PATCH 51/82] fix(session): preserve typed JSON values in native history Use type-sensitive equality for reusable transcript rows, journal prefixes, and checkpoint equivalence so a resumed Anthropic session preserves structured tool inputs. --- opencollab/adapters/storage.py | 5 +- opencollab/application/_anthropic_history.py | 14 ++-- opencollab/application/session.py | 4 +- opencollab/application/session_snapshot.py | 4 +- opencollab/domain/session.py | 18 +++++ .../test_anthropic_preserved_history.py | 66 ++++++++++++++++++- 6 files changed, 96 insertions(+), 15 deletions(-) diff --git a/opencollab/adapters/storage.py b/opencollab/adapters/storage.py index 66a83820..45da6877 100644 --- a/opencollab/adapters/storage.py +++ b/opencollab/adapters/storage.py @@ -21,6 +21,7 @@ write_regular_file_atomic, ) from opencollab.application.ports import SnapshotCapacityError +from opencollab.domain.session import same_snapshot_value MAX_SESSION_SNAPSHOT_BYTES = 64 * 1024 * 1024 _JSON_WRITE_CHUNK_CHARS = 64 * 1024 @@ -487,7 +488,7 @@ def append_snapshot_replacement( stored_messages = persisted["messages"] replace_from = min(len(stored_messages), len(messages)) for index in range(replace_from): - if stored_messages[index] != messages[index]: + if not same_snapshot_value(stored_messages[index], messages[index]): replace_from = index break record["replace_from"] = replace_from @@ -763,7 +764,7 @@ def normalize(value: dict[str, Any]) -> dict[str, Any]: normalized["session_state"] = state return normalized - return normalize(persisted) == normalize(candidate) + return same_snapshot_value(normalize(persisted), normalize(candidate)) @staticmethod def _snapshot_sequence(snapshot: dict[str, Any]) -> int: diff --git a/opencollab/application/_anthropic_history.py b/opencollab/application/_anthropic_history.py index 9e6ab2d1..5c99cae5 100644 --- a/opencollab/application/_anthropic_history.py +++ b/opencollab/application/_anthropic_history.py @@ -5,7 +5,7 @@ import copy from typing import Any -from opencollab.domain.session import SessionState +from opencollab.domain.session import SessionState, same_snapshot_value _THINKING_TYPES = frozenset({"thinking", "redacted_thinking"}) @@ -25,19 +25,19 @@ def _strip_anthropic_thinking(messages: list[dict[str, Any]], start: int = 0) -> def _edited_anthropic_history(before: list[dict], after: list[dict]) -> list[dict]: """Drop thinking bound to a changed prefix, keeping the valid older run.""" - if [message for message in before if message.get("role") == "system" and not message.get("compacted")] != [ - message for message in after if message.get("role") == "system" and not message.get("compacted") - ]: + before_system = [message for message in before if message.get("role") == "system" and not message.get("compacted")] + after_system = [message for message in after if message.get("role") == "system" and not message.get("compacted")] + if not same_snapshot_value(before_system, after_system): return _strip_anthropic_thinking(after) for index, (old, new) in enumerate(zip(before, after)): - if old != new: + if not same_snapshot_value(old, new): return _strip_anthropic_thinking(after, index) return after def _anthropic_source_changed(state: SessionState) -> bool: history = state.anthropic_history source = history.get("source_messages") if history is not None else None - return source is not None and state.messages[:len(source)] != source + return source is not None and not same_snapshot_value(state.messages[:len(source)], source) def _anthropic_history_input(state: SessionState) -> list[dict]: """Reuse the native view until its original transcript prefix changes.""" @@ -56,7 +56,7 @@ def _remember_anthropic_history( messages = _edited_anthropic_history(_anthropic_history_input(state), messages) if history is not None: messages = _edited_anthropic_history(history["messages"], messages) - if history["tools"] != tools: + if not same_snapshot_value(history["tools"], tools): messages = _strip_anthropic_thinking(messages) state.anthropic_history = copy.deepcopy({ "messages": messages, "source_count": len(state.messages), "tools": tools, diff --git a/opencollab/application/session.py b/opencollab/application/session.py index a62a5f9b..92b982fe 100644 --- a/opencollab/application/session.py +++ b/opencollab/application/session.py @@ -42,7 +42,7 @@ from opencollab.domain.agent import Agent from opencollab.domain.events import SessionRuntimeEvent as SessionEvent from opencollab.domain.pending import PendingRow, RowKind, RowStatus -from opencollab.domain.session import SessionPhase, SessionState +from opencollab.domain.session import SessionPhase, SessionState, same_snapshot_value from opencollab.domain.tool_loops import LoopState if TYPE_CHECKING: @@ -1084,7 +1084,7 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: if replace_from: current_messages = self.state.enriched_messages() for index in range(replace_from): - if current_messages[index] != stored_messages[index]: + if not same_snapshot_value(current_messages[index], stored_messages[index]): replace_from = index break # A full native rewrite can exceed the journal's remaining capacity. diff --git a/opencollab/application/session_snapshot.py b/opencollab/application/session_snapshot.py index d35dccd9..24a77ea4 100644 --- a/opencollab/application/session_snapshot.py +++ b/opencollab/application/session_snapshot.py @@ -8,7 +8,7 @@ from opencollab.application._anthropic_history import _anthropic_source_changed, _edited_anthropic_history from opencollab.domain.pending import PendingRow, RowKind, RowStatus -from opencollab.domain.session import SessionPhase, SessionState +from opencollab.domain.session import SessionPhase, SessionState, same_snapshot_value # Translate legacy terminals and the removed enqueue phase so old snapshots # retain their disposition instead of falling back to IDLE. @@ -34,7 +34,7 @@ def _serialize_anthropic_history(state: SessionState) -> dict | None: suffix_offset = count - len(view) for index, message in enumerate(view): for source_index in (index, index + suffix_offset): - if 0 <= source_index < count and message == source[source_index]: + if 0 <= source_index < count and same_snapshot_value(message, source[source_index]): stored.append(source_index) break else: diff --git a/opencollab/domain/session.py b/opencollab/domain/session.py index a833cb67..fda428ad 100644 --- a/opencollab/domain/session.py +++ b/opencollab/domain/session.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import math import uuid from dataclasses import dataclass, field from datetime import datetime, timezone @@ -13,6 +14,23 @@ MAX_SCOUT_LEDGER_CARDS = 256 +def same_snapshot_value(left: Any, right: Any) -> bool: + """Compare snapshot values without equating JSON booleans and numbers.""" + if left is right: + return True + if type(left) is not type(right): + return False + if isinstance(left, dict): + return left.keys() == right.keys() and all( + same_snapshot_value(value, right[key]) for key, value in left.items() + ) + if isinstance(left, list): + return len(left) == len(right) and all(same_snapshot_value(a, b) for a, b in zip(left, right)) + if isinstance(left, float) and left == right == 0: + return math.copysign(1, left) == math.copysign(1, right) + return left == right + + def _now_iso() -> str: return datetime.now(timezone.utc).isoformat() diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index fc7450e6..5dde2390 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -52,10 +52,11 @@ def _signatures(body): class _BoundThinkingProvider: """Compare the documented prefixes through the real Anthropic SDK wire.""" - def __init__(self, reads=2, *, thinking=True, correction=False): + def __init__(self, reads=2, *, thinking=True, correction=False, read_input=None): self.reads = reads self.thinking = thinking self.correction = correction + self.read_input = read_input or {} self.requests = [] self.origins = {} self.retired = set() @@ -96,7 +97,7 @@ def __call__(self, request): ] if number <= self.reads: content.append({"type": "tool_use", "id": f"read-{number}", "name": "file_read", - "input": {"path": "source.txt"}}) + "input": {"path": "source.txt", **self.read_input}}) elif self.correction and [tool["name"] for tool in body.get("tools", [])] == ["structured_output"]: content.append({"type": "tool_use", "id": "capture", "name": "structured_output", "input": {"answer": 42}}) else: @@ -221,6 +222,67 @@ async def test_compacted_native_view_survives_restore_and_disabling_the_shaper( assert peer.retired assert _signatures(peer.requests[-1])[-2:] == ["signed-5", "redacted-5"] + +@pytest.mark.parametrize("mode", ["projected", "journal"]) +async def test_native_snapshot_preserves_typed_tool_input(monkeypatch, tmp_path, mode): + peer = _BoundThinkingProvider(reads=1, read_input={"marker": 1}) + install_sdk_transport(monkeypatch, "anthropic", peer) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + + def read_input(blocks): + return next(block["input"] for block in blocks if block["type"] == "tool_use") + + def native_input(message): + return read_input(message["provider_state"]["anthropic_content"]) + + def sent_input(request): + return read_input([block for message in request["messages"] for block in _blocks(message["content"])]) + + class TypedProjection: + def shape(self, messages): + projected = copy.deepcopy(messages) + if len(projected) > 2: + native_input(projected[2])["marker"] = True + return projected + + path = tmp_path / "typed.json" + environment = LocalEnvironment(str(tmp_path)) + async with _client() as client: + session = build_session( + agent=_agent(), llm=client, env=environment, + shaper=TypedProjection() if mode == "projected" else None, + auto_save_path=str(path) if mode == "journal" else None, + ) + restored = None + try: + await session.add_user_message("Read the file") + assert await session.run_loop() == "done" + session.save(str(path)) + if mode == "projected": + assert sent_input(peer.requests[-1])["marker"] is True + assert type(native_input(session.messages[2])["marker"]) is int + else: + if session._auto_save_sequence + 1 == session._next_auto_save_checkpoint: + await session.enqueue_auto_save() + native_input(session.messages[2])["marker"] = True + await session.enqueue_auto_save() + journal = path.with_suffix(".json.journal") + delta = json.loads(journal.read_text().splitlines()[-1]) + assert delta["replace_from"] == 2 + assert native_input(delta["messages"][0])["marker"] is True + + restored = load_session(str(path), agent=_agent(), llm=client, env=environment) + await restored.add_user_message("Continue with the recorded tool use") + assert await restored.run_loop() == "done" + assert sent_input(peer.requests[-1])["marker"] is True + assert "signed-1" not in _signatures(peer.requests[-1]) + finally: + if restored is not None: + await restored.aclose() + await session.aclose() + await environment.cleanup() + + async def test_appended_system_and_changed_tools_retire_old_native_thinking(monkeypatch, tmp_path): monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") peer = _BoundThinkingProvider(reads=0) From d4847352e23f0eea348494656870662f76aa12f1 Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 12:58:32 +0800 Subject: [PATCH 52/82] perf(workflow): batch local recovery hard links Use the installed Python interpreter once for large local patches instead of launching a hard-link process per file. Keep the portable fallback for environments without that interpreter and for filesystems that require the rename-based path. --- opencollab/adapters/_candidate_adoption.py | 49 ++++++++++++++----- .../test_candidate_adoption_command_size.py | 7 +++ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index 5aecd97a..b09d65bc 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -5,6 +5,7 @@ import logging import posixpath import shlex +import sys from collections.abc import AsyncIterator, Awaitable, Callable, Sequence from contextlib import AsyncExitStack, asynccontextmanager from typing import Any @@ -60,7 +61,31 @@ device() { stat -c %d -- "$1" 2>/dev/null || stat -f %d -- "$1"; } ''' +_LINK_ORIGINALS = """\ +import os +import sys +from pathlib import Path + +for record in Path(sys.argv[1]).glob("[0-9]*.path"): + try: + os.link(record.read_bytes(), record.with_suffix(""), follow_symlinks=False) + except OSError: + sys.exit(1) +""" + _HOLD_ORIGINALS = _COMPARE_ENTRIES + _CHECK_PARENTS + _DEVICE + r''' +hold_original() { + if [ "$1" -ef "$2" ] || ln -P -- "$1" "$2" 2>/dev/null; then return 0; fi + source_device=$(device "$1") && recovery_device=$(device "$recovery") || return 1 + if [ "$source_device" != "$recovery_device" ]; then + printf 'cannot preserve %s across filesystems\n' "$1" >&2 + return 1 + fi + mkdir -p -m 700 -- "$recovery/replacements" && + cp -pP -- "$1" "$recovery/replacements/${2##*/}" || return 1 + needs_activation=1 +} + hold_originals() { index=0 needs_activation=0 @@ -69,20 +94,18 @@ source=$workspace_prefix$path if check_parents "$source" 2>/dev/null && [ -f "$source" ] && [ ! -L "$source" ]; then entry=$recovery/original-inodes/$index - if ! ln -P -- "$source" "$entry" 2>/dev/null; then - source_device=$(device "$source") && recovery_device=$(device "$recovery") || return 1 - if [ "$source_device" != "$recovery_device" ]; then - printf 'cannot preserve %s across filesystems\n' "$source" >&2 - return 1 - fi - mkdir -p -m 700 -- "$recovery/replacements" && - cp -pP -- "$source" "$recovery/replacements/$index" || return 1 - needs_activation=1 - fi printf '%s' "$source" > "$entry.path" || return 1 + if [ -z "$linker" ]; then hold_original "$source" "$entry" || return 1; fi index=$((index + 1)) fi done + if [ -n "$linker" ] && ! "$linker" -I -S -c "$link_script" "$recovery/original-inodes"; then + for record in "$recovery"/original-inodes/*.path; do + [ -f "$record" ] || continue + source=$(cat -- "$record" && printf .) || return 1 + hold_original "${source%.}" "${record%.path}" || return 1 + done + fi if [ "$needs_activation" -eq 1 ]; then printf 'activate-originals\n'; fi } ''' @@ -505,8 +528,12 @@ async def remove_backup(path: str, *, retain_detached: bool = False) -> None: f"workspace={shlex.quote(workspace)}; recovery={shlex.quote(backup)};\n" 'workspace_prefix="${workspace%/}/";\n' ) + # Batch local links to avoid launching a process for every file. + linker = sys.executable if len(paths) >= 64 and getattr(environment, "local_filesystem", False) else "" snapshot_script = ( - workspace_vars + _HOLD_ORIGINALS + f"hold_originals {' '.join(targets)} || exit; " + workspace_vars + f"linker={shlex.quote(linker)};\n" + + (f"link_script={shlex.quote(_LINK_ORIGINALS)};\n" if linker else "") + + _HOLD_ORIGINALS + f"hold_originals {' '.join(targets)} || exit; " + "; ".join(copies) ) target_test = f'case "${{1#"$workspace_prefix"}}" in {"|".join(targets)}) return 0;; esac; ' if targets else "" diff --git a/tests/workflows/test_candidate_adoption_command_size.py b/tests/workflows/test_candidate_adoption_command_size.py index 9ef32951..b3a28863 100644 --- a/tests/workflows/test_candidate_adoption_command_size.py +++ b/tests/workflows/test_candidate_adoption_command_size.py @@ -38,6 +38,8 @@ async def test_bulk_candidate_adoption_keeps_recovery_below_argument_limits(tmp_ if blocked: (repo / "settings/session.cache").write_text("runtime cache\n") index = (repo / ".git/index").read_bytes() + first_module = repo / "src" / names[0] + original_inode = first_module.stat().st_ino environment = _ArgumentBoundEnvironment(str(repo)) async def edit(child, inputs): @@ -71,6 +73,11 @@ async def flow(ctx, inputs): originals, _ = _detached_files(repo / ".git") assert originals[repo / "settings/value.txt"] == b"original settings\n" assert all(originals[repo / "src" / name] == b"VALUE = 1\n" for name in names) + record, = ( + path for path in (repo / ".git").glob("candidate-recovery-*/original-inodes/*.path") + if Path(path.read_text()) == first_module + ) + assert record.with_suffix("").stat().st_ino == original_inode else: assert (repo / "settings").read_text() == "new settings\n" assert not list((repo / ".git").glob("candidate-recovery-*")) From 45dafca723234a9217fc578873fc8a3dc06c76a9 Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 15:06:12 +0800 Subject: [PATCH 53/82] fix(workflow): preserve candidate recovery across mounts Keep recovery metadata in the primary directory and retain physical originals and parked outputs on their own mounts. Stage restored copies locally and compare the mode that the live filesystem can materialize before moving a candidate out of the worktree. Exercise a mixed-device patch, bind-mount identity, new and existing files, and filesystems without hard links through the public workflow API. --- docs/duo.md | 9 +- docs/duo/README.zh-CN.md | 2 +- opencollab/adapters/_candidate_adoption.py | 278 ++++++++++++------ .../test_candidate_adoption_mounts.py | 152 ++++++++++ 4 files changed, 348 insertions(+), 93 deletions(-) create mode 100644 tests/workflows/test_candidate_adoption_mounts.py diff --git a/docs/duo.md b/docs/duo.md index f3b8def5..fbfc37cb 100644 --- a/docs/duo.md +++ b/docs/duo.md @@ -206,10 +206,11 @@ Adoption applies that increment to the current source files and preserves indepe user edits and the source index. A conflicting patch leaves the current source in place. If a partially applied patch is rolled back, OpenCollab retains detached files that another process may still be writing to. The error or warning gives the -`candidate-recovery-*` directory, usually in the common Git directory and beside -the worktree when Git metadata is on another filesystem. The adjacent `path` or -`.path` record identifies each file's original location. Close the writers before -recovering their data and removing the recovery directory. +`candidate-recovery-*` directory, typically in the common Git directory. +Its `volumes/` records, when present, name additional recovery directories +on affected mounts. The adjacent `path` or `.path` record identifies each +file's original location. Close the writers and recover their data before +removing the recovery directories or worktree. Changes to ignore rules preserve files copied from the source and files added to the candidate index or commits. Newly created ignored files join delivery when explicitly added with `git add --force`. diff --git a/docs/duo/README.zh-CN.md b/docs/duo/README.zh-CN.md index c030ee90..63f23c3e 100644 --- a/docs/duo/README.zh-CN.md +++ b/docs/duo/README.zh-CN.md @@ -89,7 +89,7 @@ Duo 通过现有候选工作区接口完成隔离、变更捕获与采用。默 Git 候选从源目录当前受跟踪文件和未被忽略的未跟踪文件开始,交付补丁表示候选随后产生的增量。采用时将增量应用到当前源文件,保留独立用户修改和源索引,冲突会保留当前源目录。修改忽略规则后,已从源目录复制的文件和候选中已加入索引或提交的文件仍进入交付。新建的忽略文件经 `git add --force` 加入后进入交付。 -补丁部分应用后回滚时,OpenCollab 会保留其他进程可能仍在写入的文件。错误或警告会给出 `candidate-recovery-*` 目录。它通常在 Git 公共目录下,Git 元数据位于另一文件系统时则放在工作树旁。相邻的 `path` 或 `.path` 记录标明文件原来的位置。等待写入进程关闭文件后,再恢复其中的数据和清理该目录。 +补丁部分应用后回滚时,OpenCollab 会保留其他进程可能仍在写入的文件。错误或警告会给出 `candidate-recovery-*` 目录,通常位于 Git 公共目录下。如果包含 `volumes/`,该目录中的记录列出受影响挂载卷上的其他恢复目录。相邻的 `path` 或 `.path` 记录标明文件原来的位置。移除恢复目录或工作树前,请等待写入进程关闭文件,并恢复其中的数据。 候选运行期间源目录发生变化,运行时会报告错误并保留候选工作树。错误包含初始 Git tree,工作树中的 `refs/worktree/opencollab-source` 继续引用它,可以通过 `git show :` 恢复文件。后端提供源版本时,`CandidateRun.source_revision` 记录源 `HEAD`,采用前再次核对。源 diff 相同而 `HEAD` 已变化时,采用也会被拒绝。 diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index b09d65bc..d7f2d982 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -29,7 +29,10 @@ [ "$left" = "$right" ] elif [ -f "$1" ]; then [ ! -L "$2" ] && [ -f "$2" ] || return 1 - left=$(mode "$1") && right=$(mode "$2") && [ "$left" = "$right" ] || return 1 + left=$(mode "$1") || return 1 + right=${3:-} + if [ -z "$right" ]; then right=$(mode "$2") || return 1; fi + [ "$left" = "$right" ] || return 1 if command -v cmp >/dev/null 2>&1; then cmp -s -- "$1" "$2" else @@ -57,8 +60,62 @@ } ''' -_DEVICE = r''' -device() { stat -c %d -- "$1" 2>/dev/null || stat -f %d -- "$1"; } +_FILESYSTEM = r''' +filesystem() { stat -c '%d:%m' -- "$1" 2>/dev/null || stat -f %d -- "$1"; } +''' + +_RECOVERY_STORES = _FILESYSTEM + r''' +store_for() ( + parent=${1%/*} + filesystem_id=$(filesystem "$parent") && primary_id=$(filesystem "$recovery") || exit + if present "$1"; then + entry_id=$(filesystem "$1") || exit + if [ "$entry_id" != "$filesystem_id" ]; then + printf 'cannot hold the mounted entry %s\n' "$1" >&2 + exit 1 + fi + fi + if [ "$filesystem_id" = "$primary_id" ]; then printf '%s' "$recovery"; exit; fi + + # Prefer the mount root so Git can remove empty directories below it. + anchor= + while directory "$parent"; do + [ -w "$parent" ] && anchor=$parent + [ "$parent" != "$workspace" ] && [ "$parent" != / ] || break + parent=${parent%/*} + parent_id=$(filesystem "${parent:-/}") || exit + [ "$parent_id" = "$filesystem_id" ] || break + done + if [ -z "$anchor" ]; then + printf 'no writable recovery directory on the filesystem of %s\n' "$1" >&2 + exit 1 + fi + for location in "$recovery"/volumes/*; do + [ -f "$location" ] || continue + local_store=$(cat -- "$location" && printf .) || exit + local_store=${local_store%.} + [ "${local_store%/*}" = "$anchor" ] || continue + directory "$local_store" && [ "$(filesystem "$local_store")" = "$filesystem_id" ] || exit 1 + printf '%s' "$local_store" + exit + done + mkdir -p -m 700 -- "$recovery/volumes" || exit + location=$(mktemp "$recovery/volumes/volume-XXXXXXXX") || exit + local_store=$(mktemp -d "$anchor/.candidate-recovery-XXXXXXXX") || exit + printf '%s' "$local_store" > "$location" || { rmdir -- "$local_store"; exit 1; } + printf '%s' "$local_store" +) + +each_store() { + for storage_record in "$recovery"/volumes/*; do + [ -f "$storage_record" ] || continue + storage_path=$(cat -- "$storage_record" && printf .) || return 1 + storage_path=${storage_path%.} + [ -n "$storage_path" ] || continue + "$@" "$storage_path" || return $? + done + "$@" "$recovery" +} ''' _LINK_ORIGINALS = """\ @@ -73,16 +130,18 @@ sys.exit(1) """ -_HOLD_ORIGINALS = _COMPARE_ENTRIES + _CHECK_PARENTS + _DEVICE + r''' +_HOLD_ORIGINALS = _COMPARE_ENTRIES + _CHECK_PARENTS + _RECOVERY_STORES + r''' hold_original() { if [ "$1" -ef "$2" ] || ln -P -- "$1" "$2" 2>/dev/null; then return 0; fi - source_device=$(device "$1") && recovery_device=$(device "$recovery") || return 1 - if [ "$source_device" != "$recovery_device" ]; then - printf 'cannot preserve %s across filesystems\n' "$1" >&2 - return 1 + store=$(store_for "$1") || return 1 + if [ "$store" != "$recovery" ]; then + mkdir -p -m 700 -- "$store/original-inodes" || return 1 + held="$store/original-inodes/${2##*/}" + printf '%s' "$1" > "$held.path" && rm -- "$2.path" || return 1 + if ln -P -- "$1" "$held" 2>/dev/null; then return 0; fi fi - mkdir -p -m 700 -- "$recovery/replacements" && - cp -pP -- "$1" "$recovery/replacements/${2##*/}" || return 1 + mkdir -p -m 700 -- "$store/replacements" && + cp -pP -- "$1" "$store/replacements/${2##*/}" || return 1 needs_activation=1 } @@ -119,71 +178,82 @@ } ''' -_ACTIVATE_ORIGINALS = _COMPARE_ENTRIES + _CHECK_PARENTS + _RESTORE_SAVED + r''' -for replacement in "$recovery"/replacements/*; do - [ -f "$replacement" ] || continue - entry=$recovery/original-inodes/${replacement##*/} - live=$(cat -- "$entry.path" && printf .) || exit - live=${live%.} - check_parents "$live" && same_entry "$replacement" "$live" || { conflict "$live"; exit; } - restore_saved "$live" "$entry" || exit - if ! same_entry "$replacement" "$entry"; then - restore_saved "$entry" "$live" || exit - conflict "$live" - exit - fi - restore_saved "$replacement" "$live" || exit -done +_ACTIVATE_ORIGINALS = _COMPARE_ENTRIES + _CHECK_PARENTS + _RESTORE_SAVED + _RECOVERY_STORES + r''' +activate_store() { + for replacement in "$1"/replacements/*; do + [ -f "$replacement" ] || continue + entry=$1/original-inodes/${replacement##*/} + live=$(cat -- "$entry.path" && printf .) || return 1 + live=${live%.} + check_parents "$live" && same_entry "$replacement" "$live" || { conflict "$live"; return; } + restore_saved "$live" "$entry" || return $? + if ! same_entry "$replacement" "$entry"; then + restore_saved "$entry" "$live" || return $? + conflict "$live" + return + fi + restore_saved "$replacement" "$live" || return $? + done +} +each_store activate_store ''' -_UNDO_INCOMPLETE_ACTIVATION = _COMPARE_ENTRIES + _CHECK_PARENTS + _RESTORE_SAVED + r''' -for replacement in "$recovery"/replacements/*; do - entry=$recovery/original-inodes/${replacement##*/} - [ -f "$replacement" ] && [ -f "$entry" ] || continue - live=$(cat -- "$entry.path" && printf .) || exit - live=${live%.} - check_parents "$live" && restore_saved "$entry" "$live" || exit -done +_UNDO_INCOMPLETE_ACTIVATION = _COMPARE_ENTRIES + _CHECK_PARENTS + _RESTORE_SAVED + _RECOVERY_STORES + r''' +undo_store() { + for replacement in "$1"/replacements/*; do + entry=$1/original-inodes/${replacement##*/} + [ -f "$replacement" ] && [ -f "$entry" ] || continue + live=$(cat -- "$entry.path" && printf .) || return 1 + live=${live%.} + check_parents "$live" && restore_saved "$entry" "$live" || return $? + done +} +each_store undo_store || exit ''' _DETACHED_FILES = r''' detached=0 -for record in "$recovery"/original-inodes/*.path; do - [ -f "$record" ] || continue - path=$(cat -- "$record" && printf .) || exit - path=${path%.} - entry=${record%.path} - if ! present "$entry"; then - rm -- "$record" || exit - elif check_parents "$path" 2>/dev/null && [ ! -L "$path" ] && [ "$path" -ef "$entry" ]; then - rm -- "$record" "$entry" || exit - else - detached=1 - fi -done -for parked in "$recovery"/observed-*/entry; do - if [ -f "$parked" ] && [ ! -L "$parked" ]; then printf 'parked\n'; break; fi -done +parked=0 +collect_detached() { + for record in "$1"/original-inodes/*.path; do + [ -f "$record" ] || continue + path=$(cat -- "$record" && printf .) || return 1 + path=${path%.} + entry=${record%.path} + if ! present "$entry"; then + rm -- "$record" || return 1 + elif check_parents "$path" 2>/dev/null && [ ! -L "$path" ] && [ "$path" -ef "$entry" ]; then + rm -- "$record" "$entry" || return 1 + else + detached=1 + fi + done + for entry in "$1"/observed-*/entry; do + if [ -f "$entry" ] && [ ! -L "$entry" ]; then parked=1; break; fi + done +} +each_store collect_detached || exit +if [ "$parked" -eq 1 ]; then printf 'parked\n'; fi if [ "$detached" -eq 1 ]; then printf 'original-detached\n'; fi ''' -_RECOVERY_PARENT = _DEVICE + r''' +_RECOVERY_PARENT = _FILESYSTEM + r''' recovery_parent() { - worktree_device=$(device .) && storage_device=$(device "$1/") || return 1 + worktree_device=$(filesystem .) && storage_device=$(filesystem "$1/") || return 1 if [ "$worktree_device" = "$storage_device" ]; then printf '%s\n' "$1" return fi - cd -P -- .. && parent_device=$(device .) || return 1 + cd -P -- .. && parent_device=$(filesystem .) || return 1 if [ "$worktree_device" != "$parent_device" ]; then - printf 'no recovery directory outside the worktree on its filesystem\n' >&2 - return 1 + printf '%s\n' "$1" + return fi pwd -P } ''' -_RECOVER_PATHS = _COMPARE_ENTRIES + _CHECK_PARENTS + _DEVICE + _RESTORE_SAVED + r''' +_RECOVER_PATHS = _COMPARE_ENTRIES + _CHECK_PARENTS + _RESTORE_SAVED + _RECOVERY_STORES + r''' check_candidate_descendants() { prefix=${1#"$workspace_prefix"}; shift @@ -235,21 +305,33 @@ park_candidate_entry() { if ! present "$1"; then return 0; fi - if [ -f "$1" ] && [ ! -L "$1" ]; then - source_device=$(device "$1") && recovery_device=$(device "$recovery") || return 1 - if [ "$source_device" != "$recovery_device" ]; then - printf 'cannot park %s across filesystems\n' "$1" >&2 - return 2 - fi - fi - parked=$(mktemp -d "$recovery/observed-XXXXXXXX") || return 1 + parked=$(mktemp -d "$3/observed-XXXXXXXX") || return 1 + if [ -n "$4" ]; then printf '%s\n' "$4" > "$parked/mode" || return 1; fi printf '%s' "$1" > "$parked/path" && mv -- "$1" "$parked/entry" || return 1 - if ! same_entry "$parked/entry" "$2"; then + if ! same_entry "$parked/entry" "$2" "$4"; then restore_saved "$parked/entry" "$1" || return $? conflict "$1" fi } +project_file_mode() ( + source=$(mktemp "$recovery/mode-XXXXXXXX") || exit + expected_mode=$(mode "$1") && chmod -- "$expected_mode" "$source" || exit + sample=$(mktemp -d "$2/mode-XXXXXXXX") || exit + cp -pP -- "$source" "$sample/entry" && actual_mode=$(mode "$sample/entry") || exit + rm -- "$source" && rm -rf -- "$sample" || exit + printf '%s' "$actual_mode" +) + +restore_preimage() { + if ! present "$1"; then return 0; fi + store=${3:-} + if [ -z "$store" ]; then store=$(store_for "$2") || return 1; fi + if [ "$store" = "$recovery" ]; then restore_saved "$1" "$2"; return; fi + install=$(mktemp -d "$store/restore-XXXXXXXX") || return 1 + cp -pPR -- "$1" "$install/entry" && restore_saved "$install/entry" "$2" +} + restore_entry() ( old=$1 expected=$2 live=$3 if same_entry "$old" "$expected"; then @@ -290,27 +372,42 @@ return done rmdir -- "$live" || return 1 - restore_saved "$old" "$live" + restore_preimage "$old" "$live" fi return fi if same_entry "$live" "$old"; then return 0; fi + projected= if same_entry "$live" "$expected"; then - park_candidate_entry "$live" "$expected" || return $? - restore_saved "$old" "$live" + store=$(store_for "$live") || return 1 else - conflict "$live" + if [ ! -f "$live" ] || [ -L "$live" ] || [ ! -f "$expected" ] || [ -L "$expected" ]; then + conflict "$live" + return + fi + store=$(store_for "$live") || return 1 + if [ "$store" = "$recovery" ]; then conflict "$live"; return; fi + # A replay on the Git filesystem can represent modes unavailable on the live mount. + projected=$(project_file_mode "$expected" "$store") || return 1 + same_entry "$live" "$expected" "$projected" || { conflict "$live"; return; } fi + park_candidate_entry "$live" "$expected" "$store" "$projected" || return $? + restore_preimage "$old" "$live" "$store" ) ''' _RECHECK_PARKED = _COMPARE_ENTRIES + r''' -for parked in "$recovery"/observed-*; do - [ -d "$parked" ] || continue - path=$(cat -- "$parked/path" && printf .) || exit - path=${path%.} - same_entry "$parked/entry" "$staged/${path#"$workspace_prefix"}" || { conflict "$path"; exit; } -done +verify_parked() { + for parked in "$1"/observed-*; do + [ -d "$parked" ] || continue + path=$(cat -- "$parked/path" && printf .) || return 1 + path=${path%.} + projected= + if [ -f "$parked/mode" ]; then read -r projected < "$parked/mode" || return 1; fi + same_entry "$parked/entry" "$staged/${path#"$workspace_prefix"}" "$projected" || { conflict "$path"; return; } + done +} +each_store verify_parked || exit ''' @@ -431,24 +528,27 @@ async def allocate_backup() -> None: ), "candidate recovery storage").removesuffix("\n") async def remove_backup(path: str, *, retain_detached: bool = False) -> None: - quoted = shlex.quote(path) remove_entry = ( 'remove_entry() { rm -rf -- "$1" || ' '(find "$1" -type d ! -perm -u+w -exec chmod -- u+w {} + && rm -rf -- "$1"); }\n' ) + removal = ( + f"recovery={shlex.quote(path)}; staged={shlex.quote(posixpath.join(path, 'replay'))}; " + f"workspace_prefix={shlex.quote(workspace.rstrip('/') + '/')};\n" + + _RECOVERY_STORES + ) if retain_detached: - removal = ( - f"recovery={quoted}; staged={shlex.quote(posixpath.join(path, 'replay'))}; " - f"workspace_prefix={shlex.quote(workspace.rstrip('/') + '/')};\n" - + _RECHECK_PARKED + remove_entry + r''' -for entry in "$recovery"/*; do - case "${entry##*/}" in original-inodes|observed-*) continue;; esac - remove_entry "$entry" || exit -done + removal += remove_entry + _RECHECK_PARKED + r''' +prune_store() { + for entry in "$1"/*; do + case "${entry##*/}" in original-inodes|observed-*|volumes) continue;; esac + remove_entry "$entry" || return $? + done +} +each_store prune_store ''' - ) else: - removal = remove_entry + f"remove_entry {quoted}" + removal = remove_entry + removal + "each_store remove_entry" result = await environment.exec_cmd(removal, timeout=120) if retain_detached: _complete(result, "candidate recovery cleanup") @@ -533,8 +633,7 @@ async def remove_backup(path: str, *, retain_detached: bool = False) -> None: snapshot_script = ( workspace_vars + f"linker={shlex.quote(linker)};\n" + (f"link_script={shlex.quote(_LINK_ORIGINALS)};\n" if linker else "") - + _HOLD_ORIGINALS + f"hold_originals {' '.join(targets)} || exit; " - + "; ".join(copies) + + _HOLD_ORIGINALS + "; ".join((*copies, f"hold_originals {' '.join(targets)} || exit")) ) target_test = f'case "${{1#"$workspace_prefix"}}" in {"|".join(targets)}) return 0;; esac; ' if targets else "" recovery_script = ( @@ -554,7 +653,10 @@ async def remove_backup(path: str, *, retain_detached: bool = False) -> None: environment, workspace, recovery_script, repository=repository, )) snapshot = _complete( - await environment.exec_cmd(snapshot_command, timeout=900), "candidate recovery snapshot", + await await_owned_operation( + environment.exec_cmd(snapshot_command, timeout=900), propagate_cancellation=True, + ), + "candidate recovery snapshot", ) publication_started = False try: diff --git a/tests/workflows/test_candidate_adoption_mounts.py b/tests/workflows/test_candidate_adoption_mounts.py new file mode 100644 index 00000000..8c3c7fe8 --- /dev/null +++ b/tests/workflows/test_candidate_adoption_mounts.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import os +import shlex +from dataclasses import replace +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters.env import LocalEnvironment +from tests.workflows.test_candidate_adoption_recovery import _partial_publication +from tests.workflows.test_workflow_candidate_workspace import _git + + +def _mount_commands(repo: Path, mount: Path, *, bind: bool, hard_links: bool) -> str: + return ( + f"volume={shlex.quote(str(mount))}; root={shlex.quote(str(repo))}; " + f"volume_device={1 if bind else 2}; volume_links={int(hard_links)};\n" + + r''' +stat() { + case "$1:$2" in + -c:%d|-c:%d:%m) + mountpoint=$root; device_id=1 + case "$4" in "$volume"|"$volume"/*) mountpoint=$volume; device_id=$volume_device;; esac + if [ "$2" = %d ]; then printf '%s\n' "$device_id"; else printf '%s:%s\n' "$device_id" "$mountpoint"; fi + ;; + -c:%a) + case "$4" in + "$volume"/*) if [ "$volume_links" -eq 0 ]; then printf '755\n'; return; fi;; + esac + command stat "$@" + ;; + *) command stat "$@";; + esac +} +ln() { + if [ "$1" = -P ]; then + case "$3" in + "$volume"/*) + case "$4" in + "$volume"/*) [ "$volume_links" -eq 1 ] || return 1;; + *) return 1;; + esac + ;; + esac + fi + command ln "$@" +} +''' + ) + + +@pytest.mark.parametrize("scenario,bind,hard_links", [ + pytest.param("success", False, True, id="mixed-device-success"), + pytest.param("success", True, True, id="bind-mount-success"), + pytest.param("modified", False, True, id="restore-mounted-file"), + pytest.param("modified", False, False, id="restore-mounted-file-without-hard-links"), + pytest.param("added", True, True, id="park-new-file-on-bind-mount"), +]) +async def test_candidate_adoption_with_nested_mount(tmp_path, monkeypatch, scenario, bind, hard_links): + repo, candidate = _partial_publication(tmp_path) + mount = repo / "mounted" + mount.mkdir(exist_ok=True) + target = mount / "record.txt" + if scenario == "added": + header = "new file mode 100644\n--- /dev/null\n+++ b/mounted/record.txt\n@@ -0,0 +1 @@\n" + original = b"" + else: + original = b"original mount\n" + target.write_bytes(original) + _git(repo, "add", "mounted/record.txt") + _git(repo, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "commit", "-qm", "test: track mounted file") + header = "--- a/mounted/record.txt\n+++ b/mounted/record.txt\n@@ -1 +1 @@\n-original mount\n" + local_patch, blocked = candidate.diff.split("diff --git a/settings", 1) + diff = local_patch + "diff --git a/mounted/record.txt b/mounted/record.txt\n" + header + "+candidate mount\n" + if scenario != "success": + diff += "diff --git a/settings" + blocked + candidate = replace(candidate, diff=diff) + (repo / "user-note.txt").write_text("staged by user\n") + _git(repo, "add", "user-note.txt") + index = repo / ".git/index" + index_before = index.read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + original_fd = os.open(target, os.O_WRONLY | os.O_APPEND) if scenario == "modified" else None + published_fd = None + statuses = [] + + async def observe(command, timeout=120, **kwargs): + nonlocal published_fd + if "filesystem()" in command or "device()" in command: + command = _mount_commands(repo, mount, bind=bind, hard_links=hard_links) + command + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command and ".candidate-adopt-" in command: + statuses.append(result.returncode) + if scenario != "success": + assert target.read_bytes() == b"candidate mount\n" + published_fd = os.open(target, os.O_WRONLY | os.O_APPEND) + return result + + monkeypatch.setattr(environment, "exec_cmd", observe) + + async def flow(ctx, _): + await ctx.adopt_candidate(candidate) + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow(flow, trace=False) + assert index.read_bytes() == index_before + assert (repo / "settings/notes.txt").read_bytes() == b"existing settings\n" + if scenario == "success": + assert result.ok, result.reason + assert statuses == [0] + assert target.read_bytes() == b"candidate mount\n" + assert (repo / "record.txt").read_bytes() == b"candidate\n" + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not list(mount.glob(".candidate-recovery-*")) + return + + assert type(result.error) is RuntimeError and "candidate adoption failed" in result.reason, result.reason + assert statuses == [128] + assert (repo / "record.txt").read_bytes() == b"original\n" + if original: + assert target.read_bytes() == original + else: + assert not target.exists() + recovery, = (repo / ".git").glob("candidate-recovery-*") + location, = (recovery / "volumes").iterdir() + store = Path(location.read_text()) + assert store.parent == mount + assert not (recovery / "replay").exists() + assert not list(store.glob("restore-*")) + assert published_fd is not None + parked, = store.glob("observed-*/entry") + assert parked.with_name("path").read_text() == str(target) + retained = [(published_fd, parked, b"candidate mount\n")] + if original_fd is not None: + record, = store.glob("original-inodes/[0-9]*.path") + assert record.read_text() == str(target) + retained.append((original_fd, record.with_suffix(""), original)) + for fd, saved, initial in retained: + os.write(fd, b"late write\n") + os.fsync(fd) + assert os.fstat(fd).st_ino == saved.stat().st_ino + assert saved.read_bytes() == initial + b"late write\n" + finally: + if original_fd is not None: + os.close(original_fd) + if published_fd is not None: + os.close(published_fd) + await environment.cleanup() From a29e6c72c74a7b935eee6b7b557f6fe11986c9fd Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 15:11:14 +0800 Subject: [PATCH 54/82] fix(llm): preserve request admission through decorated clients Honor admission callbacks consumed by explicit or delegated middleware. When a legacy wrapper reaches the native pool through an inherited capability, carry the deadline start hook in the owned call without passing a new keyword through its signature. Keep both generation and summary timeouts anchored to admission. Verify queueing, middleware preparation, cancellation, caller deadlines, and late usage while preserving revalidation and legacy forwarding behavior. --- opencollab/adapters/llm/client.py | 7 +- .../application/_session_run_completion.py | 67 +++++-- opencollab/application/ports.py | 10 +- .../runtime/test_pooled_generation_timeout.py | 179 +++++++++++++++++- 4 files changed, 234 insertions(+), 29 deletions(-) diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index d49ac4a3..4f6fed2c 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -40,7 +40,7 @@ from opencollab.adapters.llm.types import LLMResponse, model_context_window from opencollab.adapters.llm.usage_ledger import record_api_usage from opencollab.application.async_timeout import await_owned_operation -from opencollab.application.ports import _NATIVE_REQUEST_REVALIDATION +from opencollab.application.ports import _NATIVE_REQUEST_ADMISSION, _NATIVE_REQUEST_REVALIDATION, _native_admission from opencollab.domain.token_estimation import estimate_request_tokens # Native data-residency endpoints share the OpenAI model parameter rules. @@ -80,6 +80,7 @@ class LLMClient: """ supports_request_revalidation = _NATIVE_REQUEST_REVALIDATION + supports_request_admission = _NATIVE_REQUEST_ADMISSION def __init__( self, @@ -126,7 +127,6 @@ def __init__( raise ValueError("request_pool must be an absolute local directory path") self.request_pool = request_pool self.request_capacity = request_capacity - self.supports_request_admission = request_pool is not None self.supports_attempt_usage = True self.supports_usage_settlement = True self.model = model @@ -352,6 +352,9 @@ def prepare_attempt() -> int | None: try: await request_owner.enter_async_context(request_slot(self.request_pool, self.request_capacity)) initial_limit = request_limit() + admission = _native_admission.get() if self.request_pool is not None else None + if admission is not None and admission[0] is self: + admission[1]() if on_admitted is not None: on_admitted() if self._anthropic: diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index 11977636..a2b71ef1 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -36,12 +36,14 @@ from opencollab.application._tool_loop_execution import _apply_completed_prefix_progress from opencollab.application.async_timeout import CallerTimeoutError, abandon_on_timeout from opencollab.application.ports import ( + _NATIVE_REQUEST_ADMISSION, _NATIVE_REQUEST_REVALIDATION, CompletionResponse, ContextTokenEstimatorPort, RequestOutputRequirementPort, RequestTokenEstimatorPort, ThinkingPrefixPort, + _native_admission, ) from opencollab.application.shaping import ShaperPipeline from opencollab.application.shaping.pipeline import context_token_estimator @@ -60,6 +62,15 @@ logger = logging.getLogger(__name__) +async def _observe_native_admission( + operation: Awaitable[CompletionResponse], owner: object, admitted: asyncio.Event, +) -> CompletionResponse: + token = _native_admission.set((owner, admitted.set)) + try: + return await operation + finally: + _native_admission.reset(token) + def _completion_keyword_parameters(complete: Callable[..., Awaitable[Any]]) -> tuple[set[str] | None, bool]: keywords: set[str] = set() outer_accepts_kwargs = False @@ -795,35 +806,50 @@ def _add_completion_controls( max_output_tokens: int, usage_accounting: _CompletionUsageAccounting, revalidate: Callable[[], int], - ) -> asyncio.Event | None: + ) -> tuple[asyncio.Event | None, object | None]: keywords, outer_accepts_kwargs = _completion_keyword_parameters(complete) def accepts(name: str) -> bool: return keywords is None or name in keywords + def accepts_control( + name: str, capability: str, native_support: object, *, follow_delegate: bool = False, + ) -> bool: + supported = getattr(self.llm, capability, False) + if not supported or accepts(name): + return bool(supported) + if not outer_accepts_kwargs or (not follow_delegate and supported is not True): + return False + # Explicit middleware opt-ins can consume the callback before forwarding. + fallback = native_support + if follow_delegate: + # A transparent proxy can return a method owned by a native client. + fallback = inspect.getattr_static(getattr(complete, "__self__", None), capability, None) + return inspect.getattr_static(self.llm, capability, fallback) is not native_support + if max_output_tokens != DEFAULT_MAX_TOKENS_PER_STEP or accepts("max_output_tokens"): kwargs["max_output_tokens"] = max_output_tokens if getattr(self.llm, "supports_attempt_usage", False) and accepts("on_attempt_usage"): kwargs["on_attempt_usage"] = usage_accounting.record_attempt if getattr(self.llm, "supports_usage_settlement", False) and accepts("on_usage"): kwargs["on_usage"] = usage_accounting.settle - revalidation = getattr(self.llm, "supports_request_revalidation", False) - # Explicit middleware opt-ins can consume the callback before forwarding. - if revalidation and ( - accepts("before_request") - or ( - outer_accepts_kwargs and revalidation is True - and inspect.getattr_static( - self.llm, "supports_request_revalidation", _NATIVE_REQUEST_REVALIDATION, - ) is not _NATIVE_REQUEST_REVALIDATION - ) - ): + if accepts_control("before_request", "supports_request_revalidation", _NATIVE_REQUEST_REVALIDATION): kwargs["before_request"] = revalidate - if getattr(self.llm, "supports_request_admission", False) and accepts("on_admitted"): + if accepts_control( + "on_admitted", "supports_request_admission", _NATIVE_REQUEST_ADMISSION, follow_delegate=True, + ): admitted = asyncio.Event() kwargs["on_admitted"] = admitted.set - return admitted - return None + return admitted, None + owner = getattr(complete, "__self__", None) + # A variadic wrapper can reach the native pool through a legacy signature. + if ( + self._per_call_timeout is not None and outer_accepts_kwargs + and getattr(self.llm, "supports_request_admission", False) + and inspect.getattr_static(owner, "supports_request_admission", None) is _NATIVE_REQUEST_ADMISSION + ): + return asyncio.Event(), owner + return None, None def _revalidate_request( self, messages: list[dict], tools: list[dict] | None, *, thinking: bool = False, @@ -859,7 +885,7 @@ def revalidate() -> int: ) return min(limit, _summary_output_limit(messages)) if _summary_output_limit is not None else limit - admitted = self._add_completion_controls( + admitted, native_owner = self._add_completion_controls( self.llm.complete, kwargs, max_output_tokens=max_output_tokens, usage_accounting=usage_accounting, revalidate=revalidate, ) @@ -895,7 +921,10 @@ async def complete_owned() -> CompletionResponse: self._draining_provider_tasks.discard(asyncio.current_task()) self._finish_run_control_if_quiesced() - owner = asyncio.create_task(complete_owned()) + operation = complete_owned() + if native_owner is not None and admitted is not None: + operation = _observe_native_admission(operation, native_owner, admitted) + owner = asyncio.create_task(operation) self._track_provider_task(owner) try: if self._per_call_timeout is None: @@ -932,7 +961,7 @@ async def _invoke_llm(self, *, max_output_tokens: int, **kwargs: Any) -> Complet protected_call = self.state.wind_down_done usage_accounting = _CompletionUsageAccounting(self, protected_call=protected_call) complete = self.llm.complete - admitted = self._add_completion_controls( + admitted, native_owner = self._add_completion_controls( complete, kwargs, max_output_tokens=max_output_tokens, usage_accounting=usage_accounting, revalidate=lambda: self._revalidate_request( kwargs["messages"], kwargs.get("tools"), thinking=kwargs.get("thinking", False), @@ -964,6 +993,8 @@ def record_abandoned(task: asyncio.Task[Any]) -> None: self._record_late_provider_result(task, protected_call=protected_call, usage_accounted=True) operation = complete_owned() + if native_owner is not None and admitted is not None: + operation = _observe_native_admission(operation, native_owner, admitted) try: response = await operation if self._per_call_timeout is None else await abandon_on_timeout( operation, diff --git a/opencollab/application/ports.py b/opencollab/application/ports.py index ef6b7023..52c96021 100644 --- a/opencollab/application/ports.py +++ b/opencollab/application/ports.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Mapping, Sequence +from contextvars import ContextVar from typing import TYPE_CHECKING, Any, Awaitable, Callable, Protocol, runtime_checkable from opencollab.domain.hooks import HookOutcome @@ -475,8 +476,15 @@ def __get__(self, instance: object, owner: type) -> bool: return True -# Native clients support the callback through their completion signature. +class _NativeRequestAdmission: + def __get__(self, instance: Any, owner: type) -> bool: + return instance is not None and instance.request_pool is not None + + +# Native clients support these callbacks through their completion signature. _NATIVE_REQUEST_REVALIDATION = _NativeRequestRevalidation() +_NATIVE_REQUEST_ADMISSION = _NativeRequestAdmission() +_native_admission: ContextVar[tuple[object, Callable[[], None]] | None] = ContextVar("native_admission", default=None) class LLMPort(Protocol): diff --git a/tests/runtime/test_pooled_generation_timeout.py b/tests/runtime/test_pooled_generation_timeout.py index 8d3b09f7..1275ab87 100644 --- a/tests/runtime/test_pooled_generation_timeout.py +++ b/tests/runtime/test_pooled_generation_timeout.py @@ -3,6 +3,7 @@ import asyncio import json import threading +from functools import wraps import pytest @@ -18,9 +19,62 @@ from tests.support.environment_test_support import local_environment_factory as local_environment_factory +class _LegacyPooledClient(LLMClient): + @wraps(LLMClient.complete) + async def complete(self, messages, tools=None, temperature=0.0, *, max_output_tokens=None): + return await super().complete(messages, tools, temperature, max_output_tokens=max_output_tokens) + + +class _ForwardingPooledClient(_LegacyPooledClient): + @wraps(_LegacyPooledClient.complete) + async def complete(self, *args, **kwargs): + return await super().complete(*args, **kwargs) + + +class _DelegatingPooledClient(_LegacyPooledClient): + @wraps(_LegacyPooledClient.complete) + async def complete(self, *args, **kwargs): + return await self.delegate(*args, **kwargs) + + +class _AdmissionMiddleware(_LegacyPooledClient): + supports_request_admission = True + + @wraps(_LegacyPooledClient.complete) + async def complete(self, *args, **kwargs): + on_admitted = kwargs.pop("on_admitted", None) + return await LLMClient.complete(self, *args, on_admitted=on_admitted, **kwargs) + + +class _PreparingPooledClient(_LegacyPooledClient): + @wraps(_LegacyPooledClient.complete) + async def complete(self, *args, **kwargs): + await asyncio.sleep(0.15) + return await LLMClient.complete(self, *args, **kwargs) + + +class _AdmissionProxy: + __slots__ = ("client",) + + def __init__(self, client): + self.client = client + + def __getattr__(self, name): + return getattr(self.client, name) + + +@pytest.fixture( + params=[LLMClient, _AdmissionMiddleware, _ForwardingPooledClient], + ids=["native", "decorated", "inherited"], +) +def pooled_client_class(request): + return request.param + + @pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("proxy", [False, True], ids=["direct-client", "attribute-proxy"]) async def test_queue_wait_leaves_the_full_provider_timeout( - tmp_path, monkeypatch, local_environment_factory, summary, + tmp_path, monkeypatch, local_environment_factory, summary, proxy, pooled_client_class, ): calls = [] @@ -30,11 +84,11 @@ async def complete(*_args, **_kwargs): return LLMResponse(content="Earlier context." if summary and len(calls) == 1 else "answer") monkeypatch.setattr(transport, "complete_openai", complete) - async with LLMClient( + async with pooled_client_class( api_key="local", request_pool=str(tmp_path), request_capacity=1, context_window=40_000, ) as client: session = build_session( - agent=Agent(name="queued", system_prompt="system"), llm=client, + agent=Agent(name="queued", system_prompt="system"), llm=_AdmissionProxy(client) if proxy else client, env=local_environment_factory(str(tmp_path)), llm_timeout=0.1, ) session.messages = history(summary) @@ -50,7 +104,7 @@ async def complete(*_args, **_kwargs): @pytest.mark.parametrize("summary", [False, True]) async def test_admitted_timeout_retains_slot_and_late_usage_until_cleanup( - tmp_path, monkeypatch, local_environment_factory, summary, + tmp_path, monkeypatch, local_environment_factory, summary, pooled_client_class, ): cancelled = asyncio.Event() release = asyncio.Event() @@ -67,7 +121,7 @@ async def complete(*_args, **_kwargs): return LLMResponse(content="Earlier context.", usage=Usage(13, 24)) monkeypatch.setattr(transport, "complete_openai", complete) - async with LLMClient( + async with pooled_client_class( api_key="local", request_pool=str(tmp_path), request_capacity=1, context_window=40_000, ) as client: session = build_session( @@ -102,7 +156,7 @@ async def complete(*_args, **_kwargs): @pytest.mark.parametrize("summary", [False, True]) @pytest.mark.parametrize("stop", ["cancel", "deadline"]) async def test_outer_cancellation_removes_the_waiter_before_admission( - tmp_path, monkeypatch, local_environment_factory, summary, stop, + tmp_path, monkeypatch, local_environment_factory, summary, stop, pooled_client_class, ): calls = [] @@ -111,7 +165,7 @@ async def complete(*_args, **_kwargs): return LLMResponse(content="answer") monkeypatch.setattr(transport, "complete_openai", complete) - async with LLMClient( + async with pooled_client_class( api_key="local", request_pool=str(tmp_path), request_capacity=1, context_window=40_000, ) as client: session = build_session( @@ -128,8 +182,9 @@ async def complete(*_args, **_kwargs): with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(owner, 1) else: - with pytest.raises(asyncio.TimeoutError): + with pytest.raises(asyncio.TimeoutError) as timeout: await asyncio.wait_for(owner, 0.12) + assert type(timeout.value) is asyncio.TimeoutError await asyncio.wait_for(asyncio.gather(*session.pending_cleanup_tasks, return_exceptions=True), 1) await asyncio.sleep(0.1) assert calls == [] @@ -139,6 +194,114 @@ async def complete(*_args, **_kwargs): await session.aclose() +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("proxy", [False, True], ids=["direct-client", "attribute-proxy"]) +async def test_legacy_pooled_forwarder_keeps_its_wrapped_signature( + tmp_path, monkeypatch, local_environment_factory, summary, proxy, +): + calls = [] + + async def complete(*_args, **_kwargs): + calls.append(True) + return LLMResponse( + content="Earlier context." if summary and len(calls) == 1 else "answer", + usage=Usage(7, 3), + ) + + monkeypatch.setattr(transport, "complete_openai", complete) + async with _ForwardingPooledClient( + api_key="local", request_pool=str(tmp_path), request_capacity=1, context_window=40_000, + ) as client: + assert client.supports_request_admission is True + session = build_session( + agent=Agent(name="legacy", system_prompt="system"), llm=_AdmissionProxy(client) if proxy else client, + env=local_environment_factory(str(tmp_path)), llm_timeout=1, + ) + session.messages = history(summary) + try: + assert await session.run_loop() == "answer" + assert len(calls) == (2 if summary else 1) + assert session.used_tokens == 10 * len(calls) + finally: + await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +async def test_inherited_admission_allows_middleware_preparation_before_pool( + tmp_path, monkeypatch, local_environment_factory, summary, +): + calls = [] + + async def complete(*_args, **_kwargs): + calls.append(True) + return LLMResponse(content="Earlier context." if summary and len(calls) == 1 else "answer") + + monkeypatch.setattr(transport, "complete_openai", complete) + async with _PreparingPooledClient( + api_key="local", request_pool=str(tmp_path), request_capacity=1, context_window=40_000, + ) as client: + session = build_session( + agent=Agent(name="middleware", system_prompt="system"), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.08, + ) + session.messages = history(summary) + try: + assert await asyncio.wait_for(session.run_loop(), 2) == "answer" + assert len(calls) == (2 if summary else 1) + finally: + await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("bypass", ["returns", "hangs", "other-client", "nonpool"]) +async def test_inherited_admission_waiting_preserves_completion_and_caller_deadline( + tmp_path, monkeypatch, local_environment_factory, summary, bypass, +): + calls = [] + provider_calls = [] + + async def complete(*_args, **_kwargs): + provider_calls.append(True) + await asyncio.Event().wait() + + async def delegate(messages, tools=None, temperature=0.0, *, max_output_tokens=None): + calls.append(True) + if bypass == "returns": + content = "Earlier context." if summary and len(calls) == 1 else "answer" + return LLMResponse(content=content, usage=Usage(7, 3)) + if bypass == "other-client": + return await other.complete(messages, tools, temperature, max_output_tokens=max_output_tokens) + await asyncio.Event().wait() + + monkeypatch.setattr(transport, "complete_openai", complete) + config = dict(api_key="local", context_window=40_000) + pool_config = dict(request_pool=str(tmp_path), request_capacity=1) + client_pool = {} if bypass == "nonpool" else pool_config + async with _DelegatingPooledClient(**config, **client_pool) as client, LLMClient(**config, **pool_config) as other: + client.delegate = delegate + session = build_session( + agent=Agent(name="legacy", system_prompt="system"), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.1, + ) + session.messages = history(summary) + try: + if bypass == "returns": + assert await asyncio.wait_for(session.run_loop(), 2) == "answer" + assert len(calls) == (2 if summary else 1) + assert session.used_tokens == 10 * len(calls) + else: + expected_timeout = GenerationTimeoutError if bypass == "nonpool" else asyncio.TimeoutError + with pytest.raises(expected_timeout) as timeout: + await asyncio.wait_for(session.run_loop(), 0.25) + assert type(timeout.value) is expected_timeout + assert calls == [True] + assert session.used_tokens == 0 + await asyncio.gather(*session.pending_cleanup_tasks, return_exceptions=True) + assert provider_calls == ([True] if bypass == "other-client" else []) + finally: + await session.aclose() + + @pytest.mark.parametrize("summary", [False, True]) @pytest.mark.parametrize("provider_error", [False, True]) async def test_blocked_ledger_releases_slot_and_preserves_late_session_usage( From a6398277831af5e1801c0b94fadf97ce7e6b4ea3 Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 17:08:35 +0800 Subject: [PATCH 55/82] fix(workflow): find writable linked worktree recovery storage Choose a writable recovery directory on the worktree filesystem when common Git metadata is read-only. Prefer a location outside the linked checkout so detached files remain writable after the checkout is removed. --- docs/duo.md | 10 +-- docs/duo/README.zh-CN.md | 2 +- opencollab/adapters/_candidate_adoption.py | 45 ++++++++++--- ...test_candidate_adoption_external_writer.py | 66 +++++++++++++++++-- 4 files changed, 101 insertions(+), 22 deletions(-) diff --git a/docs/duo.md b/docs/duo.md index fbfc37cb..b9591e64 100644 --- a/docs/duo.md +++ b/docs/duo.md @@ -207,10 +207,12 @@ user edits and the source index. A conflicting patch leaves the current source i If a partially applied patch is rolled back, OpenCollab retains detached files that another process may still be writing to. The error or warning gives the `candidate-recovery-*` directory, typically in the common Git directory. -Its `volumes/` records, when present, name additional recovery directories -on affected mounts. The adjacent `path` or `.path` record identifies each -file's original location. Close the writers and recover their data before -removing the recovery directories or worktree. +When shared Git metadata is read-only, OpenCollab prefers a writable location +on the same filesystem outside the worktree so the files survive worktree removal. +When present, `volumes/` records list additional recovery directories on affected +mounts. The adjacent `path` or `.path` record identifies each file's original +location. Close the writers and recover their data before removing the recovery +directories or worktree. Changes to ignore rules preserve files copied from the source and files added to the candidate index or commits. Newly created ignored files join delivery when explicitly added with `git add --force`. diff --git a/docs/duo/README.zh-CN.md b/docs/duo/README.zh-CN.md index 63f23c3e..58e22b6e 100644 --- a/docs/duo/README.zh-CN.md +++ b/docs/duo/README.zh-CN.md @@ -89,7 +89,7 @@ Duo 通过现有候选工作区接口完成隔离、变更捕获与采用。默 Git 候选从源目录当前受跟踪文件和未被忽略的未跟踪文件开始,交付补丁表示候选随后产生的增量。采用时将增量应用到当前源文件,保留独立用户修改和源索引,冲突会保留当前源目录。修改忽略规则后,已从源目录复制的文件和候选中已加入索引或提交的文件仍进入交付。新建的忽略文件经 `git add --force` 加入后进入交付。 -补丁部分应用后回滚时,OpenCollab 会保留其他进程可能仍在写入的文件。错误或警告会给出 `candidate-recovery-*` 目录,通常位于 Git 公共目录下。如果包含 `volumes/`,该目录中的记录列出受影响挂载卷上的其他恢复目录。相邻的 `path` 或 `.path` 记录标明文件原来的位置。移除恢复目录或工作树前,请等待写入进程关闭文件,并恢复其中的数据。 +补丁部分应用后回滚时,OpenCollab 会保留其他进程可能仍在写入的文件。错误或警告会给出 `candidate-recovery-*` 目录,通常位于 Git 公共目录下。Git 公共目录只读时,OpenCollab 优先选择同一文件系统上工作树外的可写目录,让恢复文件在移除工作树后得以保留。如果包含 `volumes/`,该目录中的记录列出受影响挂载卷上的其他恢复目录。相邻的 `path` 或 `.path` 记录标明文件原来的位置。移除恢复目录或工作树前,请等待写入进程关闭文件,并恢复其中的数据。 候选运行期间源目录发生变化,运行时会报告错误并保留候选工作树。错误包含初始 Git tree,工作树中的 `refs/worktree/opencollab-source` 继续引用它,可以通过 `git show :` 恢复文件。后端提供源版本时,`CandidateRun.source_revision` 记录源 `HEAD`,采用前再次核对。源 diff 相同而 `HEAD` 已变化时,采用也会被拒绝。 diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index d7f2d982..078b2812 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -238,18 +238,44 @@ ''' _RECOVERY_PARENT = _FILESYSTEM + r''' +writable_directory() { [ -d "$1" ] && [ -w "$1" ] && [ -x "$1" ]; } + recovery_parent() { worktree_device=$(filesystem .) && storage_device=$(filesystem "$1/") || return 1 - if [ "$worktree_device" = "$storage_device" ]; then + if [ "$worktree_device" = "$storage_device" ] && writable_directory "$1"; then printf '%s\n' "$1" return fi - cd -P -- .. && parent_device=$(filesystem .) || return 1 - if [ "$worktree_device" != "$parent_device" ]; then + workspace=$(pwd -P && printf .) || return 1 + workspace=${workspace%??} + parent=$workspace + while [ "$parent" != / ]; do + parent=${parent%/*} + parent=${parent:-/} + parent_device=$(filesystem "$parent") || return 1 + [ "$worktree_device" = "$parent_device" ] || break + if writable_directory "$parent"; then + printf '%s\n' "$parent" + return + fi + done + if [ -n "${2:-}" ]; then + temporary=$(cd -P -- "${TMPDIR:-/tmp}" >/dev/null 2>&1 && pwd -P && printf .) || temporary= + temporary=${temporary%??} + case "$temporary/" in + "${workspace%/}/"*|"${2%/}/"*) ;; + *) + if writable_directory "$temporary" && [ "$(filesystem "$temporary")" = "$worktree_device" ]; then + printf '%s\n' "$temporary" + return + fi ;; + esac + fi + if writable_directory "$1"; then printf '%s\n' "$1" - return + else + printf '%s\n' "${2:-$1}" fi - pwd -P } ''' @@ -504,19 +530,18 @@ async def _adoption_transaction( git_directory = None directory_script = f"cd -- {shlex.quote(workspace)} || exit\n" + _RECOVERY_PARENT if repository: - # Git removes private metadata with a linked worktree. A worktree on - # another volume needs a recovery directory beside it on that volume. + # Git removes private metadata with the linked worktree. directory_script += r''' -git rev-parse --absolute-git-dir && printf '\0' || exit +private=$(git rev-parse --absolute-git-dir && printf .) || exit +printf '%s\0' "${private%??}" || exit common=$(git rev-parse --git-common-dir && printf .) || exit -recovery_parent "${common%??}" +recovery_parent "${common%??}" "${private%??}" ''' else: directory_script += 'recovery_parent "${TMPDIR:-/tmp}"' recovery_parent = _complete(await environment.exec_cmd(directory_script, timeout=30), "candidate recovery storage") if repository: git_directory, recovery_parent = recovery_parent.split("\0") - git_directory = git_directory.removesuffix("\n") storage = shlex.quote(posixpath.join(workspace, recovery_parent.removesuffix("\n"))) backup = None keep_backup = False diff --git a/tests/workflows/test_candidate_adoption_external_writer.py b/tests/workflows/test_candidate_adoption_external_writer.py index 12e0cc38..fd9b67bd 100644 --- a/tests/workflows/test_candidate_adoption_external_writer.py +++ b/tests/workflows/test_candidate_adoption_external_writer.py @@ -3,6 +3,7 @@ import asyncio import os import shlex +import stat import sys import tempfile from dataclasses import replace @@ -48,6 +49,46 @@ def _delete_original(candidate): )) +async def test_candidate_adopts_with_readonly_common_git_directory(tmp_path): + source, candidate = _partial_publication(tmp_path) + repo = tmp_path / "linked" + _git(source, "worktree", "add", "--detach", str(repo)) + (repo / "user-note.txt").write_text("staged user note\n") + _git(repo, "add", "--", "user-note.txt") + index = Path(_git(repo, "rev-parse", "--absolute-git-dir").removesuffix("\n")) / "index" + index_before = index.read_bytes() + common = source / ".git" + common_mode = stat.S_IMODE(common.stat().st_mode) + target = repo / "record.txt" + environment = LocalEnvironment(str(repo)) + + async def flow(ctx, _): + await ctx.adopt_candidate(candidate) + + try: + common.chmod(0o555) + if os.access(common, os.W_OK): + pytest.skip("requires an account subject to directory permissions") + assert os.access(index.parent, os.W_OK) and os.access(repo, os.W_OK) + with target.open("rb") as original: + result = await OpenCollab(repo, environment=environment, model="offline").workflow( + flow, trace=False, timeout=30, + ) + assert result.ok, result.reason + assert target.read_bytes() == b"candidate\n" + assert (repo / "settings").read_bytes() == b"replacement settings\n" + assert index.read_bytes() == index_before + assert original.read() == b"original\n" + assert os.fstat(original.fileno()).st_nlink == 0 + assert stat.S_IMODE(common.stat().st_mode) == 0o555 + for storage in (common, index.parent, tmp_path, repo): + assert not list(storage.glob("*candidate-recovery-*")) + assert not list(index.parent.glob("opencollab-candidate-*")) + finally: + common.chmod(common_mode) + await environment.cleanup() + + @pytest.fixture def worktree_parent(tmp_path, request): if not request.param: @@ -62,14 +103,16 @@ def worktree_parent(tmp_path, request): yield Path(path) -@pytest.mark.parametrize("opened_before_apply,linked,worktree_parent,hard_links", [ - pytest.param(True, False, False, True, id="original-inode"), - pytest.param(False, True, False, True, id="candidate-inode-after-worktree-removal"), - pytest.param(False, True, True, True, id="separate-worktree-filesystem"), - pytest.param(True, True, False, False, id="original-inode-without-hard-links"), +@pytest.mark.parametrize("opened_before_apply,linked,worktree_parent,hard_links,readonly_common", [ + pytest.param(True, False, False, True, False, id="original-inode"), + pytest.param(False, True, False, True, False, id="candidate-inode-after-worktree-removal"), + pytest.param(False, True, True, True, False, id="separate-worktree-filesystem"), + pytest.param(True, True, False, False, False, id="original-inode-without-hard-links"), + pytest.param(True, True, False, True, True, id="readonly-common-original-inode"), + pytest.param(False, True, False, True, True, id="readonly-common-candidate-inode"), ], indirect=["worktree_parent"]) async def test_candidate_rollback_retains_inode_for_external_writer( - tmp_path, monkeypatch, caplog, opened_before_apply, linked, worktree_parent, hard_links, + tmp_path, monkeypatch, caplog, opened_before_apply, linked, worktree_parent, hard_links, readonly_common, ): source, candidate = _partial_publication(tmp_path) repo = source @@ -87,6 +130,8 @@ async def test_candidate_rollback_retains_inode_for_external_writer( appended = b"external write after rollback\n" index = Path(_git(repo, "rev-parse", "--absolute-git-dir").removesuffix("\n")) / "index" index_before = index.read_bytes() + common = source / ".git" + common_mode = stat.S_IMODE(common.stat().st_mode) environment = LocalEnvironment(str(repo)) execute = environment.exec_cmd writer = None @@ -126,6 +171,11 @@ async def flow(ctx, _): await ctx.adopt_candidate(candidate) try: + if readonly_common: + common.chmod(0o555) + if os.access(common, os.W_OK): + pytest.skip("requires an account subject to directory permissions") + assert os.access(index.parent, os.W_OK) and os.access(repo, os.W_OK) if opened_before_apply: await start_writer(original) result = await OpenCollab(repo, environment=environment, model="offline").workflow( @@ -150,7 +200,7 @@ async def flow(ctx, _): assert stdout == b"written\n" assert (source / "record.txt").read_bytes() == original - storage = source / ".git" if worktree_parent == tmp_path else worktree_parent + storage = common if worktree_parent == tmp_path and not readonly_common else worktree_parent backups = list(storage.glob("candidate-recovery-*")) assert backups, "candidate rollback left no recovery storage for the open descriptor" backup, = backups @@ -168,6 +218,8 @@ async def flow(ctx, _): assert any(str(backup) in message for message in caplog.messages) assert any(str(backup) in note for note in getattr(result.error, "__notes__", ())) finally: + if readonly_common: + common.chmod(common_mode) if writer is not None and writer.returncode is None: writer.kill() await writer.communicate() From b69a2333a73f8425877475bef932e91a8e6012a8 Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 17:08:36 +0800 Subject: [PATCH 56/82] fix(session): preserve legacy journal autosaves during compaction Use checkpoint-first persistence only for stores that support durable snapshot replacement. Retain journal-first autosaves and their established error handling for the existing journal interface. --- opencollab/application/session.py | 20 ++-- .../test_anthropic_preserved_history.py | 94 +++++++++++++++++++ 2 files changed, 106 insertions(+), 8 deletions(-) diff --git a/opencollab/application/session.py b/opencollab/application/session.py index 92b982fe..5326eaa0 100644 --- a/opencollab/application/session.py +++ b/opencollab/application/session.py @@ -1088,9 +1088,13 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: replace_from = index break # A full native rewrite can exceed the journal's remaining capacity. - checkpoint_first = has_native_history and replace_from == 0 and ( - preceding_write_pending or self._auto_save_message_count > 0 - ) + replacement_store = None + if ( + has_native_history and replace_from == 0 + and (preceding_write_pending or self._auto_save_message_count > 0) + and isinstance(self.store, SnapshotReplacementStorePort) + ): + replacement_store = self.store messages, meta = self._snapshot_for_save(replace_from) tracked_messages = [*stored_messages[:replace_from], *messages] if track_messages else None current_seen_hashes = set(self.state.turn.seen_result_hashes) @@ -1103,7 +1107,7 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: else current_seen_hashes - self._auto_save_seen_result_hashes ) checkpoint_messages = None - if checkpoint_first or sequence >= self._next_auto_save_checkpoint: + if replacement_store is not None or sequence >= self._next_auto_save_checkpoint: checkpoint_messages = tracked_messages if checkpoint_messages is None: checkpoint_messages = copy.deepcopy(self.state.enriched_messages()) @@ -1117,7 +1121,7 @@ def advance_cursor() -> None: self._auto_save_seen_result_hashes = current_seen_hashes def persist() -> None: - if not checkpoint_first: + if replacement_store is None: self.store.append_snapshot_delta( path, sequence=sequence, @@ -1145,10 +1149,10 @@ def persist() -> None: sequence=sequence, ) except (OSError, SnapshotCapacityError) as checkpoint_error: - if not checkpoint_first or not isinstance(self.store, SnapshotReplacementStorePort): + if replacement_store is None: raise try: - saved = self.store.append_snapshot_replacement( + saved = replacement_store.append_snapshot_replacement( path, checkpoint_messages, meta=meta, @@ -1162,7 +1166,7 @@ def persist() -> None: advance_cursor() return with self._auto_save_lock: - if checkpoint_first: + if replacement_store is not None: advance_cursor() self._next_auto_save_checkpoint = 1 << sequence.bit_length() diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index 5dde2390..32a64d65 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -864,6 +864,100 @@ def compact_journal(target, payload, **options): await environment.cleanup() +@pytest.mark.parametrize( + ("base_sequence", "failed_write", "expected_writes"), + [ + (4, "checkpoint", ["append"]), + (7, None, ["append", "checkpoint"]), + (7, "checkpoint", ["append", "checkpoint"]), + (7, "append", ["append"]), + ], + ids=["between-checkpoints", "periodic-checkpoint", "checkpoint-error", "append-error"], +) +async def test_legacy_journal_persists_native_rewrites_before_checkpoint( + monkeypatch, tmp_path, base_sequence, failed_write, expected_writes, +): + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + path = str(tmp_path / "session.json") + backend = storage_module.SessionStore() + writes = [] + sequence = base_sequence + 1 + write_error = OSError(f"controlled {failed_write} failure") + + class LegacyJournalStore: + save = backend.save + load_messages = backend.load_messages + load_snapshot = backend.load_snapshot + save_manifest = backend.save_manifest + has_snapshot = backend.has_snapshot + + def append_snapshot_delta(self, path, **delta): + if delta["sequence"] == sequence: + writes.append("append") + assert delta["replace_from"] == 0 + if failed_write == "append": + raise write_error + backend.append_snapshot_delta(path, **delta) + + def checkpoint_snapshot(self, path, messages, *, meta, sequence): + if sequence == base_sequence + 1: + writes.append("checkpoint") + if failed_write == "checkpoint": + raise write_error + backend.checkpoint_snapshot(path, messages, meta=meta, sequence=sequence) + + held_request, resume = asyncio.Event(), asyncio.Event() + peer = _BoundThinkingProvider(reads=1, thinking=False) + + async def provider(request): + response = peer(request) + if len(peer.requests) == 2: + held_request.set() + await resume.wait() + return response + + install_sdk_transport(monkeypatch, "anthropic", provider) + model = "claude-sonnet-5-5" + environment = LocalEnvironment(str(tmp_path)) + store = LegacyJournalStore() + async with _client(model) as client: + options = {"agent": _agent(thinking=False, model=model), "llm": client, "env": environment, "store": store} + session = build_session(auto_save_path=path, **options) + running = restored = None + try: + for _ in range(base_sequence): + session.save(path) + session.messages = [*session.messages, {"role": "user", "content": "Read source.txt before answering"}] + running = asyncio.create_task(session.run_loop()) + await asyncio.wait_for(held_request.wait(), 5) + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + + restored = load_session(path, **options) + saved = store.load_snapshot(path, session.agent.system_prompt) + if failed_write == "append": + assert saved["_autosave_sequence"] == base_sequence + assert restored.state.anthropic_history is None + assert len(restored.messages) == 1 + else: + assert saved["_autosave_sequence"] == sequence + assert restored.state.anthropic_history is not None + assert restored.messages == session.messages + assert sum(row["role"] == "tool" for row in restored.messages) == 1 + assert writes == expected_writes + assert session.persistence_errors == ((write_error,) if failed_write in expected_writes else ()) + + resume.set() + assert await running == "done" + finally: + resume.set() + if running is not None: + await asyncio.gather(running, return_exceptions=True) + if restored is not None: + await restored.aclose() + await session.aclose() + await environment.cleanup() + + async def test_autosave_without_native_history_uses_only_session_state(tmp_path, monkeypatch): path = tmp_path / "session.json" environment = LocalEnvironment(str(tmp_path)) From d2d57165c006324825ed94e8b40fdef8545e61ed Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 17:16:03 +0800 Subject: [PATCH 57/82] fix(llm): keep completion controls on the routed request Preserve admission callbacks that middleware already forwards. Carry inherited native controls through the owned completion when a legacy signature hides them, and match the forwarded request before claiming scoped budget and usage callbacks. Keep older completion signatures usable while preserving admission deadlines, early budget checks and usage accounting for native routing, compaction and auxiliary provider calls. --- opencollab/adapters/llm/client.py | 33 +++- .../application/_session_run_completion.py | 125 ++++++++------ opencollab/application/ports.py | 10 +- .../runtime/test_pooled_generation_timeout.py | 153 +++++++++++++++++- 4 files changed, 256 insertions(+), 65 deletions(-) diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index 4f6fed2c..0c71a2af 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -40,7 +40,11 @@ from opencollab.adapters.llm.types import LLMResponse, model_context_window from opencollab.adapters.llm.usage_ledger import record_api_usage from opencollab.application.async_timeout import await_owned_operation -from opencollab.application.ports import _NATIVE_REQUEST_ADMISSION, _NATIVE_REQUEST_REVALIDATION, _native_admission +from opencollab.application.ports import ( + _NATIVE_COMPLETION_CAPABILITY, + _NATIVE_REQUEST_ADMISSION, + _native_completion_controls, +) from opencollab.domain.token_estimation import estimate_request_tokens # Native data-residency endpoints share the OpenAI model parameter rules. @@ -79,8 +83,10 @@ class LLMClient: OpenAI-format history per request. """ - supports_request_revalidation = _NATIVE_REQUEST_REVALIDATION + supports_request_revalidation = _NATIVE_COMPLETION_CAPABILITY supports_request_admission = _NATIVE_REQUEST_ADMISSION + supports_attempt_usage = _NATIVE_COMPLETION_CAPABILITY + supports_usage_settlement = _NATIVE_COMPLETION_CAPABILITY def __init__( self, @@ -127,8 +133,6 @@ def __init__( raise ValueError("request_pool must be an absolute local directory path") self.request_pool = request_pool self.request_capacity = request_capacity - self.supports_attempt_usage = True - self.supports_usage_settlement = True self.model = model self.provider = provider self.max_retries = max(0, max_retries) @@ -319,6 +323,24 @@ async def complete( report their known usage through ``on_attempt_usage`` before retry backoff. ``on_usage`` receives the final aggregate before slot release. """ + scope = _native_completion_controls.get() + if scope is not None: + owner, submitted, admission_callback, controls = scope + # An explicitly forwarded admission callback still identifies a rewritten request. + forwarded = admission_callback is not None and on_admitted is admission_callback + if controls and (forwarded or submitted == messages): + if on_admitted is None: + on_admitted = controls.get("on_admitted") + if before_request is None: + before_request = controls.get("before_request") + if on_attempt_usage is None: + on_attempt_usage = controls.get("on_attempt_usage") + if on_usage is None: + on_usage = controls.get("on_usage") + controls.clear() + elif owner is self and on_admitted is None: + on_admitted = controls.pop("on_admitted", None) + start = time.monotonic() response = None provider_error: BaseException | None = None @@ -352,9 +374,6 @@ def prepare_attempt() -> int | None: try: await request_owner.enter_async_context(request_slot(self.request_pool, self.request_capacity)) initial_limit = request_limit() - admission = _native_admission.get() if self.request_pool is not None else None - if admission is not None and admission[0] is self: - admission[1]() if on_admitted is not None: on_admitted() if self._anthropic: diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index a2b71ef1..91882b27 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -36,14 +36,14 @@ from opencollab.application._tool_loop_execution import _apply_completed_prefix_progress from opencollab.application.async_timeout import CallerTimeoutError, abandon_on_timeout from opencollab.application.ports import ( + _NATIVE_COMPLETION_CAPABILITY, _NATIVE_REQUEST_ADMISSION, - _NATIVE_REQUEST_REVALIDATION, CompletionResponse, ContextTokenEstimatorPort, RequestOutputRequirementPort, RequestTokenEstimatorPort, ThinkingPrefixPort, - _native_admission, + _native_completion_controls, ) from opencollab.application.shaping import ShaperPipeline from opencollab.application.shaping.pipeline import context_token_estimator @@ -62,39 +62,50 @@ logger = logging.getLogger(__name__) -async def _observe_native_admission( - operation: Awaitable[CompletionResponse], owner: object, admitted: asyncio.Event, +async def _with_native_completion_controls( + operation: Awaitable[CompletionResponse], + owner: object | None, + messages: list[dict[str, Any]], + admission_callback: Callable[[], None] | None, + controls: dict[str, Callable[..., Any]], ) -> CompletionResponse: - token = _native_admission.set((owner, admitted.set)) + scope = (owner, messages, admission_callback, controls) if owner is not None and controls else None + token = _native_completion_controls.set(scope) try: return await operation finally: - _native_admission.reset(token) + controls.clear() + _native_completion_controls.reset(token) -def _completion_keyword_parameters(complete: Callable[..., Awaitable[Any]]) -> tuple[set[str] | None, bool]: +def _completion_keyword_parameters( + complete: Callable[..., Awaitable[Any]], +) -> tuple[set[str] | None, bool, set[str]]: keywords: set[str] = set() outer_accepts_kwargs = False + outer_keywords: set[str] = set() def has_fixed_keywords(target: Callable[..., Any]) -> bool: nonlocal outer_accepts_kwargs parameters = inspect.signature(target, follow_wrapped=False).parameters.values() - keywords.update( + names = { parameter.name for parameter in parameters if parameter.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY) - ) + } + keywords.update(names) accepts_kwargs = any(parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters) if target is complete: outer_accepts_kwargs = accepts_kwargs + outer_keywords.update(names) return not accepts_kwargs # Follow variadic decorators through to the first fixed signature. try: target = inspect.unwrap(complete, stop=has_fixed_keywords) if not hasattr(target, "__wrapped__") and not has_fixed_keywords(target): - return None, outer_accepts_kwargs + return None, outer_accepts_kwargs, outer_keywords except (TypeError, ValueError): pass - return keywords, outer_accepts_kwargs + return keywords, outer_accepts_kwargs, outer_keywords def _selector_names_tool_choice(selector: Any) -> bool: """Return whether a structured error selector targets ``tool_choice``.""" @@ -806,50 +817,58 @@ def _add_completion_controls( max_output_tokens: int, usage_accounting: _CompletionUsageAccounting, revalidate: Callable[[], int], - ) -> tuple[asyncio.Event | None, object | None]: - keywords, outer_accepts_kwargs = _completion_keyword_parameters(complete) + ) -> tuple[asyncio.Event | None, object | None, dict[str, Callable[..., Any]]]: + keywords, outer_accepts_kwargs, outer_keywords = _completion_keyword_parameters(complete) + owner = getattr(complete, "__self__", None) + scoped: dict[str, Callable[..., Any]] = {} def accepts(name: str) -> bool: return keywords is None or name in keywords - def accepts_control( - name: str, capability: str, native_support: object, *, follow_delegate: bool = False, + def add_control( + name: str, + capability: str, + callback: Callable[..., Any], + *, + native_support: object = _NATIVE_COMPLETION_CAPABILITY, + allow_scope: bool = True, ) -> bool: supported = getattr(self.llm, capability, False) - if not supported or accepts(name): - return bool(supported) - if not outer_accepts_kwargs or (not follow_delegate and supported is not True): + if not supported: return False - # Explicit middleware opt-ins can consume the callback before forwarding. - fallback = native_support - if follow_delegate: - # A transparent proxy can return a method owned by a native client. - fallback = inspect.getattr_static(getattr(complete, "__self__", None), capability, None) - return inspect.getattr_static(self.llm, capability, fallback) is not native_support + if name in outer_keywords or (name == "on_admitted" and accepts(name)): + kwargs[name] = callback + return True + # A native marker also appears on subclasses whose wrappers forward + # only legacy parameters. Supply their controls at native dispatch. + inherited = inspect.getattr_static(owner, capability, None) + declared = inspect.getattr_static(self.llm, capability, inherited) + if declared is native_support: + if allow_scope and inherited is native_support: + scoped[name] = callback + return True + return False + if declared is None and native_support is _NATIVE_COMPLETION_CAPABILITY: + if keywords is None or name not in keywords: + return False + # Explicit middleware opt-ins can consume controls before forwarding. + if accepts(name) or (outer_accepts_kwargs and (supported is True or name == "on_admitted")): + kwargs[name] = callback + return True + return False if max_output_tokens != DEFAULT_MAX_TOKENS_PER_STEP or accepts("max_output_tokens"): kwargs["max_output_tokens"] = max_output_tokens - if getattr(self.llm, "supports_attempt_usage", False) and accepts("on_attempt_usage"): - kwargs["on_attempt_usage"] = usage_accounting.record_attempt - if getattr(self.llm, "supports_usage_settlement", False) and accepts("on_usage"): - kwargs["on_usage"] = usage_accounting.settle - if accepts_control("before_request", "supports_request_revalidation", _NATIVE_REQUEST_REVALIDATION): - kwargs["before_request"] = revalidate - if accepts_control( - "on_admitted", "supports_request_admission", _NATIVE_REQUEST_ADMISSION, follow_delegate=True, - ): - admitted = asyncio.Event() - kwargs["on_admitted"] = admitted.set - return admitted, None - owner = getattr(complete, "__self__", None) - # A variadic wrapper can reach the native pool through a legacy signature. - if ( - self._per_call_timeout is not None and outer_accepts_kwargs - and getattr(self.llm, "supports_request_admission", False) - and inspect.getattr_static(owner, "supports_request_admission", None) is _NATIVE_REQUEST_ADMISSION - ): - return asyncio.Event(), owner - return None, None + add_control("on_attempt_usage", "supports_attempt_usage", usage_accounting.record_attempt) + add_control("on_usage", "supports_usage_settlement", usage_accounting.settle) + add_control("before_request", "supports_request_revalidation", revalidate) + admitted = asyncio.Event() + has_admission = add_control( + "on_admitted", "supports_request_admission", admitted.set, + native_support=_NATIVE_REQUEST_ADMISSION, + allow_scope=self._per_call_timeout is not None and outer_accepts_kwargs, + ) + return admitted if has_admission else None, owner, scoped def _revalidate_request( self, messages: list[dict], tools: list[dict] | None, *, thinking: bool = False, @@ -885,7 +904,7 @@ def revalidate() -> int: ) return min(limit, _summary_output_limit(messages)) if _summary_output_limit is not None else limit - admitted, native_owner = self._add_completion_controls( + admitted, native_owner, scoped = self._add_completion_controls( self.llm.complete, kwargs, max_output_tokens=max_output_tokens, usage_accounting=usage_accounting, revalidate=revalidate, ) @@ -922,8 +941,10 @@ async def complete_owned() -> CompletionResponse: self._finish_run_control_if_quiesced() operation = complete_owned() - if native_owner is not None and admitted is not None: - operation = _observe_native_admission(operation, native_owner, admitted) + if scoped or _native_completion_controls.get() is not None: + operation = _with_native_completion_controls( + operation, native_owner, messages, kwargs.get("on_admitted"), scoped, + ) owner = asyncio.create_task(operation) self._track_provider_task(owner) try: @@ -961,7 +982,7 @@ async def _invoke_llm(self, *, max_output_tokens: int, **kwargs: Any) -> Complet protected_call = self.state.wind_down_done usage_accounting = _CompletionUsageAccounting(self, protected_call=protected_call) complete = self.llm.complete - admitted, native_owner = self._add_completion_controls( + admitted, native_owner, scoped = self._add_completion_controls( complete, kwargs, max_output_tokens=max_output_tokens, usage_accounting=usage_accounting, revalidate=lambda: self._revalidate_request( kwargs["messages"], kwargs.get("tools"), thinking=kwargs.get("thinking", False), @@ -993,8 +1014,10 @@ def record_abandoned(task: asyncio.Task[Any]) -> None: self._record_late_provider_result(task, protected_call=protected_call, usage_accounted=True) operation = complete_owned() - if native_owner is not None and admitted is not None: - operation = _observe_native_admission(operation, native_owner, admitted) + if scoped or _native_completion_controls.get() is not None: + operation = _with_native_completion_controls( + operation, native_owner, kwargs["messages"], kwargs.get("on_admitted"), scoped, + ) try: response = await operation if self._per_call_timeout is None else await abandon_on_timeout( operation, diff --git a/opencollab/application/ports.py b/opencollab/application/ports.py index 52c96021..d5c02469 100644 --- a/opencollab/application/ports.py +++ b/opencollab/application/ports.py @@ -471,7 +471,7 @@ def transport_timing(self) -> dict[str, Any] | None: """ ... -class _NativeRequestRevalidation: +class _NativeCompletionCapability: def __get__(self, instance: object, owner: type) -> bool: return True @@ -482,9 +482,13 @@ def __get__(self, instance: Any, owner: type) -> bool: # Native clients support these callbacks through their completion signature. -_NATIVE_REQUEST_REVALIDATION = _NativeRequestRevalidation() +_NATIVE_COMPLETION_CAPABILITY = _NativeCompletionCapability() _NATIVE_REQUEST_ADMISSION = _NativeRequestAdmission() -_native_admission: ContextVar[tuple[object, Callable[[], None]] | None] = ContextVar("native_admission", default=None) +_native_completion_controls: ContextVar[ + tuple[object, list[dict[str, Any]], Callable[[], None] | None, dict[str, Callable[..., Any]]] | None +] = ContextVar( + "native_completion_controls", default=None, +) class LLMPort(Protocol): diff --git a/tests/runtime/test_pooled_generation_timeout.py b/tests/runtime/test_pooled_generation_timeout.py index 1275ab87..2ade403f 100644 --- a/tests/runtime/test_pooled_generation_timeout.py +++ b/tests/runtime/test_pooled_generation_timeout.py @@ -3,6 +3,7 @@ import asyncio import json import threading +from copy import deepcopy from functools import wraps import pytest @@ -63,6 +64,17 @@ def __getattr__(self, name): return getattr(self.client, name) +class _VariadicLegacyRoute(LLMClient): + async def complete(self, *args, **kwargs): + return await self.delegate(*args, **kwargs) + + +class _MetadataLegacyRoute(_VariadicLegacyRoute): + @wraps(LLMClient.complete) + async def complete(self, *args, **kwargs): + return await super().complete(*args, **kwargs) + + @pytest.fixture( params=[LLMClient, _AdmissionMiddleware, _ForwardingPooledClient], ids=["native", "decorated", "inherited"], @@ -253,7 +265,140 @@ async def complete(*_args, **_kwargs): @pytest.mark.parametrize("summary", [False, True]) -@pytest.mark.parametrize("bypass", ["returns", "hangs", "other-client", "nonpool"]) +@pytest.mark.parametrize("route", ["same-pool", "different-pool", "unpooled"]) +async def test_routed_native_completion_keeps_the_generation_deadline( + tmp_path, monkeypatch, local_environment_factory, summary, route, +): + entered = asyncio.Event() + + async def complete(*_args, **_kwargs): + entered.set() + await asyncio.Event().wait() + + monkeypatch.setattr(transport, "complete_openai", complete) + origin_pool = str(tmp_path) + (tmp_path / "recipient").mkdir() + recipient_pool = { + "same-pool": origin_pool, + "different-pool": str(tmp_path / "recipient"), + "unpooled": None, + }[route] + recipient_capacity = None if recipient_pool is None else 1 + config = dict(api_key="local", context_window=40_000) + async with ( + _DelegatingPooledClient(**config, request_pool=origin_pool, request_capacity=1) as client, + LLMClient(**config, request_pool=recipient_pool, request_capacity=recipient_capacity) as recipient, + ): + async def delegate(messages, *args, **kwargs): + return await recipient.complete(deepcopy(messages), *args, **kwargs) + + client.delegate = delegate + session = build_session( + agent=Agent(name="routed", system_prompt="system"), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.08, + ) + session.messages = history(summary) + try: + async with request_slot(recipient_pool, recipient_capacity): + owner = asyncio.create_task(session.run_loop()) + if recipient_pool is not None: + await asyncio.sleep(0.13) + assert not owner.done() and not entered.is_set() + await asyncio.wait_for(entered.wait(), 1) + with pytest.raises(GenerationTimeoutError): + await asyncio.wait_for(owner, 1) + finally: + await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("stall", [False, True]) +async def test_routed_rewrite_preserves_deadline_and_separate_preflight_usage( + tmp_path, monkeypatch, local_environment_factory, summary, stall, +): + ready = asyncio.Event() + entered = asyncio.Event() + calls = [] + + async def complete(_client, _model, messages, *_args, **_kwargs): + if messages[-1]["content"] == "Preflight": + return LLMResponse(content="ok", usage=Usage(2, 2)) + calls.append(True) + entered.set() + if stall: + await asyncio.Event().wait() + content = "Earlier context." if summary and len(calls) == 1 else "answer" + return LLMResponse(content=content, usage=Usage(7, 3)) + + monkeypatch.setattr(transport, "complete_openai", complete) + destination = tmp_path / "target" + destination.mkdir() + config = dict(api_key="local", context_window=40_000, request_capacity=1) + async with ( + _VariadicLegacyRoute(**config, request_pool=str(tmp_path)) as client, + LLMClient(**config, request_pool=str(destination)) as recipient, + ): + async def delegate(messages, *args, **kwargs): + await LLMClient.complete(client, [{"role": "user", "content": "Preflight"}]) + ready.set() + rewritten = [*messages, {"role": "system", "content": "Routed request"}] + return await recipient.complete(rewritten, *args, **kwargs) + + client.delegate = delegate + session = build_session( + agent=Agent(name="routed", system_prompt="system"), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.09, + ) + session.messages = history(summary) + try: + async with request_slot(str(destination), 1): + owner = asyncio.create_task(session.run_loop()) + await asyncio.wait_for(ready.wait(), 1) + await asyncio.sleep(0.15) + assert not owner.done() and not entered.is_set() + assert session.used_tokens == 0 + await asyncio.wait_for(entered.wait(), 1) + if stall: + with pytest.raises(GenerationTimeoutError): + await asyncio.wait_for(owner, 1) + else: + assert await asyncio.wait_for(owner, 2) == "answer" + assert len(calls) == (2 if summary else 1) + assert session.used_tokens == 10 * len(calls) + finally: + await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("client_class", [_VariadicLegacyRoute, _MetadataLegacyRoute]) +async def test_routed_legacy_completion_uses_response_usage( + tmp_path, local_environment_factory, summary, client_class, +): + calls = [] + + async def complete(messages, tools=None, temperature=0.0, *, max_output_tokens=None): + calls.append(messages) + content = "Earlier context." if summary and len(calls) == 1 else "answer" + return LLMResponse(content=content, usage=Usage(7, 3)) + + config = dict(api_key="local", context_window=40_000) + async with client_class(**config) as client: + client.delegate = complete + session = build_session( + agent=Agent(name="routed", system_prompt="system"), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=1, + ) + session.messages = history(summary) + try: + assert await session.run_loop() == "answer" + assert len(calls) == (2 if summary else 1) + assert session.used_tokens == 10 * len(calls) + finally: + await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("bypass", ["returns", "hangs", "unrelated", "nonpool"]) async def test_inherited_admission_waiting_preserves_completion_and_caller_deadline( tmp_path, monkeypatch, local_environment_factory, summary, bypass, ): @@ -269,8 +414,8 @@ async def delegate(messages, tools=None, temperature=0.0, *, max_output_tokens=N if bypass == "returns": content = "Earlier context." if summary and len(calls) == 1 else "answer" return LLMResponse(content=content, usage=Usage(7, 3)) - if bypass == "other-client": - return await other.complete(messages, tools, temperature, max_output_tokens=max_output_tokens) + if bypass == "unrelated": + return await other.complete([{"role": "user", "content": "Unrelated request"}]) await asyncio.Event().wait() monkeypatch.setattr(transport, "complete_openai", complete) @@ -297,7 +442,7 @@ async def delegate(messages, tools=None, temperature=0.0, *, max_output_tokens=N assert calls == [True] assert session.used_tokens == 0 await asyncio.gather(*session.pending_cleanup_tasks, return_exceptions=True) - assert provider_calls == ([True] if bypass == "other-client" else []) + assert provider_calls == ([True] if bypass == "unrelated" else []) finally: await session.aclose() From 8873348d170c68d1c9cc32f13193100f12cc5260 Mon Sep 17 00:00:00 2001 From: Kai Date: Thu, 19 Nov 2026 21:07:36 +0800 Subject: [PATCH 58/82] ci: allow the Python 3.12 job to finish DOM checks Two runs reached the 40-minute cap after pytest passed, before the remaining DOM checks could finish. Give only the Python 3.12 matrix job 45 minutes and retain the same checks and the other matrix limits. --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab43abe3..f04d49fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,8 @@ concurrency: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 40 + # Python 3.12 runs the DOM regressions after the Python suite. + timeout-minutes: ${{ matrix.python-version == '3.12' && 45 || 40 }} strategy: fail-fast: false matrix: From ddfdfacab76582f7d5066d63db9a1147b0fd56df Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 09:11:32 +0800 Subject: [PATCH 59/82] fix(workflow): exclude mounted recovery stores from source capture --- opencollab/adapters/_candidate_adoption.py | 53 ++++++++- opencollab/adapters/_candidate_source.py | 12 +- opencollab/adapters/candidate_workspace.py | 77 +++++++----- .../test_candidate_adoption_mounts.py | 110 ++++++++++++++++++ 4 files changed, 212 insertions(+), 40 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index 078b2812..c1855bca 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -279,6 +279,51 @@ } ''' +_RECOVERY_GIT_DIRECTORIES = r''' +private=$(git rev-parse --absolute-git-dir && printf .) || exit +private=${private%??} +common=$(git rev-parse --git-common-dir && printf .) || exit +common=${common%??} +''' + + +async def _mounted_recovery_paths(environment: Any, workspace: str) -> tuple[str, ...]: + # The volume receipts live beside the primary backup, including for linked worktrees. + command = f"cd -- {shlex.quote(workspace)} || exit\n" + _RECOVERY_PARENT + _RECOVERY_GIT_DIRECTORIES + r''' +workspace=$(pwd -P && printf .) || exit +workspace=${workspace%??} +prefix=${workspace%/}/ +primary=$(recovery_parent "$common" "$private" && printf .) || exit +primary=${primary%??} +if [ "$primary" -ef "$common" ] || [ "$primary" -ef "$private" ]; then primary=; fi +if [ "$private" -ef "$common" ]; then private=; fi +for parent in "$common" "$private" "$primary"; do + [ -n "$parent" ] || continue + for backup in "$parent"/candidate-recovery-????????; do + [ -d "$backup" ] && [ ! -L "$backup" ] && [ -O "$backup" ] || continue + volumes=$backup/volumes + [ -d "$volumes" ] && [ ! -L "$volumes" ] && [ -O "$volumes" ] || continue + for receipt in "$volumes"/volume-????????; do + [ -f "$receipt" ] && [ ! -L "$receipt" ] && [ -O "$receipt" ] || continue + store=$(cat -- "$receipt" && printf .) || continue + store=${store%.} + case "$store" in "$prefix"*) ;; *) continue;; esac + case "${store##*/}" in .candidate-recovery-????????) ;; *) continue;; esac + [ -d "$store" ] && [ ! -L "$store" ] && [ -O "$store" ] || continue + anchor=${store%/*} + top=$(cd -P -- "${anchor:-/}" && [ "$PWD" = "${anchor:-/}" ] && + GIT_DISCOVERY_ACROSS_FILESYSTEM=1 git rev-parse --show-toplevel 2>/dev/null && + printf .) || continue + [ "${top%??}" = "$workspace" ] || continue + printf '%s\0' "${store#"$prefix"}" + done + done +done +''' + result = _complete(await environment.exec_cmd(command, timeout=30), "mounted candidate recovery locations") + return tuple(dict.fromkeys(path for path in result.split("\0") if path)) + + _RECOVER_PATHS = _COMPARE_ENTRIES + _CHECK_PARENTS + _RESTORE_SAVED + _RECOVERY_STORES + r''' check_candidate_descendants() { @@ -531,11 +576,9 @@ async def _adoption_transaction( directory_script = f"cd -- {shlex.quote(workspace)} || exit\n" + _RECOVERY_PARENT if repository: # Git removes private metadata with the linked worktree. - directory_script += r''' -private=$(git rev-parse --absolute-git-dir && printf .) || exit -printf '%s\0' "${private%??}" || exit -common=$(git rev-parse --git-common-dir && printf .) || exit -recovery_parent "${common%??}" "${private%??}" + directory_script += _RECOVERY_GIT_DIRECTORIES + r''' +printf '%s\0' "$private" || exit +recovery_parent "$common" "$private" ''' else: directory_script += 'recovery_parent "${TMPDIR:-/tmp}"' diff --git a/opencollab/adapters/_candidate_source.py b/opencollab/adapters/_candidate_source.py index 302209e3..6ecf7f55 100644 --- a/opencollab/adapters/_candidate_source.py +++ b/opencollab/adapters/_candidate_source.py @@ -25,7 +25,6 @@ class _SourceSnapshot: index_revision: str patch: str excluded: tuple[str, ...] - untracked_excluded: tuple[str, ...] physical_tree: str | None modules: tuple[tuple[str, str], ...] initialized: tuple[str, ...] @@ -61,12 +60,13 @@ async def changed(self, patch: str | None) -> bool: ): return True environment = self.owner._environment + untracked_excluded = (*await self.owner._source_untracked_paths(self.root), *self.excluded) initialized = await _initialized_submodule_paths(environment, self.root) if self._included(initialized) != self._included(self.initialized): return True current_modules = dict(await _physical_submodule_trees( environment, self.root, include_paths=tuple(path for path, _tree in self.modules), - exclude_paths=self.untracked_excluded, exclude_tree_paths=self.excluded, + exclude_paths=untracked_excluded, exclude_tree_paths=self.excluded, module_paths=initialized, )) original_modules = dict(self.modules) @@ -85,7 +85,7 @@ async def changed(self, patch: str | None) -> bool: return True if self.physical_tree is None and patch == self.patch and not await _has_worktree_conversions( environment, self.root, exclude_paths=self.excluded, - exclude_untracked_paths=self.untracked_excluded, + exclude_untracked_paths=untracked_excluded, ): return False # A plain source's saved Git patch contains its physical bytes. Build @@ -93,7 +93,7 @@ async def changed(self, patch: str | None) -> bool: original = self.physical_tree or await self._original_tree() current = await _physical_tree( environment, self.root, self.revision, - exclude_paths=self.untracked_excluded, exclude_tree_paths=self.excluded, + exclude_paths=untracked_excluded, exclude_tree_paths=self.excluded, ) return current != original @@ -105,7 +105,7 @@ async def _source_snapshot(owner: Any, patch: str | None, excluded: tuple[str, . environment = owner._environment revision = await owner.source_revision() index_revision = await owner.source_index_revision() - untracked_excluded = (*owner._owned_untracked_paths(root), *excluded) + untracked_excluded = (*await owner._source_untracked_paths(root), *excluded) initialized = await _initialized_submodule_paths(environment, root) modules = await _physical_submodule_trees( environment, root, exclude_paths=untracked_excluded, exclude_tree_paths=excluded, @@ -119,6 +119,6 @@ async def _source_snapshot(owner: Any, patch: str | None, excluded: tuple[str, . environment, root, revision, exclude_paths=untracked_excluded, exclude_tree_paths=excluded, ) return _SourceSnapshot( - owner, root, revision, index_revision, patch, excluded, untracked_excluded, + owner, root, revision, index_revision, patch, excluded, physical_tree, modules, initialized, ) diff --git a/opencollab/adapters/candidate_workspace.py b/opencollab/adapters/candidate_workspace.py index ed8e0e3c..c5bc3f20 100644 --- a/opencollab/adapters/candidate_workspace.py +++ b/opencollab/adapters/candidate_workspace.py @@ -13,6 +13,7 @@ from dataclasses import dataclass from typing import Any +from opencollab.adapters._candidate_adoption import _mounted_recovery_paths from opencollab.adapters._candidate_increment import _apply_increment as _apply_increment from opencollab.adapters._candidate_increment import _safe_path from opencollab.adapters._candidate_physical import ( @@ -85,19 +86,20 @@ async def _raw_diff_at( ), "candidate tracked diff", environment, workspace, base_revision, index_file, ) + untracked_paths = (*excluded, *(_safe_path(path) + "/" for path in untracked_exclude_paths)) + untracked_pathspec = " -- . " + shlex.join( + f":(exclude,literal){path}" for path in untracked_paths + ) if untracked_paths else "" untracked = _complete( await environment.exec_cmd( f"{git} ls-files --others --exclude-standard -z" - + pathspec, + + untracked_pathspec, timeout=CANDIDATE_WORKSPACE_GIT_TIMEOUT_SECONDS, ), "candidate untracked listing", ) parts = [tracked] if tracked.strip() else [] - untracked_excluded = tuple(_safe_path(path) + "/" for path in untracked_exclude_paths) for path in (item for item in untracked.split("\0") if item): - if path.startswith(untracked_excluded): - continue patch = await _complete_patch( await environment.exec_cmd( "git -c core.quotePath=true -C " @@ -206,17 +208,21 @@ async def source_contents_changed(self) -> bool: async def physical_source_changed(self, exclude_paths: Sequence[str] = (), *, force: bool = False) -> bool: excluded = _repository_exclusions(self.workspace_prefix, exclude_paths) + untracked_exclusions = ( + *self.source_untracked_exclusions, + *await _mounted_recovery_paths(self.base_environment, self.source_workspace), + ) if await self._submodule_contents_changed( self.source_workspace, self.physical_source_submodules, initialized=self.initialized_source_submodules, - excluded=excluded, untracked_exclusions=self.source_untracked_exclusions, + excluded=excluded, untracked_exclusions=untracked_exclusions, ): return True original_tree = self.physical_base_revision if original_tree is None: if not force and not await _has_worktree_conversions( self.base_environment, self.source_workspace, strict=self.strict_capture, - exclude_paths=excluded, exclude_untracked_paths=self.source_untracked_exclusions, + exclude_paths=excluded, exclude_untracked_paths=untracked_exclusions, ): return False # Without initial conversions, the ordinary source tree already @@ -228,7 +234,7 @@ async def physical_source_changed(self, exclude_paths: Sequence[str] = (), *, fo ), "candidate original source tree").strip() current = await _physical_tree( self.base_environment, self.source_workspace, self.base_revision, - exclude_paths=self.source_untracked_exclusions, strict=self.strict_capture, + exclude_paths=untracked_exclusions, strict=self.strict_capture, ) if not exclude_paths: return current != original_tree @@ -422,6 +428,12 @@ def _owned_untracked_paths(self, repository_root: str) -> tuple[str, ...]: return () return (os.path.relpath(self._owned_save_dir, repository_root).replace(os.sep, "/"),) + async def _source_untracked_paths(self, repository_root: str) -> tuple[str, ...]: + return ( + *self._owned_untracked_paths(repository_root), + *await _mounted_recovery_paths(self._environment, repository_root), + ) + async def _repository_root(self) -> str: return _complete( await self._environment.exec_cmd( @@ -443,6 +455,7 @@ async def _workspace_prefix(self) -> str: async def _source_patch( self, repository_root: str, base_revision: str, exclude_paths: Sequence[str] = (), + *, untracked_exclusions: Sequence[str] = (), ) -> str: # Keep staged additions in a temporary index without its skip hints, # then stage the current tracked bytes without changing the source index. @@ -472,13 +485,13 @@ async def _source_patch( await _assert_patchable_submodules( self._environment, repository_root, base_revision, index_file=source_index, reject_ignored_untracked=False, - exclude_untracked_paths=self._owned_untracked_paths(repository_root), + exclude_untracked_paths=untracked_exclusions, exclude_paths=exclude_paths, ) return await _raw_diff_at( self._environment, repository_root, exclude_paths, base_revision=base_revision, index_file=source_index, - untracked_exclude_paths=self._owned_untracked_paths(repository_root), + untracked_exclude_paths=untracked_exclusions, ) finally: async def remove_index() -> None: @@ -535,6 +548,14 @@ async def _acquire( ), "candidate base revision", ).strip() + source_owned = ( + *self._owned_untracked_paths(repository_root), + *(_safe_path(path) for path in strict_untracked_exclusions), + ) + source_untracked = ( + *source_owned, + *await _mounted_recovery_paths(self._environment, repository_root), + ) if strict_capture: source_patch = await _complete_patch( await self._environment.exec_cmd( @@ -542,10 +563,7 @@ async def _acquire( + guarded_staged_diff_command( base_revision=base_revision, reject_ignored_untracked=False, - exclude_untracked_paths=( - *self._owned_untracked_paths(repository_root), - *(_safe_path(path) for path in strict_untracked_exclusions), - ), + exclude_untracked_paths=source_untracked, ), timeout=CANDIDATE_WORKSPACE_GIT_TIMEOUT_SECONDS, ), @@ -555,28 +573,28 @@ async def _acquire( async with AsyncExitStack() as capture_resources: converted = await _has_worktree_conversions( self._environment, repository_root, strict=strict_capture, + exclude_untracked_paths=source_untracked, ) source_capture = physical_context = None if converted: if not strict_capture: await _assert_patchable_submodules( self._environment, repository_root, base_revision, reject_ignored_untracked=False, - exclude_untracked_paths=self._owned_untracked_paths(repository_root), + exclude_untracked_paths=source_untracked, ) physical_context = await capture_resources.enter_async_context(_physical_git( self._environment, repository_root, strict=strict_capture, )) source_capture = await _capture_physical( self._environment, repository_root, base_revision, - exclude_paths=( - *self._owned_untracked_paths(repository_root), - *strict_untracked_exclusions, - ), + exclude_paths=source_untracked, strict=strict_capture, metadata=physical_context, ) source_patch = source_capture.patch elif not strict_capture: - source_patch = await self._source_patch(repository_root, base_revision) + source_patch = await self._source_patch( + repository_root, base_revision, untracked_exclusions=source_untracked, + ) initialized_source = await _initialized_submodule_paths( self._environment, repository_root, strict=strict_capture, source_prefix=workspace_prefix.rstrip("/"), @@ -584,7 +602,7 @@ async def _acquire( source_modules = await _physical_submodule_trees( self._environment, repository_root, strict=strict_capture, source_prefix=workspace_prefix.rstrip("/"), - exclude_paths=(*self._owned_untracked_paths(repository_root), *strict_untracked_exclusions), + exclude_paths=source_untracked, module_paths=initialized_source, ) lease = _CandidateLease( @@ -596,10 +614,7 @@ async def _acquire( workspace_prefix=workspace_prefix, physical_base_revision=source_capture.tree if source_capture is not None else None, strict_capture=strict_capture, - source_untracked_exclusions=( - *self._owned_untracked_paths(repository_root), - *strict_untracked_exclusions, - ), + source_untracked_exclusions=source_owned, physical_source_submodules=source_modules, initialized_source_submodules=initialized_source, ) @@ -774,24 +789,28 @@ async def source_diff(self, exclude_paths: Sequence[str] = ()) -> str: repository_root = await self._repository_root() excluded = _repository_exclusions(await self._workspace_prefix(), exclude_paths) if exclude_paths else () base_revision = await self.source_revision() - return await self._source_patch(repository_root, base_revision, excluded) + return await self._source_patch( + repository_root, base_revision, excluded, + untracked_exclusions=await self._source_untracked_paths(repository_root), + ) async def physical_source_revision(self, exclude_paths: Sequence[str] = ()) -> str | None: """Read conversion-aware source trees with repository-relative exclusions.""" root = await self._repository_root() excluded = tuple(_safe_path(path) for path in exclude_paths) + owned = await self._source_untracked_paths(root) modules = await _physical_submodule_trees( - self._environment, root, exclude_paths=(*self._owned_untracked_paths(root), *excluded), + self._environment, root, exclude_paths=(*owned, *excluded), exclude_tree_paths=excluded, ) revision = None if await _has_worktree_conversions( self._environment, root, exclude_paths=excluded, - exclude_untracked_paths=self._owned_untracked_paths(root), + exclude_untracked_paths=owned, ): revision = await _physical_tree( self._environment, root, await self.source_revision(), - exclude_paths=(*self._owned_untracked_paths(root), *excluded), + exclude_paths=(*owned, *excluded), exclude_tree_paths=excluded, ) return json.dumps((revision, modules)) if modules else revision @@ -805,7 +824,7 @@ async def _source_changed(self, exclude_paths: Sequence[str] = ()) -> bool: _repository_exclusions(await self._workspace_prefix(), exclude_paths) if exclude_paths else () ) - owned = self._owned_untracked_paths(repository_root) + owned = await self._source_untracked_paths(repository_root) def relative_paths(paths: Sequence[str], prefix: str) -> tuple[str, ...]: if not prefix: diff --git a/tests/workflows/test_candidate_adoption_mounts.py b/tests/workflows/test_candidate_adoption_mounts.py index 8c3c7fe8..c8487fc2 100644 --- a/tests/workflows/test_candidate_adoption_mounts.py +++ b/tests/workflows/test_candidate_adoption_mounts.py @@ -8,7 +8,9 @@ import pytest from opencollab import OpenCollab +from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace from opencollab.adapters.env import LocalEnvironment +from opencollab.application.workflow_candidates import CandidateWorkspaceTrackingError from tests.workflows.test_candidate_adoption_recovery import _partial_publication from tests.workflows.test_workflow_candidate_workspace import _git @@ -150,3 +152,111 @@ async def flow(ctx, _): if published_fd is not None: os.close(published_fd) await environment.cleanup() + + +@pytest.mark.parametrize(("source_edit", "linked"), [ + (None, False), + (None, True), + ("tracked", False), + ("untracked", False), +]) +async def test_followup_candidate_ignores_only_registered_mount_recovery(tmp_path, monkeypatch, source_edit, linked): + source, candidate = _partial_publication(tmp_path) + (source / "app.py").write_text("value = 1\n") + mounted_file = "mounted/record.txt" + (source / "mounted").mkdir() + (source / mounted_file).write_text("original\n") + (source / ".gitignore").write_text("settings/\n") + if linked: + (source / ".gitattributes").write_text("*.txt text eol=lf\n") + _git(source, "add", ".") + _git(source, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare mounted source") + repo = source + if linked: + repo = tmp_path / "linked" + _git(source, "worktree", "add", "--detach", str(repo)) + (repo / "settings").mkdir() + (repo / "settings/notes.txt").write_text("existing settings\n") + + mount = repo / "mounted" + target = repo / mounted_file + candidate = replace(candidate, diff=candidate.diff.replace("record.txt", mounted_file)) + user_data = mount / ".candidate-recovery-userdata" + user_data.mkdir() + user_note = user_data / "note.txt" + user_note.write_text("user data\n") + index = Path(_git(repo, "rev-parse", "--absolute-git-dir").removesuffix("\n")) / "index" + index_before = index.read_bytes() + environment = LocalEnvironment(str(repo)) + workspace = EnvCandidateWorkspace(environment) + execute = environment.exec_cmd + + async def with_mount(command, timeout=120, **kwargs): + if "filesystem()" in command: + command = _mount_commands(repo, mount, bind=False, hard_links=True) + command + return await execute(command, timeout, **kwargs) + + monkeypatch.setattr(environment, "exec_cmd", with_mount) + store_path = None + retained = None + with target.open("ab", buffering=0) as old_writer: + async def edit(child, _): + worktree = Path(child.workspace_root) + assert not (worktree / store_path).exists() + assert (worktree / user_note.relative_to(repo)).read_text() == "user data\n" + (worktree / "app.py").write_text("value = 2\n") + old_writer.write(b"late original write\n") + if source_edit == "tracked": + (repo / "record.txt").write_text("tracked source edit\n") + elif source_edit == "untracked": + user_note.write_text("untracked source edit\n") + + async def flow(ctx, _): + nonlocal store_path, retained + with pytest.raises(RuntimeError, match="candidate adoption failed"): + await ctx.adopt_candidate(candidate) + _git(repo, "restore", "--source=HEAD", "--worktree", mounted_file) + recovery, = (source / ".git").glob("candidate-recovery-*") + location, = (recovery / "volumes").iterdir() + store = Path(location.read_text()) + store_path = store.relative_to(repo) + retained, = (store / "original-inodes").glob("[0-9]") + assert retained.stat().st_ino == os.fstat(old_writer.fileno()).st_ino + + before = await workspace.source_diff() + assert str(store_path) not in before + assert "+user data\n" in before + if linked: + lease = await workspace._acquire("team follow-up", strict_capture=True) + try: + tree = Path(lease.candidate_workspace) + assert not (tree / store_path).exists() + assert (tree / user_note.relative_to(repo)).read_text() == "user data\n" + assert not await lease.source_contents_changed() + finally: + await lease.cleanup() + if source_edit is None: + followup = await ctx.candidate_workflow(edit, {}, label="unrelated candidate") + assert "+value = 2" in followup.diff + await ctx.adopt_candidate(followup) + else: + try: + with pytest.raises(CandidateWorkspaceTrackingError, match="source worktree changed"): + await ctx.candidate_workflow(edit, {}, label="unrelated candidate") + finally: + registered = _git(repo, "worktree", "list", "--porcelain").splitlines() + for entry in registered: + if entry.startswith("worktree ") and Path(entry[9:]) != repo: + _git(repo, "worktree", "remove", "--force", entry[9:]) + assert await workspace.source_diff() != before + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow(flow, trace=False) + assert result.ok, result.reason + assert target.read_text() == "original\n" + assert retained.read_bytes() == b"original\nlate original write\n" + assert (repo / "app.py").read_text() == ("value = 1\n" if source_edit else "value = 2\n") + assert index.read_bytes() == index_before + finally: + await environment.cleanup() From c327f58aa3bcf1fc04c786bf246d81d81ad84322 Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 09:14:10 +0800 Subject: [PATCH 60/82] fix(session): preserve autosave ownership and legacy transcripts Reject conflicting native checkpoints at compacted cursors and preserve the peer snapshot across fallback and terminal saves. Compare string subclasses by their JSON text and retain missing creation timestamps in restored legacy messages. --- opencollab/adapters/storage.py | 30 +-- opencollab/application/ports.py | 6 + opencollab/application/session.py | 68 ++++--- opencollab/domain/session.py | 45 +++-- tests/agents/test_session_construction.py | 32 ++-- tests/interfaces/test_sdk_runtime.py | 22 +++ .../test_anthropic_preserved_history.py | 175 ++++++++++++++++++ 7 files changed, 310 insertions(+), 68 deletions(-) diff --git a/opencollab/adapters/storage.py b/opencollab/adapters/storage.py index 45da6877..683c8225 100644 --- a/opencollab/adapters/storage.py +++ b/opencollab/adapters/storage.py @@ -20,7 +20,7 @@ write_regular_bytes_atomic, write_regular_file_atomic, ) -from opencollab.application.ports import SnapshotCapacityError +from opencollab.application.ports import SnapshotCapacityError, SnapshotSequenceConflict from opencollab.domain.session import same_snapshot_value MAX_SESSION_SNAPSHOT_BYTES = 64 * 1024 * 1024 @@ -466,6 +466,7 @@ def append_snapshot_replacement( *, meta: dict[str, Any], sequence: int, + reject_conflicts: bool = False, ) -> bool: state = meta.get("session_state") seen_result_hashes = state.get("seen_result_hashes", []) if isinstance(state, dict) else [] @@ -483,7 +484,15 @@ def append_snapshot_replacement( except FileNotFoundError: pass else: - if sequence <= self._snapshot_sequence(persisted): + persisted_sequence = self._snapshot_sequence(persisted) + if sequence <= persisted_sequence: + if reject_conflicts: + candidate = {**meta, _AUTOSAVE_SEQUENCE_KEY: sequence, "messages": messages} + if not self._checkpoint_states_equal(persisted, candidate): + raise SnapshotSequenceConflict( + f"refusing replacement sequence {sequence}; " + f"it conflicts with persisted state at sequence {persisted_sequence}" + ) return False stored_messages = persisted["messages"] replace_from = min(len(stored_messages), len(messages)) @@ -570,7 +579,7 @@ def checkpoint_snapshot( with _journal_operation_lock(path): persisted_sequence = self._newest_persisted_sequence(path) if persisted_sequence > sequence: - raise ValueError( + raise SnapshotSequenceConflict( "refusing stale checkpoint sequence " f"{sequence}; persisted sequence {persisted_sequence} is newer" ) @@ -584,21 +593,18 @@ def checkpoint_snapshot( # Multiple records at one cursor are ambiguous: replay skips # the second and later records, so accepting this checkpoint # would clear data that cannot be proven equivalent. - raise ValueError( + raise SnapshotSequenceConflict( "refusing checkpoint sequence " f"{sequence}; persisted journal contains duplicate sequence records" ) - if same_sequence_records: - # A same-sequence journal record may belong to another writer. - # Only compact it when the proposed checkpoint is exactly the - # state obtained by replaying the durable base and journal; - # otherwise retaining the journal is safer than silently - # discarding a peer's update. + if persisted_sequence == sequence > 0: + # A peer can own this sequence in either the base or the journal. persisted = self._load_snapshot_unlocked(path, "") if not self._checkpoint_states_equal(persisted, obj): - raise ValueError( + source = "journal" if same_sequence_records else "snapshot" + raise SnapshotSequenceConflict( "refusing checkpoint sequence " - f"{sequence}; it conflicts with persisted journal state" + f"{sequence}; it conflicts with persisted {source} state" ) try: self._atomic_json_write(path, obj) diff --git a/opencollab/application/ports.py b/opencollab/application/ports.py index d5c02469..05af0d9b 100644 --- a/opencollab/application/ports.py +++ b/opencollab/application/ports.py @@ -629,6 +629,11 @@ def checkpoint_snapshot( class SnapshotCapacityError(ValueError): """An encoded snapshot exceeds the store's write limit.""" + +class SnapshotSequenceConflict(ValueError): + """The requested cursor conflicts with a durable snapshot or delta.""" + + @runtime_checkable class SnapshotReplacementStorePort(Protocol): def append_snapshot_replacement( @@ -638,6 +643,7 @@ def append_snapshot_replacement( *, meta: dict[str, Any], sequence: int, + reject_conflicts: bool = False, ) -> bool: """Return whether the replacement was appended at a new durable cursor.""" ... diff --git a/opencollab/application/session.py b/opencollab/application/session.py index 5326eaa0..7556a668 100644 --- a/opencollab/application/session.py +++ b/opencollab/application/session.py @@ -22,6 +22,7 @@ SessionStorePort, SnapshotCapacityError, SnapshotReplacementStorePort, + SnapshotSequenceConflict, SnapshotStorePort, TracePort, ) @@ -150,6 +151,7 @@ def __init__( self._terminal_checkpoint_error: Exception | None = None self._auto_save_sequence = 0 self._auto_save_reserved_sequence = 0 + self._auto_save_conflict: str | None = None self._auto_save_message_count = 0 self._auto_save_messages: list[dict] | None = None self._auto_save_rewrite_from: int | None = 0 @@ -439,6 +441,10 @@ async def _checkpoint_terminal_snapshot(self) -> None: # The pending rows and their elapsed time are now complete. Persist # this suspension without emitting the step_end reserved for resume. return + with self._auto_save_lock: + if self._auto_save_conflict is not None: + # The autosave worker already reported the ownership conflict. + return # Journal-backed autosave normally appends a delta. A terminal turn # must also publish a self-contained base snapshot: callers and crash # recovery may read the base file directly, without replaying its @@ -535,7 +541,7 @@ def restore(self, path: str) -> None: messages=[], aid=_snapshot_int(snapshot.get("aid"), default=self.state.aid), ) - restored.replace_messages(messages) + restored.replace_messages(messages, restoring=True) pending_messages = snapshot.get("pending_messages", []) restored.pending_user_messages = ( [ @@ -751,6 +757,7 @@ def _restore_auto_save_tracking(self, snapshot: dict[str, Any]) -> None: sequence = _snapshot_nonnegative_int(snapshot.get("_autosave_sequence")) self._auto_save_sequence = sequence self._auto_save_reserved_sequence = sequence + self._auto_save_conflict = None stored_messages = snapshot.get("messages") self._auto_save_message_count = min( len(stored_messages) if isinstance(stored_messages, list) else 0, @@ -959,6 +966,7 @@ def save(self, path: str) -> None: self._auto_save_sequence = sequence self._auto_save_message_count = len(messages) self._auto_save_messages = tracked_messages + self._auto_save_conflict = None if self._auto_save_rewrite_revision == rewrite_revision: self._auto_save_rewrite_from = None self._auto_save_seen_result_hashes = set(meta["session_state"]["seen_result_hashes"]) @@ -1061,6 +1069,8 @@ def _reserve_auto_save_sequence(self) -> int: return self._auto_save_reserved_sequence def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: + if self._auto_save_conflict is not None: + raise SnapshotSequenceConflict(self._auto_save_conflict) preceding_write_pending = self._auto_save_reserved_sequence > self._auto_save_sequence sequence = self._reserve_auto_save_sequence() message_count = len(self.state.messages) @@ -1120,6 +1130,29 @@ def advance_cursor() -> None: self._auto_save_rewrite_from = None self._auto_save_seen_result_hashes = current_seen_hashes + def write_checkpoint(full_messages: list[dict]) -> bool: + try: + self.store.checkpoint_snapshot(path, full_messages, meta=meta, sequence=sequence) + except (OSError, SnapshotCapacityError) as checkpoint_error: + if replacement_store is None: + raise + try: + saved = replacement_store.append_snapshot_replacement( + path, + full_messages, + meta=meta, + sequence=sequence, + reject_conflicts=True, + ) + except SnapshotSequenceConflict: + raise + except Exception as journal_error: + raise checkpoint_error from journal_error + if not saved: + raise + return False + return True + def persist() -> None: if replacement_store is None: self.store.append_snapshot_delta( @@ -1142,33 +1175,18 @@ def persist() -> None: if sequence < self._auto_save_sequence: return try: - self.store.checkpoint_snapshot( - path, - checkpoint_messages, - meta=meta, - sequence=sequence, - ) - except (OSError, SnapshotCapacityError) as checkpoint_error: - if replacement_store is None: - raise - try: - saved = replacement_store.append_snapshot_replacement( - path, - checkpoint_messages, - meta=meta, - sequence=sequence, - ) - except Exception as journal_error: - raise checkpoint_error from journal_error - if not saved: - raise - with self._auto_save_lock: - advance_cursor() - return + compacted = write_checkpoint(checkpoint_messages) + except SnapshotSequenceConflict as conflict: + if replacement_store is not None: + # A later cursor from this stale session still refers to the same old base. + with self._auto_save_lock: + self._auto_save_conflict = str(conflict) + raise with self._auto_save_lock: if replacement_store is not None: advance_cursor() - self._next_auto_save_checkpoint = 1 << sequence.bit_length() + if compacted: + self._next_auto_save_checkpoint = 1 << sequence.bit_length() def persist_incrementally() -> None: with self._auto_save_lock: diff --git a/opencollab/domain/session.py b/opencollab/domain/session.py index fda428ad..cc6e54ba 100644 --- a/opencollab/domain/session.py +++ b/opencollab/domain/session.py @@ -15,9 +15,11 @@ def same_snapshot_value(left: Any, right: Any) -> bool: - """Compare snapshot values without equating JSON booleans and numbers.""" if left is right: return True + if isinstance(left, str) and isinstance(right, str): + # The JSON encoder writes string subclasses, including enums, as text. + return str.__eq__(left, right) if type(left) is not type(right): return False if isinstance(left, dict): @@ -35,6 +37,10 @@ def _now_iso() -> str: return datetime.now(timezone.utc).isoformat() +class _MessageTimestamp(Enum): + UNKNOWN = 0 + + class SessionPhase(Enum): IDLE = "idle" PRECHECK = "precheck" @@ -198,7 +204,7 @@ class SessionState: # Per-message creation timestamps (UTC ISO-8601), index-aligned with # ``messages``. Kept as a sidecar so ``messages`` stays a clean, # API-shaped list; merged into each message only when persisting. - message_timestamps: list[str] = field(default_factory=list) + message_timestamps: list[str | _MessageTimestamp] = field(default_factory=list) # User messages queued by the scheduler but not yet appended to # ``messages`` because the session is mid-turn or awaiting tool results. # These are persisted for observability/recovery without changing the @@ -240,7 +246,7 @@ def enriched_messages(self, start: int = 0) -> list[dict[str, Any]]: raise ValueError("message start must be a non-negative integer") self._align_timestamps() return [ - {**msg, "timestamp": ts} + dict(msg) if ts is _MessageTimestamp.UNKNOWN else {**msg, "timestamp": ts} for msg, ts in zip( self.messages[start:], self.message_timestamps[start:], @@ -306,24 +312,23 @@ def append_message(self, message: dict[str, Any]) -> None: self.messages.append(message) self.message_timestamps.append(_now_iso()) - def replace_messages(self, messages: list[dict[str, Any]]) -> None: - """Swap the conversation, re-deriving aligned timestamps. - - Precedence per message: an embedded ``timestamp`` (e.g. a resumed - transcript) wins; else the prior timestamp of the same dict object - (preserved across a slice-and-rebuild); else now. - """ + def replace_messages(self, messages: list[dict[str, Any]], *, restoring: bool = False) -> None: prior = {id(m): ts for m, ts in zip(self.messages, self.message_timestamps)} - rebuilt = [ - ( - ({k: v for k, v in m.items() if k != "timestamp"}, m["timestamp"]) - if isinstance(m, dict) and "timestamp" in m - else (m, prior.get(id(m)) or _now_iso()) - ) - for m in messages - ] - self.messages = [m for m, _ in rebuilt] - self.message_timestamps = [ts for _, ts in rebuilt] + rebuilt_messages: list[dict[str, Any]] = [] + timestamps: list[str | _MessageTimestamp] = [] + for message in messages: + if isinstance(message, dict) and "timestamp" in message: + timestamp = message["timestamp"] + message = {key: value for key, value in message.items() if key != "timestamp"} + elif id(message) in prior: + timestamp = prior[id(message)] + else: + # A legacy transcript leaves the message creation time unknown. + timestamp = _MessageTimestamp.UNKNOWN if restoring else _now_iso() + rebuilt_messages.append(message) + timestamps.append(timestamp) + self.messages = rebuilt_messages + self.message_timestamps = timestamps self.anthropic_history = None def add_used_tokens(self, tokens: int) -> None: diff --git a/tests/agents/test_session_construction.py b/tests/agents/test_session_construction.py index 87732561..1d781225 100644 --- a/tests/agents/test_session_construction.py +++ b/tests/agents/test_session_construction.py @@ -295,7 +295,13 @@ def test_checkpoint_rejects_sequence_older_than_persisted_journal(tmp_path): assert restored["_autosave_sequence"] == 2 assert restored["messages"][-1]["content"] == "newer-delta" -def test_checkpoint_rejects_conflicting_same_sequence_journal(tmp_path): +@pytest.mark.parametrize("publisher", ["journal", "snapshot"]) +@pytest.mark.parametrize( + ("published_value", "conflicting_value"), + [("published", "conflict"), (True, 1), (1, 1.0), (0.0, -0.0)], + ids=["content", "bool-number", "int-float", "signed-zero"], +) +def test_checkpoint_rejects_conflicting_same_sequence(tmp_path, publisher, published_value, conflicting_value): path = tmp_path / "same-sequence-checkpoint.json" store = SessionStore() base = [ @@ -304,20 +310,24 @@ def test_checkpoint_rejects_conflicting_same_sequence_journal(tmp_path): ] meta = {"snapshot_version": 1, "session_state": {}} store.checkpoint_snapshot(str(path), base, meta=meta, sequence=1) - store.append_snapshot_delta( - str(path), - sequence=2, - replace_from=2, - messages=[{"role": "assistant", "content": "newer-delta"}], - meta=meta, - ) + published = {"role": "assistant", "content": "newer", "metadata": {"value": published_value}} + conflict = {**published, "metadata": {"value": conflicting_value}} + if publisher == "journal": + store.append_snapshot_delta(str(path), sequence=2, replace_from=2, messages=[published], meta=meta) + else: + store.checkpoint_snapshot(str(path), [*base, published], meta=meta, sequence=2) - with pytest.raises(ValueError, match="conflicts with persisted journal state"): - store.checkpoint_snapshot(str(path), base, meta=meta, sequence=2) + with pytest.raises(ValueError, match=f"conflicts with persisted {publisher} state"): + store.checkpoint_snapshot(str(path), [*base, conflict], meta=meta, sequence=2) restored = store.load_snapshot(str(path), "system") assert restored["_autosave_sequence"] == 2 - assert restored["messages"][-1]["content"] == "newer-delta" + assert json.dumps(restored["messages"][-1]) == json.dumps(published) + + store.checkpoint_snapshot(str(path), [*base, published], meta=meta, sequence=2) + assert path.with_suffix(".json.journal").read_bytes() == b"" + store.checkpoint_snapshot(str(path), [*base, conflict], meta=meta, sequence=3) + assert store.load_snapshot(str(path), "system")["_autosave_sequence"] == 3 def test_checkpoint_rejects_ambiguous_duplicate_journal_sequence(tmp_path): path = str(tmp_path / "duplicate-sequence-checkpoint.json") diff --git a/tests/interfaces/test_sdk_runtime.py b/tests/interfaces/test_sdk_runtime.py index b036af37..2b81bbba 100644 --- a/tests/interfaces/test_sdk_runtime.py +++ b/tests/interfaces/test_sdk_runtime.py @@ -8,6 +8,7 @@ import re import shlex import sys +from enum import Enum from pathlib import Path import pytest @@ -100,6 +101,27 @@ async def test_agent_uses_real_hardened_runtime_and_persists_evidence( transcript = json.loads((artifacts / "agent.json").read_text()) assert transcript["messages"][-1]["content"] == "finished" + +async def test_agent2_persists_string_enum_prompt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + class Prompt(str, Enum): + FINISH = "finish once" + + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + artifacts = tmp_path / "agent2-run" + result = await OpenCollab(tmp_path, model="model", provider="openai").agent2( + Prompt.FINISH, tools=(), llm=ReplyLLM(), artifacts=artifacts, trace=False, + ) + + assert result.status == "completed" + assert result.output == "finished" + path = artifacts / "agent.json" + snapshot = OpenCollab.read_session_snapshot(path) + assert any(row["content"] == Prompt.FINISH.value for row in snapshot["messages"] if row["role"] == "user") + assert snapshot["messages"][-1]["content"] == "finished" + assert path.with_suffix(".json.journal").read_bytes() == b"" + + async def test_client_resolves_config_once_and_delegates_plain_arguments( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index 32a64d65..d23907df 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -6,6 +6,7 @@ import copy import json import shutil +import threading import pytest @@ -864,6 +865,180 @@ def compact_journal(target, payload, **options): await environment.cleanup() +@pytest.mark.parametrize("peer_publishes_during_fallback", [False, True], ids=["checkpoint", "fallback"]) +async def test_first_native_autosave_preserves_a_compacted_peer( + monkeypatch, tmp_path, peer_publishes_during_fallback, +): + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + (tmp_path / "source.txt").write_text("file evidence\n", encoding="utf-8") + path = tmp_path / "peer-checkpoint.json" + first_request, reply_to_first = asyncio.Event(), asyncio.Event() + second_request, reply_to_second = asyncio.Event(), asyncio.Event() + fallback_started, release_fallback = threading.Event(), threading.Event() + peer = _BoundThinkingProvider(reads=1, thinking=False) + + async def provider(request): + response = peer(request) + if len(peer.requests) == 1: + first_request.set() + await reply_to_first.wait() + elif len(peer.requests) == 2: + second_request.set() + await reply_to_second.wait() + return response + + install_sdk_transport(monkeypatch, "anthropic", provider) + model = "claude-sonnet-5-5" + environment = LocalEnvironment(str(tmp_path)) + async with _client(model) as client: + options = {"agent": _agent(thinking=False, model=model), "llm": client, "env": environment, + "auto_save_path": str(path)} + writer = build_session(**options) + stale = running = None + try: + await writer.add_user_message("Read source.txt and then answer") + await asyncio.gather(*writer.pending_cleanup_tasks) + writer.save(str(path)) + stale = load_session(str(path), **options) + running = asyncio.create_task(stale.run_loop()) + await asyncio.wait_for(first_request.wait(), 10) + + if peer_publishes_during_fallback: + append_replacement = stale.store.append_snapshot_replacement + + def fail_checkpoint(*args, **kwargs): + raise OSError("controlled failure before checkpoint publication") + + def pause_fallback(*args, **kwargs): + fallback_started.set() + if not release_fallback.wait(10): + raise TimeoutError("fallback was not released") + return append_replacement(*args, **kwargs) + + monkeypatch.setattr(stale.store, "_atomic_json_write", fail_checkpoint) + monkeypatch.setattr(stale.store, "append_snapshot_replacement", pause_fallback) + reply_to_first.set() + assert await asyncio.to_thread(fallback_started.wait, 10) + + writer.messages = [*writer.messages, {"role": "user", "content": "peer checkpoint evidence"}] + writer.save(str(path)) + published = OpenCollab.read_session_snapshot(path) + release_fallback.set() + reply_to_first.set() + await asyncio.wait_for(second_request.wait(), 10) + await asyncio.gather(*stale._auto_save_subscriber.pending_tasks) + assert OpenCollab.read_session_snapshot(path) == published + assert any("file evidence" in row["content"] for row in stale.messages if row["role"] == "tool") + + reply_to_second.set() + assert await running == "done" + assert stale.messages[-1]["content"] == "done" + assert OpenCollab.read_session_snapshot(path) == published + error, = stale.persistence_errors + assert "conflicts with persisted" in str(error) + assert path.with_suffix(".json.journal").read_bytes() == b"" + + writer.messages.append({"role": "assistant", "content": "peer continued"}) + writer.save(str(path)) + resumed = OpenCollab.read_session_snapshot(path) + assert resumed["_autosave_sequence"] == published["_autosave_sequence"] + 1 + assert resumed["messages"][-1]["content"] == "peer continued" + finally: + release_fallback.set() + reply_to_first.set() + reply_to_second.set() + if running is not None: + await asyncio.gather(running, return_exceptions=True) + if stale is not None: + await stale.aclose() + await writer.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize("legacy_timestamps", [False, True], ids=["missing-timestamps", "timestamps"]) +async def test_native_tool_step_preserves_a_near_capacity_legacy_prefix(monkeypatch, tmp_path, legacy_timestamps): + limit = 96 * 1024 + monkeypatch.setattr(storage_module, "MAX_SESSION_SNAPSHOT_BYTES", limit) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + (tmp_path / "source.txt").write_text("file evidence\n", encoding="utf-8") + model = "claude-sonnet-5-5" + agent = _agent(thinking=False, model=model) + path = tmp_path / "legacy.json" + archive = {"role": "user", "content": ""} + prompt = "Read source.txt and then answer" + messages = [ + {"role": "system", "content": agent.system_prompt, "timestamp": "2026-01-01T00:00:00+00:00"}, + archive, + {"role": "assistant", "content": "That context is archived"}, + {"role": "user", "content": prompt}, + ] + if legacy_timestamps: + for message in messages: + message["timestamp"] = "2026-01-01T00:00:00+00:00" + snapshot = { + "snapshot_version": 1, "_autosave_sequence": 2, "messages": messages, + "session_state": { + "phase": "idle", "pending_external_user_turn": { + "turn_id": "legacy-turn", "status": "queued", "content": prompt, "message_index": 3, + }, + }, + } + archive["content"] = "x" * (limit - len(json.dumps(snapshot, separators=(",", ":"))) - 100) + path.write_text(json.dumps(snapshot, separators=(",", ":")), encoding="utf-8") + assert path.stat().st_size == limit - 100 + + class ArchiveShaper: + def shape(self, messages): + return [ + {**message, "content": "[Archived context]"} + if message.get("content") == archive["content"] else message + for message in messages + ] + + second_request = asyncio.Event() + peer = _BoundThinkingProvider(reads=1, thinking=False) + + async def provider(request): + response = peer(request) + if len(peer.requests) == 2: + second_request.set() + await asyncio.Future() + return response + + install_sdk_transport(monkeypatch, "anthropic", provider) + environment = LocalEnvironment(str(tmp_path)) + async with _client(model) as client: + session = load_session( + str(path), agent=agent, llm=client, env=environment, auto_save_path=str(path), shaper=ArchiveShaper(), + ) + running = asyncio.create_task(session.run_loop()) + waiting = asyncio.create_task(second_request.wait()) + restored = None + try: + done, _ = await asyncio.wait({waiting, running}, timeout=10, return_when=asyncio.FIRST_COMPLETED) + if running in done: + pytest.fail(f"Turn ended before its second request with status {running.result()}") + assert waiting in done + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + assert session.persistence_errors == () + saved = OpenCollab.read_session_snapshot(path) + assert saved["_autosave_sequence"] == 3 + assert saved["messages"][:len(messages)] == messages + record = json.loads(path.with_suffix(".json.journal").read_text(encoding="utf-8")) + assert record["replace_from"] == len(messages) + restored = load_session(str(path), agent=agent, llm=client, env=environment) + assert restored.state.anthropic_history is not None + assert any("file evidence" in row["content"] for row in restored.messages if row["role"] == "tool") + finally: + running.cancel() + waiting.cancel() + await asyncio.gather(running, waiting, return_exceptions=True) + if restored is not None: + await restored.aclose() + await session.aclose() + await environment.cleanup() + + @pytest.mark.parametrize( ("base_sequence", "failed_write", "expected_writes"), [ From 11f91ce4c70922a95b9b9564bb24f17d533b6190 Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 09:17:05 +0800 Subject: [PATCH 61/82] fix(llm): bind scoped controls to the routed completion Match forwarded admission callbacks by identity. Preserve request identity across legacy message copies so warmups and equal-payload helper calls retain their own budgets and usage. --- opencollab/adapters/llm/client.py | 32 +++++----- .../application/_session_run_completion.py | 38 ++++++------ opencollab/application/ports.py | 43 +++++++++++++- .../runtime/test_pooled_generation_timeout.py | 59 ++++++++++++++++--- 4 files changed, 125 insertions(+), 47 deletions(-) diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index 0c71a2af..3a55a115 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -43,6 +43,7 @@ from opencollab.application.ports import ( _NATIVE_COMPLETION_CAPABILITY, _NATIVE_REQUEST_ADMISSION, + _LegacyCompletionMessage, _native_completion_controls, ) from opencollab.domain.token_estimation import estimate_request_tokens @@ -324,22 +325,21 @@ async def complete( backoff. ``on_usage`` receives the final aggregate before slot release. """ scope = _native_completion_controls.get() - if scope is not None: - owner, submitted, admission_callback, controls = scope - # An explicitly forwarded admission callback still identifies a rewritten request. - forwarded = admission_callback is not None and on_admitted is admission_callback - if controls and (forwarded or submitted == messages): - if on_admitted is None: - on_admitted = controls.get("on_admitted") - if before_request is None: - before_request = controls.get("before_request") - if on_attempt_usage is None: - on_attempt_usage = controls.get("on_attempt_usage") - if on_usage is None: - on_usage = controls.get("on_usage") - controls.clear() - elif owner is self and on_admitted is None: - on_admitted = controls.pop("on_admitted", None) + if scope is not None and scope.callbacks and scope.matches(messages, on_admitted): + controls = scope.callbacks + if on_admitted is None: + on_admitted = controls.get("on_admitted") + if before_request is None: + before_request = controls.get("before_request") + if on_attempt_usage is None: + on_attempt_usage = controls.get("on_attempt_usage") + if on_usage is None: + on_usage = controls.get("on_usage") + controls.clear() + if scope is not None and scope.admission_callback is None: + messages = [ + dict(message) if isinstance(message, _LegacyCompletionMessage) else message for message in messages + ] start = time.monotonic() response = None diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index 91882b27..7fe453f5 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -44,6 +44,7 @@ RequestTokenEstimatorPort, ThinkingPrefixPort, _native_completion_controls, + _NativeCompletionControls, ) from opencollab.application.shaping import ShaperPipeline from opencollab.application.shaping.pipeline import context_token_estimator @@ -64,17 +65,14 @@ async def _with_native_completion_controls( operation: Awaitable[CompletionResponse], - owner: object | None, - messages: list[dict[str, Any]], - admission_callback: Callable[[], None] | None, - controls: dict[str, Callable[..., Any]], + scope: _NativeCompletionControls | None, ) -> CompletionResponse: - scope = (owner, messages, admission_callback, controls) if owner is not None and controls else None token = _native_completion_controls.set(scope) try: return await operation finally: - controls.clear() + if scope is not None: + scope.callbacks.clear() _native_completion_controls.reset(token) def _completion_keyword_parameters( @@ -817,7 +815,7 @@ def _add_completion_controls( max_output_tokens: int, usage_accounting: _CompletionUsageAccounting, revalidate: Callable[[], int], - ) -> tuple[asyncio.Event | None, object | None, dict[str, Callable[..., Any]]]: + ) -> tuple[asyncio.Event | None, _NativeCompletionControls | None]: keywords, outer_accepts_kwargs, outer_keywords = _completion_keyword_parameters(complete) owner = getattr(complete, "__self__", None) scoped: dict[str, Callable[..., Any]] = {} @@ -868,7 +866,8 @@ def add_control( native_support=_NATIVE_REQUEST_ADMISSION, allow_scope=self._per_call_timeout is not None and outer_accepts_kwargs, ) - return admitted if has_admission else None, owner, scoped + scope = _NativeCompletionControls(kwargs.get("on_admitted"), scoped) if scoped else None + return admitted if has_admission else None, scope def _revalidate_request( self, messages: list[dict], tools: list[dict] | None, *, thinking: bool = False, @@ -904,10 +903,12 @@ def revalidate() -> int: ) return min(limit, _summary_output_limit(messages)) if _summary_output_limit is not None else limit - admitted, native_owner, scoped = self._add_completion_controls( + admitted, scope = self._add_completion_controls( self.llm.complete, kwargs, max_output_tokens=max_output_tokens, usage_accounting=usage_accounting, revalidate=revalidate, ) + if scope is not None: + messages = scope.prepare_messages(messages) abandoned = False if self.tracer: self.tracer.log_step(step_type="llm_call_started", payload={ @@ -941,10 +942,8 @@ async def complete_owned() -> CompletionResponse: self._finish_run_control_if_quiesced() operation = complete_owned() - if scoped or _native_completion_controls.get() is not None: - operation = _with_native_completion_controls( - operation, native_owner, messages, kwargs.get("on_admitted"), scoped, - ) + if scope is not None or _native_completion_controls.get() is not None: + operation = _with_native_completion_controls(operation, scope) owner = asyncio.create_task(operation) self._track_provider_task(owner) try: @@ -982,13 +981,16 @@ async def _invoke_llm(self, *, max_output_tokens: int, **kwargs: Any) -> Complet protected_call = self.state.wind_down_done usage_accounting = _CompletionUsageAccounting(self, protected_call=protected_call) complete = self.llm.complete - admitted, native_owner, scoped = self._add_completion_controls( - complete, kwargs, max_output_tokens=max_output_tokens, usage_accounting=usage_accounting, + admitted, scope = self._add_completion_controls( + complete, kwargs, max_output_tokens=max_output_tokens, + usage_accounting=usage_accounting, revalidate=lambda: self._revalidate_request( kwargs["messages"], kwargs.get("tools"), thinking=kwargs.get("thinking", False), thinking_params=kwargs.get("thinking_params"), ), ) + if scope is not None: + kwargs["messages"] = scope.prepare_messages(kwargs["messages"]) late_accounted = False async def complete_owned() -> CompletionResponse: @@ -1014,10 +1016,8 @@ def record_abandoned(task: asyncio.Task[Any]) -> None: self._record_late_provider_result(task, protected_call=protected_call, usage_accounted=True) operation = complete_owned() - if scoped or _native_completion_controls.get() is not None: - operation = _with_native_completion_controls( - operation, native_owner, kwargs["messages"], kwargs.get("on_admitted"), scoped, - ) + if scope is not None or _native_completion_controls.get() is not None: + operation = _with_native_completion_controls(operation, scope) try: response = await operation if self._per_call_timeout is None else await abandon_on_timeout( operation, diff --git a/opencollab/application/ports.py b/opencollab/application/ports.py index 05af0d9b..6c3b43db 100644 --- a/opencollab/application/ports.py +++ b/opencollab/application/ports.py @@ -2,6 +2,8 @@ from collections.abc import Mapping, Sequence from contextvars import ContextVar +from copy import deepcopy +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Awaitable, Callable, Protocol, runtime_checkable from opencollab.domain.hooks import HookOutcome @@ -484,9 +486,44 @@ def __get__(self, instance: Any, owner: type) -> bool: # Native clients support these callbacks through their completion signature. _NATIVE_COMPLETION_CAPABILITY = _NativeCompletionCapability() _NATIVE_REQUEST_ADMISSION = _NativeRequestAdmission() -_native_completion_controls: ContextVar[ - tuple[object, list[dict[str, Any]], Callable[[], None] | None, dict[str, Callable[..., Any]]] | None -] = ContextVar( + + +@dataclass +class _NativeCompletionControls: + admission_callback: Callable[[], None] | None + callbacks: dict[str, Callable[..., Any]] + legacy_request_key: object = field(default_factory=object, init=False) + + def prepare_messages(self, messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + if self.admission_callback is not None: + return messages + return [_LegacyCompletionMessage(message, self.legacy_request_key) for message in messages] + + def matches(self, messages: list[dict[str, Any]], on_admitted: Callable[[], None] | None) -> bool: + if self.admission_callback is not None: + return on_admitted is self.admission_callback + return any( + isinstance(message, _LegacyCompletionMessage) and message.request_key is self.legacy_request_key + for message in messages + ) + + +class _LegacyCompletionMessage(dict[str, Any]): + __slots__ = ("request_key",) + + def __init__(self, message: dict[str, Any], request_key: object) -> None: + super().__init__(message) + self.request_key = request_key + + def __deepcopy__(self, memo: dict[int, Any]) -> _LegacyCompletionMessage: + # Keep request identity when legacy routes copy messages before editing them. + result = _LegacyCompletionMessage({}, self.request_key) + memo[id(self)] = result + result.update(deepcopy(dict(self), memo)) + return result + + +_native_completion_controls: ContextVar[_NativeCompletionControls | None] = ContextVar( "native_completion_controls", default=None, ) diff --git a/tests/runtime/test_pooled_generation_timeout.py b/tests/runtime/test_pooled_generation_timeout.py index 2ade403f..e4972b08 100644 --- a/tests/runtime/test_pooled_generation_timeout.py +++ b/tests/runtime/test_pooled_generation_timeout.py @@ -75,6 +75,12 @@ async def complete(self, *args, **kwargs): return await super().complete(*args, **kwargs) +class _ExplicitAdmissionRoute(_DelegatingPooledClient): + @wraps(_LegacyPooledClient.complete) + async def complete(self, *args, on_admitted=None, **kwargs): + return await self.delegate(*args, on_admitted=on_admitted, **kwargs) + + @pytest.fixture( params=[LLMClient, _AdmissionMiddleware, _ForwardingPooledClient], ids=["native", "decorated", "inherited"], @@ -313,20 +319,42 @@ async def delegate(messages, *args, **kwargs): @pytest.mark.parametrize("summary", [False, True]) @pytest.mark.parametrize("stall", [False, True]) +@pytest.mark.parametrize( + "route_class", [_VariadicLegacyRoute, _DelegatingPooledClient, _ExplicitAdmissionRoute], + ids=["opaque", "legacy-metadata", "explicit-admission"], +) async def test_routed_rewrite_preserves_deadline_and_separate_preflight_usage( - tmp_path, monkeypatch, local_environment_factory, summary, stall, + tmp_path, monkeypatch, local_environment_factory, summary, stall, route_class, ): ready = asyncio.Event() entered = asyncio.Event() calls = [] + preflight_limits = [] + target_limit = 64 if route_class is _DelegatingPooledClient else 128 + reserved = 0 - async def complete(_client, _model, messages, *_args, **_kwargs): - if messages[-1]["content"] == "Preflight": - return LLMResponse(content="ok", usage=Usage(2, 2)) + async def complete( + _client, model, messages, *_args, max_output_tokens, before_attempt, on_attempt_usage, **_kwargs, + ): + if model == "preflight" or messages[-1]["content"] == "Preflight": + preflight_limits.append(max_output_tokens) + return LLMResponse(content="ok", usage=Usage(20, 30)) calls.append(True) + assert messages[-1]["content"] == "Routed request" + assert type(max_output_tokens) is int and max_output_tokens == target_limit + assert all(type(message) is dict for message in messages) entered.set() if stall: await asyncio.Event().wait() + assert before_attempt() == target_limit + previously_used = session.used_tokens + on_attempt_usage(Usage(2, 1)) + assert session.used_tokens == previously_used + 3 + session.max_budget_tokens = session.used_tokens + reserved + 31 + try: + assert before_attempt() == 31 + finally: + session.max_budget_tokens = 20_000 content = "Earlier context." if summary and len(calls) == 1 else "answer" return LLMResponse(content=content, usage=Usage(7, 3)) @@ -335,19 +363,28 @@ async def complete(_client, _model, messages, *_args, **_kwargs): destination.mkdir() config = dict(api_key="local", context_window=40_000, request_capacity=1) async with ( - _VariadicLegacyRoute(**config, request_pool=str(tmp_path)) as client, + route_class(**config, request_pool=str(tmp_path)) as client, LLMClient(**config, request_pool=str(destination)) as recipient, + LLMClient(api_key=__name__, model="preflight") as helper, ): async def delegate(messages, *args, **kwargs): - await LLMClient.complete(client, [{"role": "user", "content": "Preflight"}]) + nonlocal reserved + reserved = client.estimate_request_tokens(messages, kwargs.get("tools")) + if route_class is _ExplicitAdmissionRoute: + await helper.complete(deepcopy(messages)) + else: + await LLMClient.complete(client, [{"role": "user", "content": "Preflight"}]) ready.set() - rewritten = [*messages, {"role": "system", "content": "Routed request"}] + rewritten = deepcopy(messages) + [{"role": "system", "content": "Routed request"}] + if route_class is _DelegatingPooledClient: + kwargs.pop("temperature", None) + kwargs["max_output_tokens"] = min(int(kwargs["max_output_tokens"]), target_limit) return await recipient.complete(rewritten, *args, **kwargs) client.delegate = delegate session = build_session( - agent=Agent(name="routed", system_prompt="system"), llm=client, - env=local_environment_factory(str(tmp_path)), llm_timeout=0.09, + agent=Agent(name="routed", system_prompt="system", max_tokens_per_step=128), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.09, max_budget_tokens=20_000, ) session.messages = history(summary) try: @@ -357,6 +394,7 @@ async def delegate(messages, *args, **kwargs): await asyncio.sleep(0.15) assert not owner.done() and not entered.is_set() assert session.used_tokens == 0 + assert preflight_limits == [None] await asyncio.wait_for(entered.wait(), 1) if stall: with pytest.raises(GenerationTimeoutError): @@ -366,6 +404,9 @@ async def delegate(messages, *args, **kwargs): assert len(calls) == (2 if summary else 1) assert session.used_tokens == 10 * len(calls) finally: + if not owner.done(): + owner.cancel() + await asyncio.gather(owner, return_exceptions=True) await session.aclose() From a1b774c5557245146563f279d84761b1184b3d83 Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 09:27:09 +0800 Subject: [PATCH 62/82] fix(llm): settle usage for each routed provider completion Account provider retries and fallback completions within their own request scope. Keep copied results idempotent and emit usage for every settled provider call. --- opencollab/adapters/llm/client.py | 4 + .../application/_session_run_completion.py | 24 ++--- opencollab/application/_session_run_usage.py | 91 ++++++++++++++---- opencollab/application/session_run.py | 33 ------- tests/runtime/test_routed_completion_usage.py | 92 +++++++++++++++++++ 5 files changed, 179 insertions(+), 65 deletions(-) create mode 100644 tests/runtime/test_routed_completion_usage.py diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index 3a55a115..c00b957d 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -39,6 +39,7 @@ from opencollab.adapters.llm.retry import RetryTimeBudget, _scoped_native_retry_budget from opencollab.adapters.llm.types import LLMResponse, model_context_window from opencollab.adapters.llm.usage_ledger import record_api_usage +from opencollab.application._session_run_usage import _observe_native_usage, _own_native_usage from opencollab.application.async_timeout import await_owned_operation from opencollab.application.ports import ( _NATIVE_COMPLETION_CAPABILITY, @@ -370,6 +371,8 @@ def prepare_attempt() -> int | None: ) try: async with AsyncExitStack() as request_owner: + if on_usage is not None or on_attempt_usage is not None: + request_owner.enter_context(_own_native_usage()) first_token = request_owner.enter_context(recording()) try: await request_owner.enter_async_context(request_slot(self.request_pool, self.request_capacity)) @@ -447,6 +450,7 @@ def prepare_attempt() -> int | None: raise finally: settled_usage = getattr(response if response is not None else provider_error, "usage", None) + _observe_native_usage(settled_usage) transport_timing = first_token.snapshot() latency_s = time.monotonic() - start if response is not None: diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index 7fe453f5..d412056e 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -858,7 +858,7 @@ def add_control( if max_output_tokens != DEFAULT_MAX_TOKENS_PER_STEP or accepts("max_output_tokens"): kwargs["max_output_tokens"] = max_output_tokens add_control("on_attempt_usage", "supports_attempt_usage", usage_accounting.record_attempt) - add_control("on_usage", "supports_usage_settlement", usage_accounting.settle) + add_control("on_usage", "supports_usage_settlement", usage_accounting.settle_provider) add_control("before_request", "supports_request_revalidation", revalidate) admitted = asyncio.Event() has_admission = add_control( @@ -918,15 +918,10 @@ def revalidate() -> int: }) def account_response(response: CompletionResponse) -> None: - input_tokens, total_tokens = _normalize_completion_usage(response.usage) - usage_accounting.settle(response.usage) - self._record_usage_event(response.usage, total_tokens, purpose="summary", late=abandoned) + input_tokens, _total_tokens = _normalize_completion_usage(response.usage) + usage_accounting.finish(response.usage, purpose="summary", late=abandoned) self.state.add_markup_recovered(getattr(response.usage, "markup_recovered", 0)) self.state.set_context_tokens(input_tokens) - if abandoned: - self._late_provider_usage += (total_tokens,) - if self.late_provider_usage_checkpoint is not None: - self.late_provider_usage_checkpoint() self.record_llm_trace(response, time.monotonic() - start, purpose="summary", thinking=thinking) async def complete_owned() -> CompletionResponse: @@ -1000,20 +995,19 @@ async def complete_owned() -> CompletionResponse: self, complete, usage_accounting=usage_accounting, **kwargs, ) late_accounted = asyncio.current_task() in self._draining_provider_tasks - self._account_completion_usage( - response, protected_call=protected_call, late=late_accounted, - accounted_tokens=usage_accounting.tokens, - ) + usage_accounting.finish(response.usage, purpose="completion", late=late_accounted) return response finally: self._draining_provider_tasks.discard(asyncio.current_task()) self._finish_run_control_if_quiesced() def record_abandoned(task: asyncio.Task[Any]) -> None: - if late_accounted: + try: + if not late_accounted and not task.cancelled() and task.exception() is None: + usage_accounting.record_late() + finally: + self._draining_provider_tasks.discard(task) self._provider_task_done(task) - else: - self._record_late_provider_result(task, protected_call=protected_call, usage_accounted=True) operation = complete_owned() if scope is not None or _native_completion_controls.get() is not None: diff --git a/opencollab/application/_session_run_usage.py b/opencollab/application/_session_run_usage.py index 66833881..21537a53 100644 --- a/opencollab/application/_session_run_usage.py +++ b/opencollab/application/_session_run_usage.py @@ -4,8 +4,29 @@ import asyncio import time +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar from typing import Any, Awaitable, Callable +_native_usage_owner: ContextVar[object | None] = ContextVar("native_usage_owner", default=None) +_native_usage_observer: ContextVar[Callable[[Any], None] | None] = ContextVar("native_usage_observer", default=None) + + +@contextmanager +def _own_native_usage() -> Iterator[None]: + token = _native_usage_owner.set(object()) + try: + yield + finally: + _native_usage_owner.reset(token) + + +def _observe_native_usage(usage: Any) -> None: + observer = _native_usage_observer.get() + if observer is not None and usage is not None: + observer(usage) + def _nonnegative_usage_int(value: Any, field: str) -> int: if isinstance(value, bool) or not isinstance(value, int) or value < 0: @@ -39,12 +60,58 @@ def __init__(self, runner: Any, *, protected_call: bool) -> None: self.runner = runner self.protected_call = protected_call self.tokens = 0 + self._attempt_tokens: dict[object | None, int] = {} + self._native_usages: list[Any] = [] + self._settled: list[tuple[Any, int]] = [] + self._reported = False + + @contextmanager + def observe_provider_responses(self) -> Iterator[None]: + token = _native_usage_observer.set(self._native_usages.append) + try: + yield + finally: + _native_usage_observer.reset(token) def record_attempt(self, usage: Any) -> None: - self._charge(_normalize_completion_usage(usage)[1]) + owner = _native_usage_owner.get() + tokens = _normalize_completion_usage(usage)[1] + self._charge(tokens) + self._attempt_tokens[owner] = self._attempt_tokens.get(owner, 0) + tokens + + def settle_provider(self, usage: Any) -> None: + tokens = _normalize_completion_usage(usage)[1] + self._charge(tokens - self._attempt_tokens.pop(_native_usage_owner.get(), 0)) + self._settled.append((usage, tokens)) + + def finish(self, usage: Any, *, purpose: str, late: bool, error: BaseException | None = None) -> bool: + if self._reported: + return False + # Middleware may copy the chosen response or return a different native call with equal counters. + native_usage = usage + if usage is not None and all(item is not usage for item in self._native_usages): + native_usage = next((item for item in reversed(self._native_usages) if item == usage), usage) + if usage is not None and all(native_usage is not settled for settled, _tokens in self._settled): + # A routed completion can return a new response after settling another provider. + tokens = _normalize_completion_usage(usage)[1] + self._charge(tokens - sum(self._attempt_tokens.values())) + self._attempt_tokens.clear() + self._settled.append((native_usage, tokens)) + self._reported = True + record_usage = getattr(self.runner, "_record_usage_event", None) + if callable(record_usage): + for settled, tokens in self._settled: + record_usage(settled, tokens, purpose=purpose, late=late, + error=error if settled is native_usage else None) + if late: + self.record_late() + return bool(self._settled) - def settle(self, usage: Any) -> None: - self._charge(_normalize_completion_usage(usage)[1] - self.tokens) + def record_late(self) -> None: + if self._settled: + self.runner._late_provider_usage += (self.tokens,) + if self.runner.late_provider_usage_checkpoint is not None: + self.runner.late_provider_usage_checkpoint() def _charge(self, tokens: int) -> None: self.runner._charge_completion_tokens(tokens) @@ -63,21 +130,11 @@ async def _complete_with_error_usage( """Charge reported error usage inside the provider owner exactly once.""" start = time.monotonic() try: - return await complete(*args, **kwargs) + with usage_accounting.observe_provider_responses(): + return await complete(*args, **kwargs) except BaseException as exc: - usage = getattr(exc, "usage", None) - if usage is not None: - _input_tokens, total_tokens = _normalize_completion_usage(usage) - usage_accounting.settle(usage) - late = asyncio.current_task() in runner._draining_provider_tasks - record_usage = getattr(runner, "_record_usage_event", None) - if callable(record_usage): - record_usage(usage, total_tokens, purpose=usage_purpose, - late=late, error=exc) - if late: - runner._late_provider_usage += (total_tokens,) - if runner.late_provider_usage_checkpoint is not None: - runner.late_provider_usage_checkpoint() + late = asyncio.current_task() in runner._draining_provider_tasks + if usage_accounting.finish(getattr(exc, "usage", None), purpose=usage_purpose, late=late, error=exc): try: runner.record_llm_trace( None, time.monotonic() - start, purpose=usage_purpose, diff --git a/opencollab/application/session_run.py b/opencollab/application/session_run.py index 49bf68d9..affddb20 100644 --- a/opencollab/application/session_run.py +++ b/opencollab/application/session_run.py @@ -246,44 +246,11 @@ def _mark_budget_reserve_consumed(self, *, protected_call: bool | None = None) - ): self.state.budget_reserve_consumed = True - def _record_late_provider_result( - self, - task: asyncio.Future[Any], - *, - protected_call: bool = False, - usage_accounted: bool = False, - ) -> None: - """Charge a provider response that survived caller cancellation or timeout.""" - try: - response = task.result() - self._account_completion_usage( - response, protected_call=protected_call, late=True, usage_accounted=usage_accounted, - ) - except BaseException: - pass - finally: - self._draining_provider_tasks.discard(task) - self._finish_run_control_if_quiesced() - def _charge_completion_tokens(self, tokens: int) -> None: self.state.add_used_tokens(tokens) if tokens and self._on_completion_tokens is not None: self._on_completion_tokens(tokens) - def _account_completion_usage( - self, response: Any, *, protected_call: bool, late: bool, usage_accounted: bool = False, - accounted_tokens: int = 0, - ) -> None: - _input_tokens, total_tokens = _normalize_completion_usage(response.usage) - if not usage_accounted: - self._charge_completion_tokens(total_tokens - accounted_tokens) - self._record_usage_event(response.usage, total_tokens, purpose="completion", late=late) - self._mark_budget_reserve_consumed(protected_call=protected_call) - if late: - self._late_provider_usage += (total_tokens,) - if self.late_provider_usage_checkpoint is not None: - self.late_provider_usage_checkpoint() - async def run_loop(self, cancel_event: asyncio.Event | None = None) -> str: """Drive the phase FSM until the turn finishes or suspends. diff --git a/tests/runtime/test_routed_completion_usage.py b/tests/runtime/test_routed_completion_usage.py new file mode 100644 index 00000000..695a2d45 --- /dev/null +++ b/tests/runtime/test_routed_completion_usage.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import json +from copy import deepcopy + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm.client import LLMClient +from opencollab.application.run_control import RunControl +from opencollab.bootstrap import build_session +from opencollab.domain.agent import Agent +from tests.support.provider_sdk_http import completion_http_response, install_sdk_transport + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("forward_fallback", [False, True]) +@pytest.mark.parametrize("scenario", ["primary", "copied-primary", "fallback", "equal-fallback", "copied-fallback"]) +async def test_routed_completion_accounts_for_each_provider( + tmp_path, monkeypatch, summary, forward_fallback, scenario, +): + requests = [] + events = [] + answer = "Answer." if summary else "Answer." + usable_primary = scenario in {"primary", "copied-primary"} + equal_usage = scenario in {"equal-fallback", "copied-fallback"} + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + + async def handle(request): + route = request.url.path.split("/")[1] + requests.append(route) + if route == "primary": + return completion_http_response(request, text=answer if usable_primary else "", output_tokens=55) + return completion_http_response(request, text=answer, output_tokens=55 if equal_usage else 2) + + class FallbackClient(LLMClient): + async def complete( + self, messages, *, on_admitted=None, before_request=None, + on_attempt_usage=None, on_usage=None, **kwargs, + ): + controls = dict(on_admitted=on_admitted, before_request=before_request, + on_attempt_usage=on_attempt_usage, on_usage=on_usage) + first = await super().complete(messages, **kwargs, **controls) + if first.content and first.content.strip(): + return deepcopy(first) if scenario == "copied-primary" else first + final = await fallback.complete(messages, **kwargs, **(controls if forward_fallback else {})) + return deepcopy(final) if scenario == "copied-fallback" else final + + install_sdk_transport(monkeypatch, "openai", handle) + options = dict(api_key=__name__, context_window=40_000, max_retries=0, + request_pool=str(tmp_path), request_capacity=1) + control = RunControl(on_event=events.append) + async with ( + FallbackClient(**options, base_url="https://test.invalid/primary/v1") as client, + LLMClient(**options, base_url="https://test.invalid/fallback/v1") as fallback, + ): + if summary: + env = LocalEnvironment(str(tmp_path)) + session = build_session( + agent=Agent(name="routed", system_prompt="System.", max_tokens_per_step=128), llm=client, + env=env, llm_timeout=2, max_budget_tokens=20_000, run_control=control, + ) + try: + await session.add_user_message("Return a brief summary.") + text = await session._context_summarizer.asummarize(session.messages) + assert text == "Summary:\nAnswer." + assert session._context_summarizer.last_call_cacheable + tokens = session.used_tokens + finally: + await session.aclose() + await env.cleanup() + else: + result = await OpenCollab(tmp_path, config={"llm_timeout": 2, "max_output_tokens": 128}).agent( + "Return a brief answer.", llm=client, tools=(), budget=20_000, trace=False, + run_control=control, system_prompt="System.", + ) + assert result.status == "completed", result.reason + assert result.output == "Answer." + tokens = result.tokens + + expected_usage = [57] if usable_primary else [57, 57 if equal_usage else 4] + expected_purpose = "summary" if summary else "completion" + ledger = [json.loads(line) for line in (tmp_path / "usage.jsonl").read_text().splitlines()] + assert requests == (["primary"] if usable_primary else ["primary", "fallback"]) + assert tokens == sum(expected_usage) + assert [entry["usage"]["total_tokens"] for entry in ledger] == expected_usage + usage_events = [event for event in events if event.type == "usage"] + assert [event.data["total_tokens"] for event in usage_events] == expected_usage + assert {event.data["purpose"] for event in usage_events} == {expected_purpose} From 9a75589b0ce243c80d1c91c272b858df511e5144 Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 09:26:39 +0800 Subject: [PATCH 63/82] fix(session): accept JSON builtin subclasses in autosave comparisons Compare integer and float subclasses by their encoded values and compare list and dictionary subclasses recursively. Preserve the distinctions between boolean and numeric values, integer and float values, and signed zero. --- opencollab/domain/session.py | 19 ++++++---- tests/agents/test_session_construction.py | 45 +++++++++++++++++++++++ 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/opencollab/domain/session.py b/opencollab/domain/session.py index cc6e54ba..fc79aaa1 100644 --- a/opencollab/domain/session.py +++ b/opencollab/domain/session.py @@ -17,20 +17,23 @@ def same_snapshot_value(left: Any, right: Any) -> bool: if left is right: return True + # The JSON encoder writes builtin subclasses as their underlying values. if isinstance(left, str) and isinstance(right, str): - # The JSON encoder writes string subclasses, including enums, as text. return str.__eq__(left, right) - if type(left) is not type(right): - return False - if isinstance(left, dict): + if isinstance(left, dict) and isinstance(right, dict): return left.keys() == right.keys() and all( same_snapshot_value(value, right[key]) for key, value in left.items() ) - if isinstance(left, list): + if isinstance(left, list) and isinstance(right, list): return len(left) == len(right) and all(same_snapshot_value(a, b) for a, b in zip(left, right)) - if isinstance(left, float) and left == right == 0: - return math.copysign(1, left) == math.copysign(1, right) - return left == right + if isinstance(left, int) and isinstance(right, int): + return (type(left) is bool) == (type(right) is bool) and int.__eq__(left, right) + if isinstance(left, float) and isinstance(right, float): + left, right = float.__float__(left), float.__float__(right) + if left == right == 0: + return math.copysign(1, left) == math.copysign(1, right) + return left == right + return type(left) is type(right) and left == right def _now_iso() -> str: diff --git a/tests/agents/test_session_construction.py b/tests/agents/test_session_construction.py index 1d781225..e9a2baa7 100644 --- a/tests/agents/test_session_construction.py +++ b/tests/agents/test_session_construction.py @@ -6,6 +6,8 @@ import json import os import threading +from collections import OrderedDict +from enum import IntEnum import pytest @@ -295,6 +297,49 @@ def test_checkpoint_rejects_sequence_older_than_persisted_journal(tmp_path): assert restored["_autosave_sequence"] == 2 assert restored["messages"][-1]["content"] == "newer-delta" + +class _SavedAgentId(IntEnum): + READER = 12 + + +class _SavedScore(float): + pass + + +class _SavedScores(list): + pass + + +@pytest.mark.parametrize(("aid", "metadata"), [ + pytest.param(_SavedAgentId.READER, {}, id="integer-enum"), + pytest.param(12, {"scores": [_SavedScore(-0.0), _SavedScore(0.75)]}, id="float-subclass"), + pytest.param(12, OrderedDict(scores=[-0.0, 0.75]), id="dict-subclass"), + pytest.param(12, {"scores": _SavedScores([-0.0, 0.75])}, id="list-subclass"), +]) +async def test_autosave_checkpoints_json_builtin_subclasses(local_environment_factory, tmp_path, aid, metadata): + path = tmp_path / "subclasses.json" + session = _new_session( + local_environment_factory, + env=local_environment_factory(str(tmp_path)), + aid=aid, + auto_save_path=str(path), + seed_user_messages=[{"role": "user", "content": "seed", "metadata": metadata}], + ) + try: + await session.add_user_message("record this turn") + await asyncio.gather(*session.pending_cleanup_tasks) + + assert session.persistence_errors == () + saved = json.loads(path.read_bytes()) + assert saved["_autosave_sequence"] == 1 + assert saved["aid"] == 12 + assert json.dumps(saved["messages"][1]["metadata"]) == json.dumps(metadata) + assert saved["messages"][-1]["content"] == "record this turn" + assert path.with_suffix(".json.journal").read_bytes() == b"" + finally: + await session.aclose() + + @pytest.mark.parametrize("publisher", ["journal", "snapshot"]) @pytest.mark.parametrize( ("published_value", "conflicting_value"), From 06ca18f26afa9ffe98f45eab03271ac063a17088 Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 09:27:00 +0800 Subject: [PATCH 64/82] fix(llm): preserve admission through legacy routing metadata --- .../application/_session_run_completion.py | 16 ++-- .../runtime/test_pooled_generation_timeout.py | 88 +++++++++++++++++++ 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index d412056e..35bee711 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -77,10 +77,11 @@ async def _with_native_completion_controls( def _completion_keyword_parameters( complete: Callable[..., Awaitable[Any]], -) -> tuple[set[str] | None, bool, set[str]]: +) -> tuple[set[str] | None, bool, set[str], Callable[..., Any]]: keywords: set[str] = set() outer_accepts_kwargs = False outer_keywords: set[str] = set() + target = complete def has_fixed_keywords(target: Callable[..., Any]) -> bool: nonlocal outer_accepts_kwargs @@ -100,10 +101,10 @@ def has_fixed_keywords(target: Callable[..., Any]) -> bool: try: target = inspect.unwrap(complete, stop=has_fixed_keywords) if not hasattr(target, "__wrapped__") and not has_fixed_keywords(target): - return None, outer_accepts_kwargs, outer_keywords + return None, outer_accepts_kwargs, outer_keywords, target except (TypeError, ValueError): pass - return keywords, outer_accepts_kwargs, outer_keywords + return keywords, outer_accepts_kwargs, outer_keywords, target def _selector_names_tool_choice(selector: Any) -> bool: """Return whether a structured error selector targets ``tool_choice``.""" @@ -816,8 +817,13 @@ def _add_completion_controls( usage_accounting: _CompletionUsageAccounting, revalidate: Callable[[], int], ) -> tuple[asyncio.Event | None, _NativeCompletionControls | None]: - keywords, outer_accepts_kwargs, outer_keywords = _completion_keyword_parameters(complete) + keywords, outer_accepts_kwargs, outer_keywords, wrapped = _completion_keyword_parameters(complete) owner = getattr(complete, "__self__", None) + wraps_base_method = owner is not None and any( + vars(base).get("complete") is wrapped for base in type(owner).__mro__[1:] + ) + # A metadata-only signature leaves admission on the outer router's kwargs. + accepts_admission = outer_accepts_kwargs and not wraps_base_method scoped: dict[str, Callable[..., Any]] = {} def accepts(name: str) -> bool: @@ -834,7 +840,7 @@ def add_control( supported = getattr(self.llm, capability, False) if not supported: return False - if name in outer_keywords or (name == "on_admitted" and accepts(name)): + if name in outer_keywords or (name == "on_admitted" and (accepts(name) or accepts_admission)): kwargs[name] = callback return True # A native marker also appears on subclasses whose wrappers forward diff --git a/tests/runtime/test_pooled_generation_timeout.py b/tests/runtime/test_pooled_generation_timeout.py index e4972b08..ae78e922 100644 --- a/tests/runtime/test_pooled_generation_timeout.py +++ b/tests/runtime/test_pooled_generation_timeout.py @@ -18,6 +18,7 @@ from opencollab.domain.agent import Agent from tests.runtime.test_async_compaction import history from tests.support.environment_test_support import local_environment_factory as local_environment_factory +from tests.support.provider_sdk_http import completion_http_response, install_sdk_transport class _LegacyPooledClient(LLMClient): @@ -81,6 +82,16 @@ async def complete(self, *args, on_admitted=None, **kwargs): return await self.delegate(*args, on_admitted=on_admitted, **kwargs) +def _legacy_completion_signature(self, messages, tools=None, temperature=0.0, *, max_output_tokens=None): + pass + + +class _MetadataOnlyPooledRoute(LLMClient): + @wraps(_legacy_completion_signature) + async def complete(self, *args, **kwargs): + return await self.delegate(*args, **kwargs) + + @pytest.fixture( params=[LLMClient, _AdmissionMiddleware, _ForwardingPooledClient], ids=["native", "decorated", "inherited"], @@ -410,6 +421,83 @@ async def delegate(messages, *args, **kwargs): await session.aclose() +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("stall", [False, True]) +async def test_legacy_router_normalizes_messages_and_forwards_admission( + tmp_path, monkeypatch, local_environment_factory, summary, stall, +): + helper_limits = [] + target_limits = [] + ready = asyncio.Event() + reserved_tokens = 0 + + async def handler(request): + body = json.loads(request.content) + if body["model"] == "preflight": + helper_limits.append(body.get("max_tokens")) + return completion_http_response(request, output_tokens=55) + assert body["messages"][0]["content"].endswith("\nUse the answer format.") + target_limits.append(body.get("max_tokens")) + if stall: + await asyncio.Event().wait() + content = "Earlier context." if summary else "answer" + return completion_http_response(request, text=content) + + install_sdk_transport(monkeypatch, "openai", handler) + config = dict(api_key=__name__, context_window=40_000, max_retries=0) + pool = dict(request_pool=str(tmp_path), request_capacity=1) + async with ( + _MetadataOnlyPooledRoute(**config, **pool) as client, + LLMClient(**config, **pool) as recipient, + LLMClient(**config, model="preflight") as helper, + ): + async def delegate(messages, *, max_output_tokens=None, on_admitted=None, **kwargs): + nonlocal reserved_tokens + reserved_tokens = client.estimate_request_tokens(messages, kwargs.get("tools")) + await helper.complete(deepcopy(messages)) + rewritten = json.loads(json.dumps(messages)) + rewritten[0]["content"] += "\nUse the answer format." + ready.set() + return await recipient.complete( + rewritten, max_output_tokens=max_output_tokens, on_admitted=on_admitted, **kwargs, + ) + + client.delegate = delegate + session = build_session( + agent=Agent(name="router", system_prompt="system", max_tokens_per_step=128), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.10, max_budget_tokens=20_000, + ) + await session.add_user_message("Prepare an answer.") + owner = None + try: + async with request_slot(str(tmp_path), 1): + operation = ( + session._context_summarizer.asummarize(session.messages) if summary else session.run_loop() + ) + owner = asyncio.create_task(operation) + await asyncio.wait_for(ready.wait(), 2) + await asyncio.sleep(0.15) + assert not owner.done() + assert not target_limits + assert session.used_tokens == 0 + assert helper_limits == [None] + session.max_budget_tokens = reserved_tokens + 31 + if stall: + with pytest.raises(GenerationTimeoutError): + await asyncio.wait_for(owner, 2) + else: + expected = "Summary:\nEarlier context." if summary else "answer" + assert await asyncio.wait_for(owner, 2) == expected + assert session.used_tokens == 4 + assert target_limits == [31] + finally: + if owner is not None: + if not owner.done(): + owner.cancel() + await asyncio.gather(owner, return_exceptions=True) + await session.aclose() + + @pytest.mark.parametrize("summary", [False, True]) @pytest.mark.parametrize("client_class", [_VariadicLegacyRoute, _MetadataLegacyRoute]) async def test_routed_legacy_completion_uses_response_usage( From f24b04ad63bf7548447d63d70f2bbfed8671ca6a Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 09:31:23 +0800 Subject: [PATCH 65/82] fix(workflow): rediscover retained mount recovery after permission changes --- opencollab/adapters/_candidate_adoption.py | 46 ++++++++++++------- .../test_candidate_adoption_mounts.py | 33 +++++++++---- 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py index c1855bca..286bd25b 100644 --- a/opencollab/adapters/_candidate_adoption.py +++ b/opencollab/adapters/_candidate_adoption.py @@ -240,6 +240,13 @@ _RECOVERY_PARENT = _FILESYSTEM + r''' writable_directory() { [ -d "$1" ] && [ -w "$1" ] && [ -x "$1" ]; } +recovery_temporary() { + temporary=$(cd -P -- "${TMPDIR:-/tmp}" >/dev/null 2>&1 && pwd -P && printf .) || return 1 + temporary=${temporary%??} + case "$temporary/" in "${1%/}/"*|"${2%/}/"*) return 1;; esac + printf '%s\n' "$temporary" +} + recovery_parent() { worktree_device=$(filesystem .) && storage_device=$(filesystem "$1/") || return 1 if [ "$worktree_device" = "$storage_device" ] && writable_directory "$1"; then @@ -260,16 +267,12 @@ fi done if [ -n "${2:-}" ]; then - temporary=$(cd -P -- "${TMPDIR:-/tmp}" >/dev/null 2>&1 && pwd -P && printf .) || temporary= + temporary=$(recovery_temporary "$workspace" "$2" && printf .) || temporary= temporary=${temporary%??} - case "$temporary/" in - "${workspace%/}/"*|"${2%/}/"*) ;; - *) - if writable_directory "$temporary" && [ "$(filesystem "$temporary")" = "$worktree_device" ]; then - printf '%s\n' "$temporary" - return - fi ;; - esac + if writable_directory "$temporary" && [ "$(filesystem "$temporary")" = "$worktree_device" ]; then + printf '%s\n' "$temporary" + return + fi fi if writable_directory "$1"; then printf '%s\n' "$1" @@ -288,18 +291,13 @@ async def _mounted_recovery_paths(environment: Any, workspace: str) -> tuple[str, ...]: - # The volume receipts live beside the primary backup, including for linked worktrees. + # Permissions may have changed since a recovery backup chose its parent. command = f"cd -- {shlex.quote(workspace)} || exit\n" + _RECOVERY_PARENT + _RECOVERY_GIT_DIRECTORIES + r''' workspace=$(pwd -P && printf .) || exit workspace=${workspace%??} prefix=${workspace%/}/ -primary=$(recovery_parent "$common" "$private" && printf .) || exit -primary=${primary%??} -if [ "$primary" -ef "$common" ] || [ "$primary" -ef "$private" ]; then primary=; fi -if [ "$private" -ef "$common" ]; then private=; fi -for parent in "$common" "$private" "$primary"; do - [ -n "$parent" ] || continue - for backup in "$parent"/candidate-recovery-????????; do +collect_recovery_paths() { + for backup in "$1"/candidate-recovery-????????; do [ -d "$backup" ] && [ ! -L "$backup" ] && [ -O "$backup" ] || continue volumes=$backup/volumes [ -d "$volumes" ] && [ ! -L "$volumes" ] && [ -O "$volumes" ] || continue @@ -318,7 +316,21 @@ async def _mounted_recovery_paths(environment: Any, workspace: str) -> tuple[str printf '%s\0' "${store#"$prefix"}" done done +} + +collect_recovery_paths "$common" +if [ ! "$private" -ef "$common" ]; then collect_recovery_paths "$private"; fi +parent=$workspace +while [ "$parent" != / ]; do + parent=${parent%/*} + parent=${parent:-/} + if [ ! "$parent" -ef "$common" ] && [ ! "$parent" -ef "$private" ]; then + collect_recovery_paths "$parent" + fi done +if temporary=$(recovery_temporary "$workspace" "$private" && printf .); then + collect_recovery_paths "${temporary%??}" +fi ''' result = _complete(await environment.exec_cmd(command, timeout=30), "mounted candidate recovery locations") return tuple(dict.fromkeys(path for path in result.split("\0") if path)) diff --git a/tests/workflows/test_candidate_adoption_mounts.py b/tests/workflows/test_candidate_adoption_mounts.py index c8487fc2..e797c060 100644 --- a/tests/workflows/test_candidate_adoption_mounts.py +++ b/tests/workflows/test_candidate_adoption_mounts.py @@ -2,6 +2,7 @@ import os import shlex +import stat from dataclasses import replace from pathlib import Path @@ -154,13 +155,16 @@ async def flow(ctx, _): await environment.cleanup() -@pytest.mark.parametrize(("source_edit", "linked"), [ - (None, False), - (None, True), - ("tracked", False), - ("untracked", False), +@pytest.mark.parametrize(("source_edit", "linked", "readonly_common"), [ + (None, False, False), + (None, True, False), + pytest.param(None, True, True, id="linked-permissions-restored"), + ("tracked", False, False), + ("untracked", False, False), ]) -async def test_followup_candidate_ignores_only_registered_mount_recovery(tmp_path, monkeypatch, source_edit, linked): +async def test_followup_candidate_ignores_only_registered_mount_recovery( + tmp_path, monkeypatch, source_edit, linked, readonly_common, +): source, candidate = _partial_publication(tmp_path) (source / "app.py").write_text("value = 1\n") mounted_file = "mounted/record.txt" @@ -188,6 +192,8 @@ async def test_followup_candidate_ignores_only_registered_mount_recovery(tmp_pat user_note.write_text("user data\n") index = Path(_git(repo, "rev-parse", "--absolute-git-dir").removesuffix("\n")) / "index" index_before = index.read_bytes() + common = source / ".git" + common_mode = stat.S_IMODE(common.stat().st_mode) environment = LocalEnvironment(str(repo)) workspace = EnvCandidateWorkspace(environment) execute = environment.exec_cmd @@ -214,10 +220,19 @@ async def edit(child, _): async def flow(ctx, _): nonlocal store_path, retained - with pytest.raises(RuntimeError, match="candidate adoption failed"): - await ctx.adopt_candidate(candidate) + try: + if readonly_common: + common.chmod(0o555) + if os.access(common, os.W_OK): + pytest.skip("requires an account subject to directory permissions") + with pytest.raises(RuntimeError, match="candidate adoption failed"): + await ctx.adopt_candidate(candidate) + finally: + if readonly_common: + common.chmod(common_mode) _git(repo, "restore", "--source=HEAD", "--worktree", mounted_file) - recovery, = (source / ".git").glob("candidate-recovery-*") + primary = repo.parent if readonly_common else common + recovery, = primary.glob("candidate-recovery-*") location, = (recovery / "volumes").iterdir() store = Path(location.read_text()) store_path = store.relative_to(repo) From 677bbff00d177e217f95974f74c0a25c11e665c8 Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 09:37:24 +0800 Subject: [PATCH 66/82] fix(llm): reconcile fallback retry usage by provider Settle a native provider with tracked attempts when middleware forwards only the retry callback. Preserve the provider error on usage events when a fallback completes the request. --- opencollab/adapters/llm/client.py | 7 +-- opencollab/application/_session_run_usage.py | 43 +++++++++------ tests/runtime/test_routed_completion_usage.py | 54 ++++++++++++++----- 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index c00b957d..48cdd667 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -450,7 +450,6 @@ def prepare_attempt() -> int | None: raise finally: settled_usage = getattr(response if response is not None else provider_error, "usage", None) - _observe_native_usage(settled_usage) transport_timing = first_token.snapshot() latency_s = time.monotonic() - start if response is not None: @@ -458,9 +457,11 @@ def prepare_attempt() -> int | None: response.transport_timing = transport_timing except (AttributeError, TypeError): pass - if on_usage is not None and settled_usage is not None: + if settled_usage is not None: try: - on_usage(settled_usage) + if on_usage is not None: + on_usage(settled_usage) + _observe_native_usage(settled_usage, provider_error) except BaseException as exc: exc.usage = settled_usage settlement_error = exc diff --git a/opencollab/application/_session_run_usage.py b/opencollab/application/_session_run_usage.py index 21537a53..f069b5ce 100644 --- a/opencollab/application/_session_run_usage.py +++ b/opencollab/application/_session_run_usage.py @@ -10,7 +10,9 @@ from typing import Any, Awaitable, Callable _native_usage_owner: ContextVar[object | None] = ContextVar("native_usage_owner", default=None) -_native_usage_observer: ContextVar[Callable[[Any], None] | None] = ContextVar("native_usage_observer", default=None) +_native_usage_observer: ContextVar[Callable[[Any, BaseException | None], None] | None] = ContextVar( + "native_usage_observer", default=None, +) @contextmanager @@ -22,10 +24,10 @@ def _own_native_usage() -> Iterator[None]: _native_usage_owner.reset(token) -def _observe_native_usage(usage: Any) -> None: +def _observe_native_usage(usage: Any, error: BaseException | None) -> None: observer = _native_usage_observer.get() - if observer is not None and usage is not None: - observer(usage) + if observer is not None: + observer(usage, error) def _nonnegative_usage_int(value: Any, field: str) -> int: @@ -61,13 +63,13 @@ def __init__(self, runner: Any, *, protected_call: bool) -> None: self.protected_call = protected_call self.tokens = 0 self._attempt_tokens: dict[object | None, int] = {} - self._native_usages: list[Any] = [] + self._native_usages: list[tuple[Any, BaseException | None]] = [] self._settled: list[tuple[Any, int]] = [] self._reported = False @contextmanager def observe_provider_responses(self) -> Iterator[None]: - token = _native_usage_observer.set(self._native_usages.append) + token = _native_usage_observer.set(self._observe_provider_response) try: yield finally: @@ -80,29 +82,38 @@ def record_attempt(self, usage: Any) -> None: self._attempt_tokens[owner] = self._attempt_tokens.get(owner, 0) + tokens def settle_provider(self, usage: Any) -> None: + self._settle(usage, self._attempt_tokens.pop(_native_usage_owner.get(), 0)) + + def _observe_provider_response(self, usage: Any, error: BaseException | None) -> None: + self._native_usages.append((usage, error)) + owner = _native_usage_owner.get() + # A router can forward the retry hook and keep the final callback for its chosen response. + if owner is not None and owner in self._attempt_tokens: + self.settle_provider(usage) + + def _settle(self, usage: Any, attempt_tokens: int) -> None: tokens = _normalize_completion_usage(usage)[1] - self._charge(tokens - self._attempt_tokens.pop(_native_usage_owner.get(), 0)) + self._charge(tokens - attempt_tokens) self._settled.append((usage, tokens)) def finish(self, usage: Any, *, purpose: str, late: bool, error: BaseException | None = None) -> bool: if self._reported: return False # Middleware may copy the chosen response or return a different native call with equal counters. - native_usage = usage - if usage is not None and all(item is not usage for item in self._native_usages): - native_usage = next((item for item in reversed(self._native_usages) if item == usage), usage) + native = next((item for item in self._native_usages if item[0] is usage), None) + if usage is not None and native is None: + native = next((item for item in reversed(self._native_usages) if item[0] == usage), None) + native_usage = usage if native is None else native[0] if usage is not None and all(native_usage is not settled for settled, _tokens in self._settled): - # A routed completion can return a new response after settling another provider. - tokens = _normalize_completion_usage(usage)[1] - self._charge(tokens - sum(self._attempt_tokens.values())) - self._attempt_tokens.clear() - self._settled.append((native_usage, tokens)) + attempt_tokens = self._attempt_tokens.pop(None, 0) if native is None else 0 + self._settle(native_usage, attempt_tokens) self._reported = True record_usage = getattr(self.runner, "_record_usage_event", None) if callable(record_usage): for settled, tokens in self._settled: + provider_error = next((failure for item, failure in self._native_usages if item is settled), None) record_usage(settled, tokens, purpose=purpose, late=late, - error=error if settled is native_usage else None) + error=error if settled is native_usage and error is not None else provider_error) if late: self.record_late() return bool(self._settled) diff --git a/tests/runtime/test_routed_completion_usage.py b/tests/runtime/test_routed_completion_usage.py index 695a2d45..75d452bb 100644 --- a/tests/runtime/test_routed_completion_usage.py +++ b/tests/runtime/test_routed_completion_usage.py @@ -7,16 +7,22 @@ from opencollab import OpenCollab from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm import retry from opencollab.adapters.llm.client import LLMClient +from opencollab.adapters.llm.responses_errors import ResponsesProtocolError from opencollab.application.run_control import RunControl from opencollab.bootstrap import build_session from opencollab.domain.agent import Agent -from tests.support.provider_sdk_http import completion_http_response, install_sdk_transport +from tests.runtime.test_responses_error_usage_http import _payload +from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport @pytest.mark.parametrize("summary", [False, True]) @pytest.mark.parametrize("forward_fallback", [False, True]) -@pytest.mark.parametrize("scenario", ["primary", "copied-primary", "fallback", "equal-fallback", "copied-fallback"]) +@pytest.mark.parametrize("scenario", [ + "primary", "copied-primary", "fallback", "equal-fallback", "copied-fallback", + "failed-primary", "failed-primary-attempt-callback-only", +]) async def test_routed_completion_accounts_for_each_provider( tmp_path, monkeypatch, summary, forward_fallback, scenario, ): @@ -25,14 +31,20 @@ async def test_routed_completion_accounts_for_each_provider( answer = "Answer." if summary else "Answer." usable_primary = scenario in {"primary", "copied-primary"} equal_usage = scenario in {"equal-fallback", "copied-fallback"} + failed_primary = scenario.startswith("failed-primary") monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + monkeypatch.setattr(retry, "extract_retry_after_seconds", lambda _error: 0.0) + monkeypatch.setattr(retry.random, "uniform", lambda _lo, _hi: 0.0) async def handle(request): route = request.url.path.split("/")[1] requests.append(route) if route == "primary": + if failed_primary: + content_type, payload = _payload(json.loads(request.content), {"kind": "failed"}) + return http_response(request, 200, headers={"content-type": content_type}, content=payload) return completion_http_response(request, text=answer if usable_primary else "", output_tokens=55) return completion_http_response(request, text=answer, output_tokens=55 if equal_usage else 2) @@ -43,19 +55,29 @@ async def complete( ): controls = dict(on_admitted=on_admitted, before_request=before_request, on_attempt_usage=on_attempt_usage, on_usage=on_usage) - first = await super().complete(messages, **kwargs, **controls) - if first.content and first.content.strip(): - return deepcopy(first) if scenario == "copied-primary" else first + primary_controls = dict(controls) + if scenario == "failed-primary-attempt-callback-only": + primary_controls.pop("on_usage") + try: + first = await super().complete(messages, **kwargs, **primary_controls) + except ResponsesProtocolError: + assert failed_primary + else: + if first.content and first.content.strip(): + return deepcopy(first) if scenario == "copied-primary" else first final = await fallback.complete(messages, **kwargs, **(controls if forward_fallback else {})) - return deepcopy(final) if scenario == "copied-fallback" else final + return deepcopy(final) if failed_primary or scenario == "copied-fallback" else final install_sdk_transport(monkeypatch, "openai", handle) - options = dict(api_key=__name__, context_window=40_000, max_retries=0, - request_pool=str(tmp_path), request_capacity=1) + options = dict(api_key=__name__, context_window=40_000, request_pool=str(tmp_path), request_capacity=1) control = RunControl(on_event=events.append) async with ( - FallbackClient(**options, base_url="https://test.invalid/primary/v1") as client, - LLMClient(**options, base_url="https://test.invalid/fallback/v1") as fallback, + FallbackClient( + **options, base_url="https://test.invalid/primary/v1", + wire_protocol="responses" if failed_primary else "chat_completions", + max_retries=1 if failed_primary else 0, + ) as client, + LLMClient(**options, base_url="https://test.invalid/fallback/v1", max_retries=0) as fallback, ): if summary: env = LocalEnvironment(str(tmp_path)) @@ -81,12 +103,20 @@ async def complete( assert result.output == "Answer." tokens = result.tokens - expected_usage = [57] if usable_primary else [57, 57 if equal_usage else 4] + expected_usage = [74 if failed_primary else 57] + if not usable_primary: + expected_usage.append(57 if equal_usage else 4) expected_purpose = "summary" if summary else "completion" ledger = [json.loads(line) for line in (tmp_path / "usage.jsonl").read_text().splitlines()] - assert requests == (["primary"] if usable_primary else ["primary", "fallback"]) + expected_requests = ["primary", "primary"] if failed_primary else ["primary"] + if not usable_primary: + expected_requests.append("fallback") + assert requests == expected_requests assert tokens == sum(expected_usage) assert [entry["usage"]["total_tokens"] for entry in ledger] == expected_usage usage_events = [event for event in events if event.type == "usage"] assert [event.data["total_tokens"] for event in usage_events] == expected_usage assert {event.data["purpose"] for event in usage_events} == {expected_purpose} + if failed_primary: + assert [entry["status"] for entry in ledger] == ["error", "success"] + assert [event.data["error_type"] for event in usage_events] == ["ResponsesTransientEventError", None] From 5282ba0aba3ed31236ca73cf801fe77052bd7bc0 Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 09:41:04 +0800 Subject: [PATCH 67/82] fix(llm): preserve usage identity through response copies Deduplicate copied provider usage after middleware redacts raw telemetry. Keep the native request identity out of exported usage fields. --- opencollab/adapters/llm/types.py | 13 +++++++++++ opencollab/application/_session_run_usage.py | 20 +++++++++++++++++ tests/runtime/test_routed_completion_usage.py | 22 ++++++++++++++----- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/opencollab/adapters/llm/types.py b/opencollab/adapters/llm/types.py index 96ef7033..2d26bcf7 100644 --- a/opencollab/adapters/llm/types.py +++ b/opencollab/adapters/llm/types.py @@ -13,6 +13,13 @@ ) +class _NativeUsageOrigin: + __slots__ = () + + def __deepcopy__(self, _memo: dict[int, Any]) -> _NativeUsageOrigin: + return self + + @dataclass class Usage: """Token accounting for one completion. @@ -23,6 +30,9 @@ class Usage: added again to compute the budget total. """ + # Retain request identity through copies without adding it to usage telemetry. + __slots__ = ("__dict__", "__weakref__", "_native_usage_origin") + input_tokens: int = 0 output_tokens: int = 0 cache_read_tokens: int | None = 0 @@ -39,6 +49,9 @@ class Usage: context_output_tokens: int | None = None context_tokens_estimated: bool | None = None + def __post_init__(self) -> None: + self._native_usage_origin = _NativeUsageOrigin() + @property def total_tokens(self) -> int: return self.input_tokens + self.output_tokens diff --git a/opencollab/application/_session_run_usage.py b/opencollab/application/_session_run_usage.py index f069b5ce..11fd8240 100644 --- a/opencollab/application/_session_run_usage.py +++ b/opencollab/application/_session_run_usage.py @@ -7,6 +7,7 @@ from collections.abc import Iterator from contextlib import contextmanager from contextvars import ContextVar +from dataclasses import fields from typing import Any, Awaitable, Callable _native_usage_owner: ContextVar[object | None] = ContextVar("native_usage_owner", default=None) @@ -57,6 +58,14 @@ def _normalize_completion_usage(usage: Any) -> tuple[int, int]: return input_tokens if context_tokens is None else context_tokens, max(reported_total, input_tokens + output_tokens) +def _same_usage_measurements(native: Any, usage: Any) -> bool: + # dataclasses.replace can rebuild usage while redacting only provider telemetry. + return type(native) is type(usage) and all( + getattr(native, field.name) == getattr(usage, field.name) + for field in fields(usage) if field.name != "raw_usage" + ) + + class _CompletionUsageAccounting: def __init__(self, runner: Any, *, protected_call: bool) -> None: self.runner = runner @@ -101,8 +110,19 @@ def finish(self, usage: Any, *, purpose: str, late: bool, error: BaseException | return False # Middleware may copy the chosen response or return a different native call with equal counters. native = next((item for item in self._native_usages if item[0] is usage), None) + origin = getattr(usage, "_native_usage_origin", None) + if native is None and origin is not None: + native = next( + (item for item in self._native_usages if getattr(item[0], "_native_usage_origin", None) is origin), + None, + ) if usage is not None and native is None: native = next((item for item in reversed(self._native_usages) if item[0] == usage), None) + if native is None and origin is not None: + native = next( + (item for item in reversed(self._native_usages) if _same_usage_measurements(item[0], usage)), + None, + ) native_usage = usage if native is None else native[0] if usage is not None and all(native_usage is not settled for settled, _tokens in self._settled): attempt_tokens = self._attempt_tokens.pop(None, 0) if native is None else 0 diff --git a/tests/runtime/test_routed_completion_usage.py b/tests/runtime/test_routed_completion_usage.py index 75d452bb..04786cf2 100644 --- a/tests/runtime/test_routed_completion_usage.py +++ b/tests/runtime/test_routed_completion_usage.py @@ -2,6 +2,7 @@ import json from copy import deepcopy +from dataclasses import replace import pytest @@ -20,7 +21,8 @@ @pytest.mark.parametrize("summary", [False, True]) @pytest.mark.parametrize("forward_fallback", [False, True]) @pytest.mark.parametrize("scenario", [ - "primary", "copied-primary", "fallback", "equal-fallback", "copied-fallback", + "primary", "copied-primary", "scrubbed-primary", "replaced-primary", + "fallback", "equal-fallback", "copied-fallback", "scrubbed-fallback", "replaced-fallback", "failed-primary", "failed-primary-attempt-callback-only", ]) async def test_routed_completion_accounts_for_each_provider( @@ -29,8 +31,8 @@ async def test_routed_completion_accounts_for_each_provider( requests = [] events = [] answer = "Answer." if summary else "Answer." - usable_primary = scenario in {"primary", "copied-primary"} - equal_usage = scenario in {"equal-fallback", "copied-fallback"} + usable_primary = scenario in {"primary", "copied-primary", "scrubbed-primary", "replaced-primary"} + equal_usage = scenario in {"equal-fallback", "copied-fallback", "scrubbed-fallback", "replaced-fallback"} failed_primary = scenario.startswith("failed-primary") monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") @@ -48,6 +50,14 @@ async def handle(request): return completion_http_response(request, text=answer if usable_primary else "", output_tokens=55) return completion_http_response(request, text=answer, output_tokens=55 if equal_usage else 2) + def copy_response(response): + if scenario.startswith("replaced-"): + return replace(response, usage=replace(response.usage, raw_usage={})) + copied = deepcopy(response) + if scenario.startswith("scrubbed-"): + copied.usage.raw_usage.clear() + return copied + class FallbackClient(LLMClient): async def complete( self, messages, *, on_admitted=None, before_request=None, @@ -64,9 +74,11 @@ async def complete( assert failed_primary else: if first.content and first.content.strip(): - return deepcopy(first) if scenario == "copied-primary" else first + return copy_response(first) if scenario != "primary" else first final = await fallback.complete(messages, **kwargs, **(controls if forward_fallback else {})) - return deepcopy(final) if failed_primary or scenario == "copied-fallback" else final + if failed_primary or scenario in {"copied-fallback", "scrubbed-fallback", "replaced-fallback"}: + return copy_response(final) + return final install_sdk_transport(monkeypatch, "openai", handle) options = dict(api_key=__name__, context_window=40_000, request_pool=str(tmp_path), request_capacity=1) From 9279aebf0ec786065b983866e47d1688d15365b5 Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 10:06:02 +0800 Subject: [PATCH 68/82] fix(session): preserve late provider budget settlement Route late completion notifications through the shared recorder while keeping protected budget accounting intact. --- .../application/_session_run_completion.py | 8 +++----- opencollab/application/session_run.py | 19 ++++++++++++++++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index 35bee711..2e1d98f0 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -1008,12 +1008,10 @@ async def complete_owned() -> CompletionResponse: self._finish_run_control_if_quiesced() def record_abandoned(task: asyncio.Task[Any]) -> None: - try: - if not late_accounted and not task.cancelled() and task.exception() is None: - usage_accounting.record_late() - finally: - self._draining_provider_tasks.discard(task) + if late_accounted: self._provider_task_done(task) + else: + self._record_late_provider_result(task, accounting=usage_accounting) operation = complete_owned() if scope is not None or _native_completion_controls.get() is not None: diff --git a/opencollab/application/session_run.py b/opencollab/application/session_run.py index affddb20..b9d54014 100644 --- a/opencollab/application/session_run.py +++ b/opencollab/application/session_run.py @@ -30,7 +30,7 @@ _TeamBudgetStop, _TokenBudgetStop, ) -from opencollab.application._session_run_usage import _normalize_completion_usage +from opencollab.application._session_run_usage import _CompletionUsageAccounting, _normalize_completion_usage from opencollab.application.context_pressure import ContextPressure from opencollab.application.events import SessionEventFactory, default_session_event_factory from opencollab.application.ports import ( @@ -251,6 +251,23 @@ def _charge_completion_tokens(self, tokens: int) -> None: if tokens and self._on_completion_tokens is not None: self._on_completion_tokens(tokens) + def _record_late_provider_result( + self, task: asyncio.Future[Any], *, protected_call: bool = False, + accounting: _CompletionUsageAccounting | None = None, + ) -> None: + try: + response = task.result() + if accounting is None: + accounting = _CompletionUsageAccounting(self, protected_call=protected_call) + accounting.finish(response.usage, purpose="completion", late=True) + else: + accounting.record_late() + except BaseException: + pass + finally: + self._draining_provider_tasks.discard(task) + self._finish_run_control_if_quiesced() + async def run_loop(self, cancel_event: asyncio.Event | None = None) -> str: """Drive the phase FSM until the turn finishes or suspends. From 16dfde1b5ec88b36051ae5021df3c23f753d751a Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 23:28:09 +0800 Subject: [PATCH 69/82] fix(llm): preserve accounting across context annotations --- opencollab/adapters/llm/client.py | 4 +- .../application/_session_run_completion.py | 10 +++- opencollab/application/_session_run_usage.py | 47 ++++++++++++++----- tests/runtime/test_routed_completion_usage.py | 35 ++++++++++++-- 4 files changed, 76 insertions(+), 20 deletions(-) diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index 48cdd667..1fcc1be2 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -461,7 +461,9 @@ def prepare_attempt() -> int | None: try: if on_usage is not None: on_usage(settled_usage) - _observe_native_usage(settled_usage, provider_error) + _observe_native_usage( + settled_usage, provider_error, transport_timing if response is not None else None, + ) except BaseException as exc: exc.usage = settled_usage settlement_error = exc diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index 2e1d98f0..ca2e7f2a 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -925,7 +925,10 @@ def revalidate() -> int: def account_response(response: CompletionResponse) -> None: input_tokens, _total_tokens = _normalize_completion_usage(response.usage) - usage_accounting.finish(response.usage, purpose="summary", late=abandoned) + usage_accounting.finish( + response.usage, purpose="summary", late=abandoned, + transport_timing=getattr(response, "transport_timing", None), + ) self.state.add_markup_recovered(getattr(response.usage, "markup_recovered", 0)) self.state.set_context_tokens(input_tokens) self.record_llm_trace(response, time.monotonic() - start, purpose="summary", thinking=thinking) @@ -1001,7 +1004,10 @@ async def complete_owned() -> CompletionResponse: self, complete, usage_accounting=usage_accounting, **kwargs, ) late_accounted = asyncio.current_task() in self._draining_provider_tasks - usage_accounting.finish(response.usage, purpose="completion", late=late_accounted) + usage_accounting.finish( + response.usage, purpose="completion", late=late_accounted, + transport_timing=getattr(response, "transport_timing", None), + ) return response finally: self._draining_provider_tasks.discard(asyncio.current_task()) diff --git a/opencollab/application/_session_run_usage.py b/opencollab/application/_session_run_usage.py index 11fd8240..b24ffeff 100644 --- a/opencollab/application/_session_run_usage.py +++ b/opencollab/application/_session_run_usage.py @@ -7,11 +7,10 @@ from collections.abc import Iterator from contextlib import contextmanager from contextvars import ContextVar -from dataclasses import fields from typing import Any, Awaitable, Callable _native_usage_owner: ContextVar[object | None] = ContextVar("native_usage_owner", default=None) -_native_usage_observer: ContextVar[Callable[[Any, BaseException | None], None] | None] = ContextVar( +_native_usage_observer: ContextVar[Callable[[Any, BaseException | None, object | None], None] | None] = ContextVar( "native_usage_observer", default=None, ) @@ -25,10 +24,10 @@ def _own_native_usage() -> Iterator[None]: _native_usage_owner.reset(token) -def _observe_native_usage(usage: Any, error: BaseException | None) -> None: +def _observe_native_usage(usage: Any, error: BaseException | None, transport_timing: object | None) -> None: observer = _native_usage_observer.get() if observer is not None: - observer(usage, error) + observer(usage, error, transport_timing) def _nonnegative_usage_int(value: Any, field: str) -> int: @@ -59,10 +58,13 @@ def _normalize_completion_usage(usage: Any) -> tuple[int, int]: def _same_usage_measurements(native: Any, usage: Any) -> bool: - # dataclasses.replace can rebuild usage while redacting only provider telemetry. + # Context estimates and tool-markup annotations do not change provider usage. return type(native) is type(usage) and all( - getattr(native, field.name) == getattr(usage, field.name) - for field in fields(usage) if field.name != "raw_usage" + getattr(native, name) == getattr(usage, name) + for name in ( + "input_tokens", "output_tokens", "total_tokens", + "cache_read_tokens", "cache_creation_tokens", "reasoning_tokens", + ) ) @@ -72,7 +74,7 @@ def __init__(self, runner: Any, *, protected_call: bool) -> None: self.protected_call = protected_call self.tokens = 0 self._attempt_tokens: dict[object | None, int] = {} - self._native_usages: list[tuple[Any, BaseException | None]] = [] + self._native_usages: list[tuple[Any, BaseException | None, object | None]] = [] self._settled: list[tuple[Any, int]] = [] self._reported = False @@ -93,8 +95,10 @@ def record_attempt(self, usage: Any) -> None: def settle_provider(self, usage: Any) -> None: self._settle(usage, self._attempt_tokens.pop(_native_usage_owner.get(), 0)) - def _observe_provider_response(self, usage: Any, error: BaseException | None) -> None: - self._native_usages.append((usage, error)) + def _observe_provider_response( + self, usage: Any, error: BaseException | None, transport_timing: object | None, + ) -> None: + self._native_usages.append((usage, error, transport_timing)) owner = _native_usage_owner.get() # A router can forward the retry hook and keep the final callback for its chosen response. if owner is not None and owner in self._attempt_tokens: @@ -105,7 +109,10 @@ def _settle(self, usage: Any, attempt_tokens: int) -> None: self._charge(tokens - attempt_tokens) self._settled.append((usage, tokens)) - def finish(self, usage: Any, *, purpose: str, late: bool, error: BaseException | None = None) -> bool: + def finish( + self, usage: Any, *, purpose: str, late: bool, error: BaseException | None = None, + transport_timing: object | None = None, + ) -> bool: if self._reported: return False # Middleware may copy the chosen response or return a different native call with equal counters. @@ -116,6 +123,22 @@ def finish(self, usage: Any, *, purpose: str, late: bool, error: BaseException | (item for item in self._native_usages if getattr(item[0], "_native_usage_origin", None) is origin), None, ) + if native is None and transport_timing is not None: + # The native request's timing object survives replacements of the response and usage. + native = next( + (item for item in self._native_usages + if item[2] is transport_timing and _same_usage_measurements(item[0], usage)), + None, + ) + if native is None and origin is not None: + raw_usage = getattr(usage, "raw_usage", None) + if raw_usage is not None: + # A dataclass replacement normally retains the original provider payload. + native = next( + (item for item in self._native_usages + if getattr(item[0], "raw_usage", None) is raw_usage and _same_usage_measurements(item[0], usage)), + None, + ) if usage is not None and native is None: native = next((item for item in reversed(self._native_usages) if item[0] == usage), None) if native is None and origin is not None: @@ -131,7 +154,7 @@ def finish(self, usage: Any, *, purpose: str, late: bool, error: BaseException | record_usage = getattr(self.runner, "_record_usage_event", None) if callable(record_usage): for settled, tokens in self._settled: - provider_error = next((failure for item, failure in self._native_usages if item is settled), None) + provider_error = next((failure for item, failure, _ in self._native_usages if item is settled), None) record_usage(settled, tokens, purpose=purpose, late=late, error=error if settled is native_usage and error is not None else provider_error) if late: diff --git a/tests/runtime/test_routed_completion_usage.py b/tests/runtime/test_routed_completion_usage.py index 04786cf2..2065e38c 100644 --- a/tests/runtime/test_routed_completion_usage.py +++ b/tests/runtime/test_routed_completion_usage.py @@ -23,6 +23,8 @@ @pytest.mark.parametrize("scenario", [ "primary", "copied-primary", "scrubbed-primary", "replaced-primary", "fallback", "equal-fallback", "copied-fallback", "scrubbed-fallback", "replaced-fallback", + "annotated-primary", "annotated-fallback", "annotated-primary-after-helper", + "annotated-primary-after-helper-without-timing", "failed-primary", "failed-primary-attempt-callback-only", ]) async def test_routed_completion_accounts_for_each_provider( @@ -31,8 +33,14 @@ async def test_routed_completion_accounts_for_each_provider( requests = [] events = [] answer = "Answer." if summary else "Answer." - usable_primary = scenario in {"primary", "copied-primary", "scrubbed-primary", "replaced-primary"} - equal_usage = scenario in {"equal-fallback", "copied-fallback", "scrubbed-fallback", "replaced-fallback"} + independent_helper = scenario.startswith("annotated-primary-after-helper") + usable_primary = independent_helper or scenario in { + "primary", "copied-primary", "scrubbed-primary", "replaced-primary", + "annotated-primary", + } + equal_usage = independent_helper or scenario in { + "equal-fallback", "copied-fallback", "scrubbed-fallback", "replaced-fallback", "annotated-fallback", + } failed_primary = scenario.startswith("failed-primary") monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") @@ -53,6 +61,17 @@ async def handle(request): def copy_response(response): if scenario.startswith("replaced-"): return replace(response, usage=replace(response.usage, raw_usage={})) + if scenario.startswith("annotated-"): + removes_timing = scenario == "annotated-primary-after-helper-without-timing" + usage = replace( + response.usage, + context_tokens=7, + context_output_tokens=5, + context_tokens_estimated=True, + raw_usage=response.usage.raw_usage if removes_timing else {}, + ) + timing = None if removes_timing else response.transport_timing + return replace(response, usage=usage, transport_timing=timing) copied = deepcopy(response) if scenario.startswith("scrubbed-"): copied.usage.raw_usage.clear() @@ -74,9 +93,13 @@ async def complete( assert failed_primary else: if first.content and first.content.strip(): + if independent_helper: + await fallback.complete(messages, **kwargs) return copy_response(first) if scenario != "primary" else first final = await fallback.complete(messages, **kwargs, **(controls if forward_fallback else {})) - if failed_primary or scenario in {"copied-fallback", "scrubbed-fallback", "replaced-fallback"}: + if failed_primary or scenario in { + "copied-fallback", "scrubbed-fallback", "replaced-fallback", "annotated-fallback", + }: return copy_response(final) return final @@ -102,6 +125,8 @@ async def complete( text = await session._context_summarizer.asummarize(session.messages) assert text == "Summary:\nAnswer." assert session._context_summarizer.last_call_cacheable + if scenario.startswith("annotated-"): + assert session.state.context_tokens == 7 tokens = session.used_tokens finally: await session.aclose() @@ -121,11 +146,11 @@ async def complete( expected_purpose = "summary" if summary else "completion" ledger = [json.loads(line) for line in (tmp_path / "usage.jsonl").read_text().splitlines()] expected_requests = ["primary", "primary"] if failed_primary else ["primary"] - if not usable_primary: + if not usable_primary or independent_helper: expected_requests.append("fallback") assert requests == expected_requests assert tokens == sum(expected_usage) - assert [entry["usage"]["total_tokens"] for entry in ledger] == expected_usage + assert [entry["usage"]["total_tokens"] for entry in ledger] == expected_usage + ([57] if independent_helper else []) usage_events = [event for event in events if event.type == "usage"] assert [event.data["total_tokens"] for event in usage_events] == expected_usage assert {event.data["purpose"] for event in usage_events} == {expected_purpose} From 5b7e4317beff785248d87f3d14733812f4dd9f33 Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 23:32:30 +0800 Subject: [PATCH 70/82] fix(session): retain compacted tool history across transcript edits --- opencollab/application/_anthropic_history.py | 90 +++++++++++- opencollab/application/session_snapshot.py | 5 +- .../test_anthropic_preserved_history.py | 131 ++++++++++++++++++ 3 files changed, 221 insertions(+), 5 deletions(-) diff --git a/opencollab/application/_anthropic_history.py b/opencollab/application/_anthropic_history.py index 5c99cae5..21bec2bd 100644 --- a/opencollab/application/_anthropic_history.py +++ b/opencollab/application/_anthropic_history.py @@ -3,8 +3,10 @@ from __future__ import annotations import copy +from collections import Counter from typing import Any +from opencollab.application.shaping.pipeline import _group_spans, is_complete_tool_exchange from opencollab.domain.session import SessionState, same_snapshot_value _THINKING_TYPES = frozenset({"thinking", "redacted_thinking"}) @@ -39,13 +41,97 @@ def _anthropic_source_changed(state: SessionState) -> bool: source = history.get("source_messages") if history is not None else None return source is not None and not same_snapshot_value(state.messages[:len(source)], source) +def _anchor_key(message: dict) -> tuple[str | None, str | None]: + role = message.get("role") + content = message.get("tool_call_id", message.get("content")) + if isinstance(content, list) and content and isinstance(content[0], dict): + content = content[0].get("text") + return ( + str.__str__(role) if isinstance(role, str) else None, + str.__str__(content) if isinstance(content, str) else None, + ) + +def _projection_anchor( + source: list[dict], view: list[dict], index: int, counts: Counter[tuple[str | None, str | None]], +) -> int | None: + offset = len(view) - len(source) + if offset and counts[_anchor_key(source[index])] != 1: + return None + candidates = (index, index + offset) if offset else (index,) + matches = [ + candidate for candidate in candidates + if 0 <= candidate < len(view) and same_snapshot_value(source[index], view[candidate]) + ] + return matches[0] if len(matches) == 1 else None + +def _rebuild_edited_span(source: list[dict], view: list[dict], messages: list[dict]) -> list[dict]: + start = 0 + while start < min(len(source), len(messages)) and same_snapshot_value(source[start], messages[start]): + start += 1 + + # Exact anchors keep independent compaction intact. A summary or transformed + # message between the edit and its nearest anchors may depend on the edit. + counts = Counter(map(_anchor_key, source)) if len(view) != len(source) else Counter() + view_start = source_start = 0 + for index in range(min(start, len(source) - 1), -1, -1): + anchor = _projection_anchor(source, view, index, counts) + if anchor is not None: + keep = int(index < start) + view_start, source_start = anchor + keep, index + keep + break + + if len(messages) >= len(source): + stop = len(source) + while stop > start and same_snapshot_value(source[stop - 1], messages[stop - 1]): + stop -= 1 + for index in range(stop - 1, len(source)): + anchor = _projection_anchor(source, view, index, counts) + if anchor is not None: + skip = int(index < stop) + view_stop, source_stop = anchor + skip, index + skip + if view_stop >= view_start and source_stop >= source_start: + return [ + *view[:view_start], *messages[source_start:source_stop], + *view[view_stop:], *messages[len(source):], + ] + return [*view[:view_start], *messages[source_start:]] + +def _tool_exchange_spans(messages: list[dict]) -> dict[tuple[str, ...], tuple[int, int] | None]: + exchanges: dict[tuple[str, ...], tuple[int, int] | None] = {} + for start, stop in _group_spans(messages): + calls = messages[start].get("tool_calls") + if not isinstance(calls, list) or not all(isinstance(call, dict) for call in calls): + continue + if is_complete_tool_exchange(messages, (start, stop)): + call_ids = tuple(call["id"] for call in calls) + exchanges[call_ids] = None if call_ids in exchanges else (start, stop) + return exchanges + +def _rebase_anthropic_history(history: dict, messages: list[dict]) -> list[dict]: + source = history["source_messages"] + view = history["messages"] + rebased = _rebuild_edited_span(source, view, messages) + projections = _tool_exchange_spans(view) + if projections: + previous = _tool_exchange_spans(source) + current = _tool_exchange_spans(messages) + for call_ids, target in _tool_exchange_spans(rebased).items(): + old = previous.get(call_ids) + new = current.get(call_ids) + projection = projections.get(call_ids) + if target is None or old is None or new is None or projection is None: + continue + if same_snapshot_value(source[old[0]:old[1]], messages[new[0]:new[1]]): + # Clear and trim shapers preserve the exchange while shortening its results. + rebased[target[0]:target[1]] = view[projection[0]:projection[1]] + return _edited_anthropic_history(view, rebased) + def _anthropic_history_input(state: SessionState) -> list[dict]: - """Reuse the native view until its original transcript prefix changes.""" history = state.anthropic_history if history is None: return state.messages if _anthropic_source_changed(state): - return _edited_anthropic_history(history["source_messages"], state.messages) + return copy.deepcopy(_rebase_anthropic_history(history, state.messages)) return copy.deepcopy([*history["messages"], *state.messages[history["source_count"]:]]) def _remember_anthropic_history( diff --git a/opencollab/application/session_snapshot.py b/opencollab/application/session_snapshot.py index 24a77ea4..ef211b43 100644 --- a/opencollab/application/session_snapshot.py +++ b/opencollab/application/session_snapshot.py @@ -6,7 +6,7 @@ import math from typing import Any -from opencollab.application._anthropic_history import _anthropic_source_changed, _edited_anthropic_history +from opencollab.application._anthropic_history import _anthropic_source_changed, _rebase_anthropic_history from opencollab.domain.pending import PendingRow, RowKind, RowStatus from opencollab.domain.session import SessionPhase, SessionState, same_snapshot_value @@ -25,8 +25,7 @@ def _serialize_anthropic_history(state: SessionState) -> dict | None: view = history["messages"] if _anthropic_source_changed(state): # Retire the stale projection before restore binds it to the new transcript. - view = _edited_anthropic_history(history["source_messages"], source) - view = _edited_anthropic_history(history["messages"], view) + view = _rebase_anthropic_history(history, source) count = len(source) # Aligned and suffix-aligned entries can reuse the persisted transcript. diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index d23907df..fa7822b6 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -284,6 +284,101 @@ def shape(self, messages): await environment.cleanup() +@pytest.mark.parametrize("moment", ["live", "before-export", "after-restore"]) +@pytest.mark.parametrize("target", ["recent-user", "first-user", "both-users", "insert", "delete", "cleared-result"]) +async def test_source_edits_keep_unrelated_native_tool_compaction(monkeypatch, tmp_path, moment, target): + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + marker = "Catalogue entry from the old archive." + clear_marker = "[Old tool result content cleared]" + archive = tmp_path / "archive.txt" + archive.write_text(f"{marker}\n" * 400) + peer = _BoundThinkingProvider(reads=2, thinking=False) + context_limit = 8_000 + resumed_requests = [] + resuming = False + + def transport(request): + if resuming: + resumed_requests.append(json.loads(request.content)) + if len(request.content) > context_limit: + return http_response(request, 400, json={"type": "error", "error": { + "type": "invalid_request_error", "message": "prompt is too long for the context window", + }}) + peer.read_input = {"path": archive.name, "limit": 400} if not peer.requests else {} + return peer(request) + + install_sdk_transport(monkeypatch, "anthropic", transport) + first_user = "Inspect the archive" + recent_user = "Use a formal tone" + first_edit = (first_user, "Review the archive entries") + recent_edit = (recent_user, "Use an informal tone") + inserted = "Additional context from the editor" + edits = { + "recent-user": [recent_edit], + "first-user": [first_edit], + "both-users": [first_edit, recent_edit], + "insert": [(None, inserted)], + "delete": [(first_user, None)], + "cleared-result": [(marker, "The entries in the old archive have been revised.")], + }[target] + + def edit(session): + if target == "cleared-result": + result = next(message for message in session.messages if message["role"] == "tool") + assert marker in result["content"] + result["content"] = edits[0][1] + elif target == "insert": + session.messages.insert(1, {"role": "user", "content": inserted}) + elif target == "delete": + del session.messages[1] + else: + for original, revised in edits: + message = next(message for message in session.messages if message.get("content") == original) + message["content"] = revised + + model = "claude-sonnet-5-5" + env = LocalEnvironment(str(tmp_path)) + path = str(tmp_path / "compacted.json") + async with _client(model) as client: + options = {"agent": _agent(thinking=False, model=model), "llm": client, "env": env} + session = build_session( + **options, shaper=ToolOutputClearShaper(trigger_tokens=2, target_tokens=1, keep_recent=1), + ) + try: + await session.add_user_message(first_user) + assert await session.run_loop() == "done" + await session.add_user_message(recent_user) + assert await session.run_loop() == "done" + assert marker in json.dumps(peer.requests[1]["messages"]) + assert clear_marker in json.dumps(peer.requests[-1]["messages"]) + assert marker in json.dumps(session.messages) + if moment != "after-restore": + edit(session) + if moment != "live": + session.save(path) + await session.aclose() + session = load_session(path, **options) + if moment == "after-restore": + edit(session) + session.runner.shaper = None + + await session.add_user_message("Summarize the evidence") + resuming = True + assert await session.run_loop() == "done" + assert len(resumed_requests) == 1 + continued = json.dumps(resumed_requests[0]["messages"]) + for original, revised in edits: + if original is not None: + assert original not in continued + if revised is not None: + assert revised in continued + assert marker not in continued + assert (clear_marker in continued) is (target != "cleared-result") + finally: + await session.aclose() + await env.cleanup() + + async def test_appended_system_and_changed_tools_retire_old_native_thinking(monkeypatch, tmp_path): monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") peer = _BoundThinkingProvider(reads=0) @@ -519,6 +614,42 @@ def test_restored_unchanged_prefix_preserves_projection_and_tracks_future_nested assert state.anthropic_history["source_messages"][1]["content"][0]["text"] == "Old requirement" +def test_edit_in_summary_invalidates_it_even_when_a_later_message_is_identical(): + state = SessionState(messages=[ + {"role": "system", "content": "System instructions"}, + {"role": "user", "content": "First request"}, + {"role": "assistant", "content": "Acknowledged"}, + {"role": "user", "content": "Second request"}, + {"role": "assistant", "content": "Acknowledged"}, + ]) + summary = {"role": "system", "compacted": True, "content": "The first request was acknowledged"} + _remember_anthropic_history(state, [state.messages[0], summary, state.messages[4]], None) + state.messages[2]["content"] = "The first request still needs a decision" + assert _anthropic_history_input(state) == state.messages + + +def test_disjoint_edits_do_not_reuse_a_projection_for_reused_tool_call_ids(): + call = { + "role": "assistant", "content": None, "tool_calls": [{ + "id": "read-again", "type": "function", + "function": {"name": "file_read", "arguments": '{"path":"archive.txt"}'}, + }], + } + state = SessionState(messages=[ + {"role": "user", "content": "Read the old archive"}, + call, {"role": "tool", "tool_call_id": "read-again", "content": "Old file data"}, + {"role": "user", "content": "Check the archive again"}, + copy.deepcopy(call), {"role": "tool", "tool_call_id": "read-again", "content": "Revised file data"}, + {"role": "user", "content": "Summarize what changed"}, + ]) + view = copy.deepcopy(state.messages) + view[2]["content"] = "[Old tool result content cleared]" + _remember_anthropic_history(state, view, None) + state.messages[0]["content"] = "Read the archive entries" + state.messages[-1]["content"] = "Summarize the latest entries" + assert _anthropic_history_input(state) == state.messages + + @pytest.mark.parametrize("checkpoint", [False, True], ids=["journal", "checkpoint"]) async def test_first_tool_step_autosaves_an_edit_already_used_in_the_request(monkeypatch, tmp_path, checkpoint): monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") From 334eacc4bb7b45bb87c876f59d2574283a1ee66e Mon Sep 17 00:00:00 2001 From: Kai Date: Fri, 20 Nov 2026 23:38:24 +0800 Subject: [PATCH 71/82] fix(workflow): preserve bulk write locks under descriptor limits --- .../adapters/tools/_file_lock_worker.py | 56 +++++++ opencollab/adapters/tools/base.py | 124 +++++++++++++-- tests/tools/test_local_file_lock_workers.py | 147 ++++++++++++++++++ .../test_candidate_adoption_command_size.py | 34 +++- 4 files changed, 345 insertions(+), 16 deletions(-) create mode 100644 opencollab/adapters/tools/_file_lock_worker.py create mode 100644 tests/tools/test_local_file_lock_workers.py diff --git a/opencollab/adapters/tools/_file_lock_worker.py b/opencollab/adapters/tools/_file_lock_worker.py new file mode 100644 index 00000000..5e8643cd --- /dev/null +++ b/opencollab/adapters/tools/_file_lock_worker.py @@ -0,0 +1,56 @@ +"""Hold one ordered group of existing local file locks for a parent process.""" + +import contextlib +import json +import select +import sys +import time + +from filelock import FileLock, Timeout + + +def _parent_closed(wait: float | None = 0) -> bool: + # The parent sends one setup message, then keeps its pipe open as ownership. + readable, _, _ = select.select([sys.stdin.buffer], [], [], wait) + return bool(readable) + + +def _reply(status: str, detail: str = "") -> None: + print(json.dumps((status, detail)), flush=True) + + +def main() -> None: + setup = sys.stdin.buffer.readline() + if not setup: + return + paths, timeout, poll = json.loads(setup) + try: + with contextlib.ExitStack() as held: + for path, lock_path in paths: + lock = FileLock(lock_path, timeout=0) + deadline = time.monotonic() + timeout + while not _parent_closed(): + try: + lock.acquire(timeout=0) + except Timeout: + remaining = deadline - time.monotonic() + if remaining <= 0: + _reply("timeout", f"timed out acquiring write lock for {path}") + return + if _parent_closed(min(poll, remaining)): + return + else: + held.callback(lock.release) + break + else: + return + _reply("ready") + _parent_closed(None) + except BrokenPipeError: + return + except OSError as exc: + _reply("error", str(exc)) + + +if __name__ == "__main__": + main() diff --git a/opencollab/adapters/tools/base.py b/opencollab/adapters/tools/base.py index a4d2651e..2b31da5f 100644 --- a/opencollab/adapters/tools/base.py +++ b/opencollab/adapters/tools/base.py @@ -4,15 +4,20 @@ import asyncio import contextlib +import errno import hashlib +import json import os import posixpath +import resource +import sys import tempfile from collections.abc import AsyncIterator, Sequence from typing import Any from filelock import FileLock, Timeout +from opencollab.application.async_timeout import await_owned_operation from opencollab.application.tool_execution import DeferredCall, ToolRuntime HOST_WRITE_LOCK_TIMEOUT_SECONDS = 10.0 @@ -51,19 +56,118 @@ def _write_lock_paths( return sorted(targets) +def _local_file_lock_capacity() -> int: + limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + if limit == resource.RLIM_INFINITY: + return sys.maxsize + # Leave descriptors for the environment's subprocesses and pending I/O. + return max(1, limit - max(16, limit // 4)) + + +def _available_local_file_locks(capacity: int) -> int: + try: + return capacity - len(os.listdir("/dev/fd")) + except OSError as exc: + return 0 if exc.errno == errno.EMFILE else capacity + + +async def _close_lock_holder(process: asyncio.subprocess.Process) -> None: + assert process.stdin is not None + process.stdin.close() + try: + await asyncio.wait_for(process.wait(), 2) + except asyncio.TimeoutError: + with contextlib.suppress(ProcessLookupError): + process.kill() + await process.wait() + + +@contextlib.asynccontextmanager +async def _hold_lock_group(paths: Sequence[str], env: Any) -> AsyncIterator[None]: + process = None + monitor = None + worker = os.path.join(os.path.dirname(__file__), "_file_lock_worker.py") + + async def spawn() -> None: + nonlocal process + process = await asyncio.create_subprocess_exec( + sys.executable, worker, + stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, + ) + + try: + await await_owned_operation(spawn(), propagate_cancellation=True) + assert process is not None and process.stdin is not None and process.stdout is not None + options = ( + [(path, _host_lock_path(path, env)) for path in paths], + HOST_WRITE_LOCK_TIMEOUT_SECONDS, HOST_WRITE_LOCK_POLL_SECONDS, + ) + process.stdin.write(json.dumps(options).encode() + b"\n") + await process.stdin.drain() + response = await process.stdout.readline() + if not response: + raise RuntimeError("local write lock holder exited during acquisition") + status, detail = json.loads(response) + if status == "timeout": + raise TimeoutError(detail) + if status != "ready": + raise RuntimeError(f"local write lock holder failed: {detail}") + owner = asyncio.current_task() + assert owner is not None + + async def watch_exit() -> None: + returncode = await process.wait() + owner.cancel(f"local write lock holder exited unexpectedly ({returncode})") + + monitor = asyncio.create_task(watch_exit()) + yield + finally: + if monitor is not None: + monitor.cancel() + if process is not None: + await await_owned_operation(_close_lock_holder(process), propagate_cancellation=True) + + +@contextlib.asynccontextmanager +async def _local_write_locks(paths: Sequence[str], env: Any) -> AsyncIterator[FileLock | None]: + capacity = _local_file_lock_capacity() + if len(paths) <= capacity and len(paths) <= _available_local_file_locks(capacity): + async with contextlib.AsyncExitStack() as locks: + target = None + try: + for path in paths: + target = await locks.enter_async_context(_target_write_lock(path, env)) + except OSError as exc: + if exc.errno != errno.EMFILE: + raise + else: + yield target + return + # Every holder uses the same per-path locks. Contention order stays the + # same across groups, direct callers and older installations. + group_size = min(512, capacity) + async with contextlib.AsyncExitStack() as locks: + for start in range(0, len(paths), group_size): + await locks.enter_async_context(_hold_lock_group(paths[start:start + group_size], env)) + yield None + + @contextlib.asynccontextmanager async def host_write_locks(paths: Sequence[str], env: Any) -> AsyncIterator[Any]: """Own edited paths and their ancestors through publication or recovery.""" - if env is not None and not getattr(env, "local_filesystem", False): - grouped = getattr(env, "file_write_locks", None) - if callable(grouped): - async with grouped(paths) as target: - yield target - return - if getattr(env, "file_write_lock_scope", None) == "workspace": - async with _target_write_lock(paths[0], env) as target: - yield target - return + if env is None or getattr(env, "local_filesystem", False): + async with _local_write_locks(_write_lock_paths(paths, env), env) as target: + yield target + return + grouped = getattr(env, "file_write_locks", None) + if callable(grouped): + async with grouped(paths) as target: + yield target + return + if getattr(env, "file_write_lock_scope", None) == "workspace": + async with _target_write_lock(paths[0], env) as target: + yield target + return async with contextlib.AsyncExitStack() as locks: target = None for path in _write_lock_paths(paths, env): diff --git a/tests/tools/test_local_file_lock_workers.py b/tests/tools/test_local_file_lock_workers.py new file mode 100644 index 00000000..10e815d3 --- /dev/null +++ b/tests/tools/test_local_file_lock_workers.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import asyncio +import sys + +import pytest +from filelock import FileLock, Timeout + +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.tools import base + + +async def test_large_group_acquires_existing_flocks_in_path_order(tmp_path, monkeypatch): + monkeypatch.setattr(base, "_local_file_lock_capacity", lambda: 2) + monkeypatch.setattr(base, "HOST_WRITE_LOCK_TIMEOUT_SECONDS", 0.2) + env = LocalEnvironment(str(tmp_path)) + first, blocked, last = ( + FileLock(base._host_lock_path(path, env), timeout=0) for path in ("a.txt", "b.txt", "c.txt") + ) + other = FileLock(base._host_lock_path("other.txt", env), timeout=0) + paths = ("c.txt", "a.txt", "b.txt") + try: + with blocked, pytest.raises(TimeoutError, match="b.txt"): + async with base.host_write_locks(paths, env): + pytest.fail("group acquired a lock already held by an existing writer") + with first, last: + pass + async with base.host_write_locks(paths, env): + for lock in (first, blocked, last): + with pytest.raises(Timeout): + lock.acquire() + with other: + pass + finally: + await env.cleanup() + + +async def test_cancelled_acquisition_releases_the_partial_group(tmp_path, monkeypatch): + monkeypatch.setattr(base, "_local_file_lock_capacity", lambda: 2) + env = LocalEnvironment(str(tmp_path)) + paths = ("a.txt", "b.txt", "c.txt") + first, blocked, last = (FileLock(base._host_lock_path(path, env), timeout=0) for path in paths) + + async def acquire_group(): + async with base.host_write_locks(paths, env): + pytest.fail("group acquired a lock already held by an existing writer") + + async def wait_until_first_is_held(): + while True: + try: + with first: + pass + except Timeout: + return + await asyncio.sleep(0.02) + + try: + with blocked: + owner = asyncio.create_task(acquire_group()) + try: + await asyncio.wait_for(wait_until_first_is_held(), 5) + with last: + pass + finally: + owner.cancel() + with pytest.raises(asyncio.CancelledError): + await owner + with first, last: + pass + finally: + await env.cleanup() + + +_OWNER = """ +import asyncio +import sys + +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.tools import base + + +async def main(): + base._local_file_lock_capacity = lambda: 2 + env = LocalEnvironment(sys.argv[1]) + entered = asyncio.Event() + + async def hold(): + async with base.host_write_locks(sys.argv[2:], env): + entered.set() + await asyncio.Event().wait() + + task = asyncio.create_task(hold()) + try: + await asyncio.wait_for(entered.wait(), 5) + print('ready', flush=True) + await asyncio.to_thread(sys.stdin.readline) + task.cancel() + try: + await task + except asyncio.CancelledError: + print('cancelled', flush=True) + await asyncio.to_thread(sys.stdin.readline) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await env.cleanup() + + +asyncio.run(main()) +""" + + +@pytest.mark.parametrize("crash", [False, True]) +async def test_large_group_releases_existing_flocks_after_owner_stops(tmp_path, crash): + paths = ("a.txt", "b.txt", "c.txt") + env = LocalEnvironment(str(tmp_path)) + old_locks = [FileLock(base._host_lock_path(path, env), timeout=2) for path in paths] + owner = await asyncio.create_subprocess_exec( + sys.executable, "-c", _OWNER, str(tmp_path), *paths, + stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, + ) + + def confirm_released(): + for lock in old_locks: + with lock: + pass + + try: + assert await asyncio.wait_for(owner.stdout.readline(), 10) == b"ready\n" + for lock in old_locks: + with pytest.raises(Timeout): + lock.acquire(timeout=0) + if crash: + owner.kill() + await owner.wait() + else: + owner.stdin.write(b"cancel\n") + await owner.stdin.drain() + assert await asyncio.wait_for(owner.stdout.readline(), 10) == b"cancelled\n" + assert owner.returncode is None + await asyncio.to_thread(confirm_released) + finally: + owner.stdin.close() + if owner.returncode is None: + owner.kill() + await owner.wait() + await env.cleanup() diff --git a/tests/workflows/test_candidate_adoption_command_size.py b/tests/workflows/test_candidate_adoption_command_size.py index b3a28863..c8f10584 100644 --- a/tests/workflows/test_candidate_adoption_command_size.py +++ b/tests/workflows/test_candidate_adoption_command_size.py @@ -1,6 +1,9 @@ from __future__ import annotations import errno +import os +import resource +from contextlib import ExitStack from pathlib import Path import pytest @@ -18,14 +21,30 @@ async def exec_cmd(self, command, timeout=120.0, **kwargs): return await super().exec_cmd(command, timeout, **kwargs) +@pytest.fixture +def bounded_file_descriptors(): + previous = soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + limit = 256 if soft == resource.RLIM_INFINITY else min(soft, 256) + resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) + try: + yield limit + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, previous) + + @pytest.mark.parametrize("blocked", [False, True]) -async def test_bulk_candidate_adoption_keeps_recovery_below_argument_limits(tmp_path, blocked): +@pytest.mark.parametrize("file_count,already_open", [(900, 0), (120, 120)], ids=["bulk", "busy-process"]) +async def test_bulk_candidate_adoption_keeps_recovery_below_process_limits( + tmp_path, blocked, bounded_file_descriptors, file_count, already_open, +): + if already_open and bounded_file_descriptors < 256: + pytest.skip("the busy process case uses a 256 descriptor limit") # Long absolute paths expand the recovery script past the temp-file limit # even though the repository-relative patch fits under one megabyte. repo = tmp_path.joinpath(*("nested-" + "workspace-" * 8,) * 5, "repo") (repo / "src").mkdir(parents=True) (repo / "settings").mkdir() - names = [f"{number:04d}_" + "module_" * 10 + ".py" for number in range(900)] + names = [f"{number:04d}_" + "module_" * 10 + ".py" for number in range(file_count)] for name in names: (repo / "src" / name).write_text("VALUE = 1\n") (repo / "settings/value.txt").write_text("original settings\n") @@ -54,11 +73,14 @@ async def edit(child, inputs): async def flow(ctx, inputs): candidate = await ctx.candidate_workflow(edit, {}, label="bulk-rewrite") assert len(candidate.effective_diff.encode()) < 1_000_000 - if blocked: - with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + with ExitStack() as descriptors: + for _ in range(already_open): + descriptors.enter_context(open(os.devnull, "rb")) + if blocked: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await ctx.adopt_candidate(candidate) + else: await ctx.adopt_candidate(candidate) - else: - await ctx.adopt_candidate(candidate) return "finished" try: From 1429bcf427adef4e508f6d883cd2495918e9ad9f Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 21 Nov 2026 00:20:36 +0800 Subject: [PATCH 72/82] test(workflow): isolate descriptor limits from the suite process --- .../test_candidate_adoption_command_size.py | 52 +++++++++++++------ 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/tests/workflows/test_candidate_adoption_command_size.py b/tests/workflows/test_candidate_adoption_command_size.py index c8f10584..654abda2 100644 --- a/tests/workflows/test_candidate_adoption_command_size.py +++ b/tests/workflows/test_candidate_adoption_command_size.py @@ -1,8 +1,11 @@ from __future__ import annotations +import asyncio import errno import os import resource +import subprocess +import sys from contextlib import ExitStack from pathlib import Path @@ -21,24 +24,29 @@ async def exec_cmd(self, command, timeout=120.0, **kwargs): return await super().exec_cmd(command, timeout, **kwargs) -@pytest.fixture -def bounded_file_descriptors(): - previous = soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) - limit = 256 if soft == resource.RLIM_INFINITY else min(soft, 256) - resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) - try: - yield limit - finally: - resource.setrlimit(resource.RLIMIT_NOFILE, previous) - - @pytest.mark.parametrize("blocked", [False, True]) @pytest.mark.parametrize("file_count,already_open", [(900, 0), (120, 120)], ids=["bulk", "busy-process"]) -async def test_bulk_candidate_adoption_keeps_recovery_below_process_limits( - tmp_path, blocked, bounded_file_descriptors, file_count, already_open, -): - if already_open and bounded_file_descriptors < 256: +def test_bulk_candidate_adoption_keeps_recovery_below_process_limits(tmp_path, blocked, file_count, already_open): + soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + limit = 256 if soft == resource.RLIM_INFINITY else min(soft, 256) + if already_open and limit < 256: pytest.skip("the busy process case uses a 256 descriptor limit") + completed = subprocess.run( + [ + sys.executable, "-m", __name__, str(tmp_path), + str(int(blocked)), str(file_count), str(already_open), str(limit), + ], + cwd=Path(__file__).resolve().parents[2], + close_fds=True, capture_output=True, text=True, timeout=180, + ) + if completed.returncode: + pytest.fail( + f"candidate child exited with status {completed.returncode}\n{completed.stdout}{completed.stderr}", + pytrace=False, + ) + + +async def _exercise_bulk_candidate(tmp_path: Path, blocked: bool, file_count: int, already_open: int) -> None: # Long absolute paths expand the recovery script past the temp-file limit # even though the repository-relative patch fits under one megabyte. repo = tmp_path.joinpath(*("nested-" + "workspace-" * 8,) * 5, "repo") @@ -60,6 +68,7 @@ async def test_bulk_candidate_adoption_keeps_recovery_below_process_limits( first_module = repo / "src" / names[0] original_inode = first_module.stat().st_ino environment = _ArgumentBoundEnvironment(str(repo)) + adoption_failure: RuntimeError | None = None async def edit(child, inputs): folder = Path(child.workspace_root) @@ -71,14 +80,16 @@ async def edit(child, inputs): return "updated" async def flow(ctx, inputs): + nonlocal adoption_failure candidate = await ctx.candidate_workflow(edit, {}, label="bulk-rewrite") assert len(candidate.effective_diff.encode()) < 1_000_000 with ExitStack() as descriptors: for _ in range(already_open): descriptors.enter_context(open(os.devnull, "rb")) if blocked: - with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory") as failed: await ctx.adopt_candidate(candidate) + adoption_failure = failed.value else: await ctx.adopt_candidate(candidate) return "finished" @@ -86,7 +97,8 @@ async def flow(ctx, inputs): try: result = await OpenCollab(repo, environment=environment).workflow(flow, {}, trace=False) assert result.ok, result.reason - assert not environment.revoked + if environment.revoked: + raise AssertionError("candidate adoption revoked the source environment") from adoption_failure expected = "VALUE = 1\n" if blocked else "VALUE = 2\n" assert all((repo / "src" / name).read_text() == expected for name in names) if blocked: @@ -107,3 +119,9 @@ async def flow(ctx, inputs): assert not list((repo / ".git").rglob(".candidate-command-*.sh")) finally: await environment.cleanup() + + +if __name__ == "__main__": + directory, blocked, file_count, already_open, limit = sys.argv[1:] + resource.setrlimit(resource.RLIMIT_NOFILE, (int(limit), int(limit))) + asyncio.run(_exercise_bulk_candidate(Path(directory), bool(int(blocked)), int(file_count), int(already_open))) From d27516bcdf6e049a5ddcd44c1bcff0888ac013aa Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 21 Nov 2026 00:22:49 +0800 Subject: [PATCH 73/82] ci: allow the full Python test suite to finish --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f04d49fb..8259d51f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,8 +15,8 @@ concurrency: jobs: test: runs-on: ubuntu-latest - # Python 3.12 runs the DOM regressions after the Python suite. - timeout-minutes: ${{ matrix.python-version == '3.12' && 45 || 40 }} + # Allow the full suite and the Python 3.12 DOM checks to finish on hosted runners. + timeout-minutes: 45 strategy: fail-fast: false matrix: From bce552a027c92b0d9bbcba6ebd2314dda4bb040a Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 21 Nov 2026 11:54:49 +0800 Subject: [PATCH 74/82] fix(tools): retain file locks through interrupted adoption --- .../adapters/tools/_file_lock_worker.py | 15 +-- opencollab/adapters/tools/base.py | 120 ++++++++++-------- tests/tools/test_local_file_lock_workers.py | 44 +++++-- .../test_candidate_adoption_command_size.py | 9 +- 4 files changed, 116 insertions(+), 72 deletions(-) diff --git a/opencollab/adapters/tools/_file_lock_worker.py b/opencollab/adapters/tools/_file_lock_worker.py index 5e8643cd..0ea53bc1 100644 --- a/opencollab/adapters/tools/_file_lock_worker.py +++ b/opencollab/adapters/tools/_file_lock_worker.py @@ -2,6 +2,7 @@ import contextlib import json +import math import select import sys import time @@ -9,12 +10,6 @@ from filelock import FileLock, Timeout -def _parent_closed(wait: float | None = 0) -> bool: - # The parent sends one setup message, then keeps its pipe open as ownership. - readable, _, _ = select.select([sys.stdin.buffer], [], [], wait) - return bool(readable) - - def _reply(status: str, detail: str = "") -> None: print(json.dumps((status, detail)), flush=True) @@ -24,12 +19,14 @@ def main() -> None: if not setup: return paths, timeout, poll = json.loads(setup) + owner = select.poll() + owner.register(int(sys.argv[1]), select.POLLIN) try: with contextlib.ExitStack() as held: for path, lock_path in paths: lock = FileLock(lock_path, timeout=0) deadline = time.monotonic() + timeout - while not _parent_closed(): + while not owner.poll(0): try: lock.acquire(timeout=0) except Timeout: @@ -37,7 +34,7 @@ def main() -> None: if remaining <= 0: _reply("timeout", f"timed out acquiring write lock for {path}") return - if _parent_closed(min(poll, remaining)): + if owner.poll(math.ceil(min(poll, remaining) * 1000)): return else: held.callback(lock.release) @@ -45,7 +42,7 @@ def main() -> None: else: return _reply("ready") - _parent_closed(None) + owner.poll() except BrokenPipeError: return except OSError as exc: diff --git a/opencollab/adapters/tools/base.py b/opencollab/adapters/tools/base.py index 2b31da5f..67c183d4 100644 --- a/opencollab/adapters/tools/base.py +++ b/opencollab/adapters/tools/base.py @@ -10,6 +10,8 @@ import os import posixpath import resource +import socket +import subprocess import sys import tempfile from collections.abc import AsyncIterator, Sequence @@ -71,67 +73,84 @@ def _available_local_file_locks(capacity: int) -> int: return 0 if exc.errno == errno.EMFILE else capacity -async def _close_lock_holder(process: asyncio.subprocess.Process) -> None: - assert process.stdin is not None - process.stdin.close() - try: - await asyncio.wait_for(process.wait(), 2) - except asyncio.TimeoutError: - with contextlib.suppress(ProcessLookupError): - process.kill() - await process.wait() +async def _reap_lock_holders(processes: list[subprocess.Popen]) -> None: + loop = asyncio.get_running_loop() + deadline = loop.time() + 2 + while pending := [process for process in processes if process.poll() is None]: + if loop.time() >= deadline: + for process in pending: + process.kill() + deadline = float("inf") + await asyncio.sleep(HOST_WRITE_LOCK_POLL_SECONDS) @contextlib.asynccontextmanager -async def _hold_lock_group(paths: Sequence[str], env: Any) -> AsyncIterator[None]: - process = None - monitor = None +async def _hold_lock_groups(paths: Sequence[str], env: Any, group_size: int) -> AsyncIterator[None]: + owner = asyncio.current_task() + assert owner is not None + processes: list[subprocess.Popen] = [] + ready = 0 worker = os.path.join(os.path.dirname(__file__), "_file_lock_worker.py") - async def spawn() -> None: - nonlocal process - process = await asyncio.create_subprocess_exec( - sys.executable, worker, - stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, - ) - + async def watch_exit() -> None: + while True: + for index in range(ready): + returncode = processes[index].poll() + if returncode is not None: + owner.cancel(f"local write lock holder exited unexpectedly ({returncode})") + return + await asyncio.sleep(HOST_WRITE_LOCK_POLL_SECONDS) + + # Startup sockets close before publication, leaving one descriptor for + # the lifetime of all lock holders. + read_end, write_end = os.pipe() + monitor = asyncio.create_task(watch_exit()) try: - await await_owned_operation(spawn(), propagate_cancellation=True) - assert process is not None and process.stdin is not None and process.stdout is not None - options = ( - [(path, _host_lock_path(path, env)) for path in paths], - HOST_WRITE_LOCK_TIMEOUT_SECONDS, HOST_WRITE_LOCK_POLL_SECONDS, - ) - process.stdin.write(json.dumps(options).encode() + b"\n") - await process.stdin.drain() - response = await process.stdout.readline() - if not response: - raise RuntimeError("local write lock holder exited during acquisition") - status, detail = json.loads(response) - if status == "timeout": - raise TimeoutError(detail) - if status != "ready": - raise RuntimeError(f"local write lock holder failed: {detail}") - owner = asyncio.current_task() - assert owner is not None - - async def watch_exit() -> None: - returncode = await process.wait() - owner.cancel(f"local write lock holder exited unexpectedly ({returncode})") - - monitor = asyncio.create_task(watch_exit()) + for start in range(0, len(paths), group_size): + setup, child = socket.socketpair() + with setup, child: + # Isolate terminal interrupts and avoid per-child pidfds. + processes.append(subprocess.Popen( + [sys.executable, worker, str(read_end)], start_new_session=True, + stdin=child, stdout=child, pass_fds=(read_end,), + )) + child.close() + reader, writer = await asyncio.open_connection(sock=setup) + try: + options = ( + [(path, _host_lock_path(path, env)) for path in paths[start:start + group_size]], + HOST_WRITE_LOCK_TIMEOUT_SECONDS, HOST_WRITE_LOCK_POLL_SECONDS, + ) + writer.write(json.dumps(options).encode() + b"\n") + await writer.drain() + response = await reader.readline() + finally: + writer.close() + with contextlib.suppress(BrokenPipeError, ConnectionResetError): + await await_owned_operation(writer.wait_closed(), propagate_cancellation=True) + if not response: + raise RuntimeError("local write lock holder exited during acquisition") + status, detail = json.loads(response) + if status == "timeout": + raise TimeoutError(detail) + if status != "ready": + raise RuntimeError(f"local write lock holder failed: {detail}") + ready += 1 + os.close(read_end) + read_end = -1 yield finally: - if monitor is not None: - monitor.cancel() - if process is not None: - await await_owned_operation(_close_lock_holder(process), propagate_cancellation=True) + monitor.cancel() + if read_end >= 0: + os.close(read_end) + os.close(write_end) + await await_owned_operation(_reap_lock_holders(processes), propagate_cancellation=True) @contextlib.asynccontextmanager async def _local_write_locks(paths: Sequence[str], env: Any) -> AsyncIterator[FileLock | None]: capacity = _local_file_lock_capacity() - if len(paths) <= capacity and len(paths) <= _available_local_file_locks(capacity): + if len(paths) == 1 or len(paths) <= _available_local_file_locks(capacity): async with contextlib.AsyncExitStack() as locks: target = None try: @@ -145,10 +164,7 @@ async def _local_write_locks(paths: Sequence[str], env: Any) -> AsyncIterator[Fi return # Every holder uses the same per-path locks. Contention order stays the # same across groups, direct callers and older installations. - group_size = min(512, capacity) - async with contextlib.AsyncExitStack() as locks: - for start in range(0, len(paths), group_size): - await locks.enter_async_context(_hold_lock_group(paths[start:start + group_size], env)) + async with _hold_lock_groups(paths, env, min(512, capacity)): yield None diff --git a/tests/tools/test_local_file_lock_workers.py b/tests/tools/test_local_file_lock_workers.py index 10e815d3..740ab2ea 100644 --- a/tests/tools/test_local_file_lock_workers.py +++ b/tests/tools/test_local_file_lock_workers.py @@ -1,6 +1,8 @@ from __future__ import annotations import asyncio +import os +import signal import sys import pytest @@ -77,22 +79,32 @@ async def wait_until_first_is_held(): from opencollab.adapters.env import LocalEnvironment from opencollab.adapters.tools import base +from opencollab.application.async_timeout import await_owned_operation async def main(): base._local_file_lock_capacity = lambda: 2 - env = LocalEnvironment(sys.argv[1]) + stop, workspace, *paths = sys.argv[1:] + env = LocalEnvironment(workspace) entered = asyncio.Event() async def hold(): - async with base.host_write_locks(sys.argv[2:], env): + async with base.host_write_locks(paths, env): entered.set() - await asyncio.Event().wait() + try: + await asyncio.Event().wait() + finally: + if stop == 'terminal': + print('recovering', flush=True) + await await_owned_operation(asyncio.to_thread(sys.stdin.readline)) task = asyncio.create_task(hold()) try: await asyncio.wait_for(entered.wait(), 5) print('ready', flush=True) + if stop == 'terminal': + await task + return await asyncio.to_thread(sys.stdin.readline) task.cancel() try: @@ -106,18 +118,21 @@ async def hold(): await env.cleanup() -asyncio.run(main()) +try: + asyncio.run(main()) +except KeyboardInterrupt: + sys.exit(130) """ -@pytest.mark.parametrize("crash", [False, True]) -async def test_large_group_releases_existing_flocks_after_owner_stops(tmp_path, crash): +@pytest.mark.parametrize("stop", ["cancel", "crash", "terminal"]) +async def test_large_group_releases_existing_flocks_after_owner_stops(tmp_path, stop): paths = ("a.txt", "b.txt", "c.txt") env = LocalEnvironment(str(tmp_path)) old_locks = [FileLock(base._host_lock_path(path, env), timeout=2) for path in paths] owner = await asyncio.create_subprocess_exec( - sys.executable, "-c", _OWNER, str(tmp_path), *paths, - stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, + sys.executable, "-c", _OWNER, stop, str(tmp_path), *paths, + stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, start_new_session=True, ) def confirm_released(): @@ -125,14 +140,25 @@ def confirm_released(): with lock: pass + def confirm_held_during_recovery(): + with pytest.raises(Timeout), old_locks[0].acquire(timeout=0.25): + pass + try: assert await asyncio.wait_for(owner.stdout.readline(), 10) == b"ready\n" for lock in old_locks: with pytest.raises(Timeout): lock.acquire(timeout=0) - if crash: + if stop == "crash": owner.kill() await owner.wait() + elif stop == "terminal": + os.killpg(owner.pid, signal.SIGINT) + assert await asyncio.wait_for(owner.stdout.readline(), 10) == b"recovering\n" + await asyncio.to_thread(confirm_held_during_recovery) + owner.stdin.write(b"finish\n") + await owner.stdin.drain() + assert await asyncio.wait_for(owner.wait(), 10) == 130 else: owner.stdin.write(b"cancel\n") await owner.stdin.drain() diff --git a/tests/workflows/test_candidate_adoption_command_size.py b/tests/workflows/test_candidate_adoption_command_size.py index 654abda2..0029058d 100644 --- a/tests/workflows/test_candidate_adoption_command_size.py +++ b/tests/workflows/test_candidate_adoption_command_size.py @@ -24,8 +24,13 @@ async def exec_cmd(self, command, timeout=120.0, **kwargs): return await super().exec_cmd(command, timeout, **kwargs) -@pytest.mark.parametrize("blocked", [False, True]) -@pytest.mark.parametrize("file_count,already_open", [(900, 0), (120, 120)], ids=["bulk", "busy-process"]) +@pytest.mark.parametrize(("blocked", "file_count", "already_open"), [ + pytest.param(False, 900, 0, id="bulk-False"), + pytest.param(True, 900, 0, id="bulk-True"), + pytest.param(False, 120, 120, id="busy-process-False"), + pytest.param(True, 120, 120, id="busy-process-True"), + pytest.param(False, 900, 232, id="near-descriptor-limit"), +]) def test_bulk_candidate_adoption_keeps_recovery_below_process_limits(tmp_path, blocked, file_count, already_open): soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE) limit = 256 if soft == resource.RLIM_INFINITY else min(soft, 256) From d7c4153fc63988b51ccb251b03a48bc2a50a8862 Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 21 Nov 2026 11:50:51 +0800 Subject: [PATCH 75/82] fix(storage): resume autosaves with legacy Anthropic history --- opencollab/adapters/storage.py | 28 +++++++---- opencollab/application/session_snapshot.py | 12 ++++- tests/agents/test_session_construction.py | 47 +++++++++++++++++++ tests/runtime/test_exported_session_resume.py | 46 ++++++++++++++++++ 4 files changed, 121 insertions(+), 12 deletions(-) diff --git a/opencollab/adapters/storage.py b/opencollab/adapters/storage.py index 683c8225..83c040f0 100644 --- a/opencollab/adapters/storage.py +++ b/opencollab/adapters/storage.py @@ -21,6 +21,7 @@ write_regular_file_atomic, ) from opencollab.application.ports import SnapshotCapacityError, SnapshotSequenceConflict +from opencollab.application.session_snapshot import decode_anthropic_history from opencollab.domain.session import same_snapshot_value MAX_SESSION_SNAPSHOT_BYTES = 64 * 1024 * 1024 @@ -751,26 +752,33 @@ def _newest_persisted_sequence(self, path: str) -> int: def _checkpoint_states_equal( persisted: dict[str, Any], candidate: dict[str, Any] ) -> bool: - """Compare a replayed state with a checkpoint after replay normalization. - - Journal replay materializes ``session_state.seen_result_hashes`` and - sorts it, while a caller's full checkpoint payload may omit that empty - field or retain insertion order. Those representations are - semantically identical; comparing the raw dictionaries would reject a - legitimate idempotent compaction and leave the journal unbounded. - """ - def normalize(value: dict[str, Any]) -> dict[str, Any]: normalized = dict(value) raw_state = normalized.get("session_state") state = dict(raw_state) if isinstance(raw_state, dict) else {} + # Journal replay materializes and sorts the seen-result set. seen = state.get("seen_result_hashes", []) if isinstance(seen, list) and all(isinstance(item, str) for item in seen): state["seen_result_hashes"] = sorted(set(seen)) normalized["session_state"] = state return normalized - return same_snapshot_value(normalize(persisted), normalize(candidate)) + persisted = normalize(persisted) + candidate = normalize(candidate) + if same_snapshot_value(persisted, candidate): + history = persisted["session_state"].get("anthropic_history") + return history is None or decode_anthropic_history(history, persisted["messages"]) is not None + + for snapshot in (persisted, candidate): + state = snapshot["session_state"] + history = state.get("anthropic_history") + if history is None: + return False + decoded = decode_anthropic_history(history, snapshot["messages"]) + if decoded is None: + return False + state["anthropic_history"] = {**history, **decoded} + return same_snapshot_value(persisted, candidate) @staticmethod def _snapshot_sequence(snapshot: dict[str, Any]) -> int: diff --git a/opencollab/application/session_snapshot.py b/opencollab/application/session_snapshot.py index ef211b43..81757610 100644 --- a/opencollab/application/session_snapshot.py +++ b/opencollab/application/session_snapshot.py @@ -40,7 +40,8 @@ def _serialize_anthropic_history(state: SessionState) -> dict | None: stored.append(message) return {"messages": stored, "source_count": count, "tools": history["tools"]} -def _restore_anthropic_history(value: object, messages: list[dict]) -> dict | None: +def decode_anthropic_history(value: object, messages: list[dict]) -> dict | None: + """Resolve native history references without copying nested messages.""" if not isinstance(value, dict): return None count = value.get("source_count") @@ -57,6 +58,10 @@ def _restore_anthropic_history(value: object, messages: list[dict]) -> dict | No if count > len(messages): return None source = messages[:count] + # References use runtime messages. Snapshot timestamps are separate metadata. + for index, message in enumerate(source): + if "timestamp" in message: + source[index] = {key: item for key, item in message.items() if key != "timestamp"} elif ( not isinstance(source, list) or len(source) != count or not all(isinstance(message, dict) for message in source) @@ -70,7 +75,10 @@ def _restore_anthropic_history(value: object, messages: list[dict]) -> dict | No restored_view.append(source[message]) else: return None - return copy.deepcopy({"messages": restored_view, "source_count": count, "tools": tools, "source_messages": source}) + return {"messages": restored_view, "source_count": count, "tools": tools, "source_messages": source} + +def _restore_anthropic_history(value: object, messages: list[dict]) -> dict | None: + return copy.deepcopy(decode_anthropic_history(value, messages)) def _restore_phase(raw: object) -> SessionPhase: value = str(raw) diff --git a/tests/agents/test_session_construction.py b/tests/agents/test_session_construction.py index e9a2baa7..85899e11 100644 --- a/tests/agents/test_session_construction.py +++ b/tests/agents/test_session_construction.py @@ -13,6 +13,7 @@ from opencollab.adapters.storage import SessionStore from opencollab.application.autosave import AutoSaveSubscriber +from opencollab.application.ports import SnapshotSequenceConflict from opencollab.application.scheduler_types import LaunchSpec from opencollab.application.session import SessionRuntime from opencollab.bootstrap import build_session as Session @@ -374,6 +375,52 @@ def test_checkpoint_rejects_conflicting_same_sequence(tmp_path, publisher, publi store.checkpoint_snapshot(str(path), [*base, conflict], meta=meta, sequence=3) assert store.load_snapshot(str(path), "system")["_autosave_sequence"] == 3 +@pytest.mark.parametrize("publisher", ["snapshot", "journal"]) +@pytest.mark.parametrize( + "change", ["encoding", "transcript", "timestamp", "projection", "source", "tools", "malformed"], +) +def test_same_sequence_compares_native_history(tmp_path, publisher, change): + path = str(tmp_path / "native-checkpoint.json") + store = SessionStore() + system = {"role": "system", "content": "system"} + user = {"role": "user", "content": "request"} + published = [system, {**user, "timestamp": "2026-01-01T00:00:00+00:00"}] + legacy = {"messages": [system, user], "source_count": 2, "tools": None} + indexed = {**legacy, "messages": [0, 1]} + candidate = published.copy() + if change == "transcript": + candidate[1] = {**published[1], "content": "new request"} + elif change == "timestamp": + candidate[1] = {**published[1], "timestamp": "2026-01-01T00:00:01+00:00"} + elif change == "projection": + indexed["messages"] = [0, {**user, "content": "edited projection"}] + elif change == "source": + indexed.update(messages=[system, user], source_messages=[system, {**user, "content": "old source"}]) + elif change == "tools": + indexed["tools"] = [{"name": "lookup"}] + elif change == "malformed": + legacy["messages"] = indexed["messages"] = [0, 2] + + meta = {"session_state": {"anthropic_history": legacy}} + if publisher == "snapshot": + store.checkpoint_snapshot(path, published, meta=meta, sequence=1) + else: + store.append_snapshot_delta(path, messages=published, meta=meta, sequence=1, replace_from=0) + previous = store.load_snapshot(path, "system") + updated_meta = {"session_state": {"anthropic_history": indexed}} + if change == "encoding": + assert not store.append_snapshot_replacement( + path, candidate, meta=updated_meta, sequence=1, reject_conflicts=True, + ) + store.checkpoint_snapshot(path, candidate, meta=updated_meta, sequence=1) + assert store.load_snapshot(path, "system")["session_state"]["anthropic_history"] == indexed + else: + with pytest.raises(SnapshotSequenceConflict): + store.append_snapshot_replacement(path, candidate, meta=updated_meta, sequence=1, reject_conflicts=True) + with pytest.raises(SnapshotSequenceConflict, match=f"conflicts with persisted {publisher} state"): + store.checkpoint_snapshot(path, candidate, meta=updated_meta, sequence=1) + assert store.load_snapshot(path, "system") == previous + def test_checkpoint_rejects_ambiguous_duplicate_journal_sequence(tmp_path): path = str(tmp_path / "duplicate-sequence-checkpoint.json") store = SessionStore() diff --git a/tests/runtime/test_exported_session_resume.py b/tests/runtime/test_exported_session_resume.py index 0a5f527b..53785128 100644 --- a/tests/runtime/test_exported_session_resume.py +++ b/tests/runtime/test_exported_session_resume.py @@ -8,10 +8,12 @@ import pytest from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.storage import SessionStore from opencollab.bootstrap import build_session, load_session from opencollab.domain.agent import Agent from opencollab.tools import builtin_tools +from tests.support.provider_sdk_http import completion_http_response, install_sdk_transport from tests.support.session_run_test_support import FakeLLM, llm_response, tool_call @@ -172,6 +174,50 @@ async def test_export_without_original_autosave_can_start_autosaving(tmp_path, t await restored.aclose() await restored.env.cleanup() +async def test_old_anthropic_export_can_resume_an_existing_autosave_twice(tmp_path, monkeypatch): + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + install_sdk_transport(monkeypatch, "anthropic", completion_http_response) + exported = tmp_path / "exported.json" + target = tmp_path / "automatic.json" + agent = Agent(name="reviewer", system_prompt="Remember the prior conversation.", model="claude-sonnet-5-5", + provider="anthropic", thinking=False, max_tokens_per_step=1024) + environment = LocalEnvironment(str(tmp_path)) + try: + async with LLMClient(model=agent.model, provider="anthropic", max_retries=0, + api_key="fake-key", base_url="https://example.invalid/v1") as client: + options = {"agent": agent, "llm": client, "env": environment} + original = build_session(**options) + try: + await original.add_user_message("Remember the review deadline.") + assert await original.run_loop() == "ok" + original.save(str(exported)) + messages = original.messages.copy() + finally: + await original.aclose() + + old_export = json.loads(exported.read_text()) + history = old_export["session_state"]["anthropic_history"] + history["messages"] = [messages[row] if type(row) is int else row for row in history["messages"]] + exported.write_text(json.dumps(old_export)) + first = load_session(str(exported), **options, auto_save_path=str(target)) + await first.aclose() + + old_target = json.loads(target.read_text()) + assert old_target["_autosave_sequence"] == 1 + assert "timestamp" in old_target["messages"][1] + old_target["session_state"]["anthropic_history"] = history + target.write_text(json.dumps(old_target)) + + resumed = load_session(str(exported), **options, auto_save_path=str(target)) + try: + assert resumed.messages == messages + assert resumed.state.anthropic_history["messages"] == history["messages"] + assert json.loads(target.read_text())["_autosave_sequence"] == 1 + finally: + await resumed.aclose() + finally: + await environment.cleanup() + class SnapshotOnlyStore: """Exercise full-snapshot persistence without journal support.""" From d58903a9ce76bb7a5403787086e679a209a043f1 Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 21 Nov 2026 11:52:27 +0800 Subject: [PATCH 76/82] fix(session): preserve compacted Anthropic spans across edits Align unchanged transcript groups with the provider history so edits outside a summary keep its compacted view. Rebuild affected spans from the raw transcript and preserve intact tool exchanges as a unit. --- opencollab/application/_anthropic_history.py | 199 +++++++++++------- tests/runtime/test_anthropic_history_edits.py | 150 +++++++++++++ 2 files changed, 278 insertions(+), 71 deletions(-) create mode 100644 tests/runtime/test_anthropic_history_edits.py diff --git a/opencollab/application/_anthropic_history.py b/opencollab/application/_anthropic_history.py index 21bec2bd..64330ac2 100644 --- a/opencollab/application/_anthropic_history.py +++ b/opencollab/application/_anthropic_history.py @@ -4,6 +4,8 @@ import copy from collections import Counter +from collections.abc import Iterator +from itertools import chain from typing import Any from opencollab.application.shaping.pipeline import _group_spans, is_complete_tool_exchange @@ -41,89 +43,144 @@ def _anthropic_source_changed(state: SessionState) -> bool: source = history.get("source_messages") if history is not None else None return source is not None and not same_snapshot_value(state.messages[:len(source)], source) -def _anchor_key(message: dict) -> tuple[str | None, str | None]: +def _anchor_key(group: list[dict]) -> tuple: + message = group[0] + calls = message.get("tool_calls") + if ( + isinstance(calls, list) and all(isinstance(call, dict) for call in calls) + and is_complete_tool_exchange(group, (0, len(group))) + ): + return ("exchange", tuple(str.__str__(call["id"]) for call in calls)) role = message.get("role") content = message.get("tool_call_id", message.get("content")) if isinstance(content, list) and content and isinstance(content[0], dict): content = content[0].get("text") return ( - str.__str__(role) if isinstance(role, str) else None, + "message", str.__str__(role) if isinstance(role, str) else None, str.__str__(content) if isinstance(content, str) else None, ) -def _projection_anchor( - source: list[dict], view: list[dict], index: int, counts: Counter[tuple[str | None, str | None]], -) -> int | None: - offset = len(view) - len(source) - if offset and counts[_anchor_key(source[index])] != 1: - return None - candidates = (index, index + offset) if offset else (index,) - matches = [ - candidate for candidate in candidates - if 0 <= candidate < len(view) and same_snapshot_value(source[index], view[candidate]) +def _ordered_anchors(candidates: list[tuple[int, int]]) -> Iterator[tuple[int, int]]: + # A moved group can cross several candidates. Keep the anchors on both sides + # of the move and let the caller rebuild the reordered region from the source. + highest = -1 + increasing = set() + for old, new in candidates: + if new > highest: + increasing.add(old) + highest = new + lowest = float("inf") + ordered = [] + for old, new in reversed(candidates): + if new < lowest and old in increasing: + ordered.append((old, new)) + lowest = min(lowest, new) + yield from reversed(ordered) + +def _align_groups( + before: list[list[dict]], after: list[list[dict]], *, projected: bool = False, +) -> Iterator[tuple[int, int]]: + old_keys = list(map(_anchor_key, before)) + new_keys = list(map(_anchor_key, after)) + old_counts, new_counts = Counter(old_keys), Counter(new_keys) + + def matches(old: int, new: int) -> bool: + key = old_keys[old] + if key != new_keys[new]: + return False + if same_snapshot_value(before[old], after[new]): + return True + if not projected: + return False + if key[0] == "exchange": + return old_counts[key] == new_counts[key] == 1 + return same_snapshot_value(_strip_anthropic_thinking(before[old]), after[new]) + + def equal_edges(old: int, new: int, old_stop: int, new_stop: int) -> tuple[int, int]: + size = min(old_stop - old, new_stop - new) + prefix = suffix = 0 + while prefix < size and matches(old + prefix, new + prefix): + prefix += 1 + while suffix < size - prefix and matches(old_stop - suffix - 1, new_stop - suffix - 1): + suffix += 1 + return prefix, suffix + + # Matching runs at the ends can anchor repeated text without a unique key. + prefix, suffix = equal_edges(0, 0, len(before), len(after)) + yield from ((index, index) for index in range(prefix)) + old_stop, new_stop = len(before) - suffix, len(after) - suffix + old_middle, new_middle = old_keys[prefix:old_stop], new_keys[prefix:new_stop] + old_counts_middle, new_counts_middle = Counter(old_middle), Counter(new_middle) + new_positions = { + key: index for index, key in enumerate(new_middle, prefix) if new_counts_middle[key] == 1 + } + candidates = [ + (old, new_positions[key]) for old, key in enumerate(old_middle, prefix) + if old_counts_middle[key] == 1 and key in new_positions and matches(old, new_positions[key]) ] - return matches[0] if len(matches) == 1 else None - -def _rebuild_edited_span(source: list[dict], view: list[dict], messages: list[dict]) -> list[dict]: - start = 0 - while start < min(len(source), len(messages)) and same_snapshot_value(source[start], messages[start]): - start += 1 - - # Exact anchors keep independent compaction intact. A summary or transformed - # message between the edit and its nearest anchors may depend on the edit. - counts = Counter(map(_anchor_key, source)) if len(view) != len(source) else Counter() - view_start = source_start = 0 - for index in range(min(start, len(source) - 1), -1, -1): - anchor = _projection_anchor(source, view, index, counts) - if anchor is not None: - keep = int(index < start) - view_start, source_start = anchor + keep, index + keep - break - - if len(messages) >= len(source): - stop = len(source) - while stop > start and same_snapshot_value(source[stop - 1], messages[stop - 1]): - stop -= 1 - for index in range(stop - 1, len(source)): - anchor = _projection_anchor(source, view, index, counts) - if anchor is not None: - skip = int(index < stop) - view_stop, source_stop = anchor + skip, index + skip - if view_stop >= view_start and source_stop >= source_start: - return [ - *view[:view_start], *messages[source_start:source_stop], - *view[view_stop:], *messages[len(source):], - ] - return [*view[:view_start], *messages[source_start:]] - -def _tool_exchange_spans(messages: list[dict]) -> dict[tuple[str, ...], tuple[int, int] | None]: - exchanges: dict[tuple[str, ...], tuple[int, int] | None] = {} - for start, stop in _group_spans(messages): - calls = messages[start].get("tool_calls") - if not isinstance(calls, list) or not all(isinstance(call, dict) for call in calls): - continue - if is_complete_tool_exchange(messages, (start, stop)): - call_ids = tuple(call["id"] for call in calls) - exchanges[call_ids] = None if call_ids in exchanges else (start, stop) - return exchanges + + old = new = prefix + for next_old, next_new in chain(_ordered_anchors(candidates), ((old_stop, new_stop),)): + head, tail = equal_edges(old, new, next_old, next_new) + for offset in range(head): + yield old + offset, new + offset + if next_old - old == next_new - new: + for offset in range(head, next_old - old - tail): + if matches(old + offset, new + offset): + yield old + offset, new + offset + for offset in range(-tail, 0): + yield next_old + offset, next_new + offset + if next_old < old_stop: + yield next_old, next_new + old, new = next_old + 1, next_new + 1 + yield from ((old_stop + offset, new_stop + offset) for offset in range(suffix)) def _rebase_anthropic_history(history: dict, messages: list[dict]) -> list[dict]: - source = history["source_messages"] view = history["messages"] - rebased = _rebuild_edited_span(source, view, messages) - projections = _tool_exchange_spans(view) - if projections: - previous = _tool_exchange_spans(source) - current = _tool_exchange_spans(messages) - for call_ids, target in _tool_exchange_spans(rebased).items(): - old = previous.get(call_ids) - new = current.get(call_ids) - projection = projections.get(call_ids) - if target is None or old is None or new is None or projection is None: - continue - if same_snapshot_value(source[old[0]:old[1]], messages[new[0]:new[1]]): - # Clear and trim shapers preserve the exchange while shortening its results. - rebased[target[0]:target[1]] = view[projection[0]:projection[1]] + source, projection, current = ( + [rows[start:stop] for start, stop in _group_spans(rows)] + for rows in (history["source_messages"], view, messages) + ) + unchanged = dict(_align_groups(source, current)) + first_edit = 0 + while unchanged.get(first_edit) == first_edit: + first_edit += 1 + end_edit = len(source) + last_match = unchanged.get(end_edit - 1) + while end_edit > first_edit and last_match is not None and unchanged.get(end_edit - 1) == last_match: + end_edit -= 1 + last_match -= 1 + source_counts = Counter(map(_anchor_key, source)) + projection_counts = Counter(map(_anchor_key, projection)) + + def projected_groups() -> Iterator[tuple[int, int, list[list[dict]]]]: + old = seen = 0 + anchors = _align_groups(source, projection, projected=True) + for next_old, next_seen in chain(anchors, ((len(source), len(projection)),)): + if old < next_old: + gap = projection[seen:next_seen] + # The intact ends can retain whole slices. An isolated rewritten + # exchange inside the edits needs a unique call-ID anchor. + if next_old <= first_edit or old >= end_edit or all( + key[0] != "exchange" or source_counts[key] == projection_counts[key] == 1 + for key in map(_anchor_key, gap) + ): + yield old, next_old, gap + if next_old < len(source): + yield next_old, next_old + 1, [projection[next_seen]] + old, seen = next_old + 1, next_seen + 1 + + rebased: list[dict] = [] + taken = 0 + for start, stop, replacement in projected_groups(): + start_new = unchanged.get(start) + # A summary covers its whole source interval, including possible insertions. + if start_new is None or any(unchanged.get(index) != start_new + index - start for index in range(start, stop)): + continue + rebased.extend(chain.from_iterable(current[taken:start_new])) + rebased.extend(chain.from_iterable(replacement)) + taken = start_new + stop - start + rebased.extend(chain.from_iterable(current[taken:])) return _edited_anthropic_history(view, rebased) def _anthropic_history_input(state: SessionState) -> list[dict]: diff --git a/tests/runtime/test_anthropic_history_edits.py b/tests/runtime/test_anthropic_history_edits.py new file mode 100644 index 00000000..82be42af --- /dev/null +++ b/tests/runtime/test_anthropic_history_edits.py @@ -0,0 +1,150 @@ +import json +from copy import deepcopy + +import pytest + +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm.client import LLMClient +from opencollab.application._anthropic_history import _anthropic_history_input, _remember_anthropic_history +from opencollab.application.shaping import AutoCompactShaper +from opencollab.bootstrap import build_session, load_session +from opencollab.bootstrap.context_policy import ContextPolicy +from opencollab.domain.agent import Agent +from opencollab.domain.session import SessionState +from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport + + +@pytest.mark.parametrize("save_after_edit", [False, True], ids=["edit-after-restore", "edit-before-save"]) +async def test_edit_of_repeated_turn_keeps_saved_summary(monkeypatch, tmp_path, save_after_edit): + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + document = "EARLY DOCUMENT " + ("Prior grant application details. " * 650) + summary = "The grant application was acknowledged." + correction = "Continue using Friday as the revised deadline." + requests = [] + summaries = [] + + def summarize(messages): + summaries.append(messages) + return summary + + def peer(request): + requests.append(json.loads(request.content)) + if len(requests) > 3 and len(request.content) > 8_000: + return http_response(request, 400, json={ + "type": "error", + "error": {"type": "invalid_request_error", "message": "prompt is too long for the context window"}, + }) + return completion_http_response(request, text="Acknowledged") + + install_sdk_transport(monkeypatch, "anthropic", peer) + model = "claude-sonnet-5-5" + agent = Agent( + name="reviewer", system_prompt="Review the grant application.", model=model, provider="anthropic", + ) + env = LocalEnvironment(str(tmp_path)) + snapshot = str(tmp_path / "session.json") + async with LLMClient( + model=model, provider="anthropic", api_key="test-placeholder", # pragma: allowlist secret + base_url="https://provider.invalid/v1", max_retries=0, + ) as client: + settings = {"agent": agent, "llm": client, "env": env, "max_budget_tokens": 100_000} + session = build_session( + **settings, + shaper=AutoCompactShaper( + trigger_tokens=600, target_tokens=300, keep_recent_groups=3, summarizer=summarize, + estimate_tokens=lambda rows: sum(len(str(row["content"])) for row in rows) // 4, + ), + ) + try: + for prompt in (document, "Continue", "Continue"): + await session.add_user_message(prompt) + assert await session.run_loop() == "Acknowledged" + assert summary in json.dumps(requests[-1]["messages"]) + assert document in json.dumps(session.messages) + if save_after_edit: + session.messages[5]["content"] = correction + session.save(snapshot) + await session.aclose() + session = load_session(snapshot, **settings, context_policy=ContextPolicy(name="no_history_compaction")) + if not save_after_edit: + session.messages[5]["content"] = correction + await session.add_user_message("What should we do next?") + assert await session.run_loop() == "Acknowledged" + assert len(requests) == 4 + assert len(summaries) == 1 + assert session.messages[5]["content"] == correction + wire = json.dumps(requests[-1]["messages"]) + assert summary in wire + assert correction in wire + assert "EARLY DOCUMENT" not in wire + finally: + await session.aclose() + await env.cleanup() + + +@pytest.mark.parametrize("edit", ["insert-outside-summary", "insert-inside-summary", "delete-inside-summary"]) +def test_disjoint_edits_keep_independent_summaries_and_compacted_tools(edit): + system = {"role": "system", "content": "Review the two grant applications."} + first_document = {"role": "user", "content": "First application"} + first_reply = {"role": "assistant", "content": "First review"} + first_summary = {"role": "system", "compacted": True, "content": "Summary of the first application"} + bridge = {"role": "user", "content": "Read the archive for context"} + call = {"role": "assistant", "content": None, "tool_calls": [ + {"id": "read-archive", "type": "function", "function": {"name": "file_read", "arguments": "{}"}}, + ]} + result = {"role": "tool", "tool_call_id": "read-archive", "content": "Archive data"} + cleared = {**result, "content": "[Old tool result content cleared]"} + second_document = {"role": "user", "content": "Second application"} + second_reply = {"role": "assistant", "content": "Second review"} + second_summary = {"role": "system", "compacted": True, "content": "Summary of the second application"} + repeat = {"role": "user", "content": "Continue"} + acknowledgement = {"role": "assistant", "content": "Acknowledged"} + state = SessionState(messages=[ + system, first_document, first_reply, bridge, call, result, + second_document, second_reply, repeat, acknowledgement, repeat, + ]) + _remember_anthropic_history(state, [ + system, first_summary, bridge, call, cleared, second_summary, repeat, acknowledgement, repeat, + ], None) + + revised_bridge = {"role": "user", "content": "Check the archive's dates"} + revised_tail = {"role": "user", "content": "Continue with the new dates"} + insertion = {"role": "user", "content": "Check the updated budget"} + state.messages[3] = revised_bridge + state.messages[-1] = revised_tail + if edit == "insert-outside-summary": + state.messages.insert(8, insertion) + remaining = [second_summary, insertion] + elif edit == "insert-inside-summary": + state.messages.insert(7, insertion) + remaining = [second_document, insertion, second_reply] + else: + del state.messages[7] + remaining = [second_document] + assert _anthropic_history_input(state) == [ + system, first_summary, revised_bridge, call, cleared, *remaining, repeat, acknowledgement, revised_tail, + ] + + +@pytest.mark.parametrize("edited_index", [1, -1], ids=["earlier-edit", "later-edit"]) +def test_reused_tool_ids_keep_a_compacted_exchange_in_an_unchanged_end(edited_index): + call = {"role": "assistant", "content": None, "tool_calls": [ + {"id": "read-archive", "type": "function", "function": {"name": "file_read", "arguments": "{}"}}, + ]} + result = {"role": "tool", "tool_call_id": "read-archive", "content": "First archive version"} + state = SessionState(messages=[ + {"role": "system", "content": "Review the grant application."}, + {"role": "user", "content": "Read the original archive"}, + call, result, + {"role": "user", "content": "Read the updated archive"}, + deepcopy(call), {**result, "content": "New archive version"}, + {"role": "user", "content": "Compare the dates"}, + ]) + view = deepcopy(state.messages) + view[3]["content"] = "[Old tool result content cleared]" + _remember_anthropic_history(state, view, None) + state.messages[edited_index]["content"] = "Look up the deadline" + view[edited_index]["content"] = "Look up the deadline" + assert _anthropic_history_input(state) == view From 261d5bea76cb3c6cd8379038927327e152663e24 Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 21 Nov 2026 11:49:33 +0800 Subject: [PATCH 77/82] fix(llm): settle responses with redacted usage details once Match copies by provider payload or transport identity and the charged token counters. Prefer the usage payload when a router retains timing from a different request. --- opencollab/application/_session_run_usage.py | 32 +++--- tests/runtime/test_routed_completion_usage.py | 98 +++++++++++++++++-- 2 files changed, 105 insertions(+), 25 deletions(-) diff --git a/opencollab/application/_session_run_usage.py b/opencollab/application/_session_run_usage.py index b24ffeff..7e10f5fe 100644 --- a/opencollab/application/_session_run_usage.py +++ b/opencollab/application/_session_run_usage.py @@ -57,14 +57,17 @@ def _normalize_completion_usage(usage: Any) -> tuple[int, int]: return input_tokens if context_tokens is None else context_tokens, max(reported_total, input_tokens + output_tokens) -def _same_usage_measurements(native: Any, usage: Any) -> bool: - # Context estimates and tool-markup annotations do not change provider usage. +def _same_token_counts(native: Any, usage: Any) -> bool: return type(native) is type(usage) and all( getattr(native, name) == getattr(usage, name) - for name in ( - "input_tokens", "output_tokens", "total_tokens", - "cache_read_tokens", "cache_creation_tokens", "reasoning_tokens", - ) + for name in ("input_tokens", "output_tokens", "total_tokens") + ) + + +def _same_usage_measurements(native: Any, usage: Any) -> bool: + return _same_token_counts(native, usage) and all( + getattr(native, name) == getattr(usage, name) + for name in ("cache_read_tokens", "cache_creation_tokens", "reasoning_tokens") ) @@ -123,22 +126,21 @@ def finish( (item for item in self._native_usages if getattr(item[0], "_native_usage_origin", None) is origin), None, ) - if native is None and transport_timing is not None: - # The native request's timing object survives replacements of the response and usage. - native = next( - (item for item in self._native_usages - if item[2] is transport_timing and _same_usage_measurements(item[0], usage)), - None, - ) if native is None and origin is not None: raw_usage = getattr(usage, "raw_usage", None) if raw_usage is not None: - # A dataclass replacement normally retains the original provider payload. + # The usage payload takes precedence when a router carries timing from another call. native = next( (item for item in self._native_usages - if getattr(item[0], "raw_usage", None) is raw_usage and _same_usage_measurements(item[0], usage)), + if getattr(item[0], "raw_usage", None) is raw_usage and _same_token_counts(item[0], usage)), None, ) + if native is None and transport_timing is not None: + native = next( + (item for item in self._native_usages + if item[2] is transport_timing and _same_token_counts(item[0], usage)), + None, + ) if usage is not None and native is None: native = next((item for item in reversed(self._native_usages) if item[0] == usage), None) if native is None and origin is not None: diff --git a/tests/runtime/test_routed_completion_usage.py b/tests/runtime/test_routed_completion_usage.py index 2065e38c..b0f99ae1 100644 --- a/tests/runtime/test_routed_completion_usage.py +++ b/tests/runtime/test_routed_completion_usage.py @@ -14,6 +14,7 @@ from opencollab.application.run_control import RunControl from opencollab.bootstrap import build_session from opencollab.domain.agent import Agent +from opencollab.tools import builtin_tools from tests.runtime.test_responses_error_usage_http import _payload from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport @@ -25,6 +26,8 @@ "fallback", "equal-fallback", "copied-fallback", "scrubbed-fallback", "replaced-fallback", "annotated-primary", "annotated-fallback", "annotated-primary-after-helper", "annotated-primary-after-helper-without-timing", + "cache-redacted-primary-after-helper", "cache-redacted-primary-after-helper-without-timing", + "cache-redacted-fallback", "fallback-with-primary-timing", "failed-primary", "failed-primary-attempt-callback-only", ]) async def test_routed_completion_accounts_for_each_provider( @@ -33,13 +36,17 @@ async def test_routed_completion_accounts_for_each_provider( requests = [] events = [] answer = "Answer." if summary else "Answer." - independent_helper = scenario.startswith("annotated-primary-after-helper") + independent_helper = "-primary-after-helper" in scenario + cache_redacted = scenario.startswith("cache-redacted-") + annotated = scenario.startswith("annotated-") or cache_redacted + retains_primary_timing = scenario == "fallback-with-primary-timing" usable_primary = independent_helper or scenario in { "primary", "copied-primary", "scrubbed-primary", "replaced-primary", "annotated-primary", } - equal_usage = independent_helper or scenario in { - "equal-fallback", "copied-fallback", "scrubbed-fallback", "replaced-fallback", "annotated-fallback", + equal_usage = independent_helper or retains_primary_timing or scenario in { + "equal-fallback", "copied-fallback", "scrubbed-fallback", "replaced-fallback", + "annotated-fallback", "cache-redacted-fallback", } failed_primary = scenario.startswith("failed-primary") monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) @@ -55,19 +62,25 @@ async def handle(request): if failed_primary: content_type, payload = _payload(json.loads(request.content), {"kind": "failed"}) return http_response(request, 200, headers={"content-type": content_type}, content=payload) - return completion_http_response(request, text=answer if usable_primary else "", output_tokens=55) - return completion_http_response(request, text=answer, output_tokens=55 if equal_usage else 2) + text = answer if route != "primary" or usable_primary else "" + output_tokens = 55 if route == "primary" or equal_usage else 2 + payload = completion_http_response(request, text=text, output_tokens=output_tokens).json() + payload["usage"]["prompt_tokens_details"] = { + "cached_tokens": 0 if retains_primary_timing and route == "fallback" else 1, + } + return http_response(request, 200, json=payload) def copy_response(response): if scenario.startswith("replaced-"): return replace(response, usage=replace(response.usage, raw_usage={})) - if scenario.startswith("annotated-"): - removes_timing = scenario == "annotated-primary-after-helper-without-timing" + if annotated: + removes_timing = scenario.endswith("-without-timing") usage = replace( response.usage, context_tokens=7, context_output_tokens=5, context_tokens_estimated=True, + cache_read_tokens=None if cache_redacted else response.usage.cache_read_tokens, raw_usage=response.usage.raw_usage if removes_timing else {}, ) timing = None if removes_timing else response.transport_timing @@ -97,8 +110,10 @@ async def complete( await fallback.complete(messages, **kwargs) return copy_response(first) if scenario != "primary" else first final = await fallback.complete(messages, **kwargs, **(controls if forward_fallback else {})) - if failed_primary or scenario in { - "copied-fallback", "scrubbed-fallback", "replaced-fallback", "annotated-fallback", + if retains_primary_timing: + return replace(final, usage=replace(final.usage), transport_timing=first.transport_timing) + if failed_primary or annotated or scenario in { + "copied-fallback", "scrubbed-fallback", "replaced-fallback", }: return copy_response(final) return final @@ -125,7 +140,7 @@ async def complete( text = await session._context_summarizer.asummarize(session.messages) assert text == "Summary:\nAnswer." assert session._context_summarizer.last_call_cacheable - if scenario.startswith("annotated-"): + if annotated: assert session.state.context_tokens == 7 tokens = session.used_tokens finally: @@ -157,3 +172,66 @@ async def complete( if failed_primary: assert [entry["status"] for entry in ledger] == ["error", "success"] assert [event.data["error_type"] for event in usage_events] == ["ResponsesTransientEventError", None] + + +@pytest.mark.parametrize("cache_read_tokens", [None, 0]) +async def test_cache_redaction_preserves_budget_for_tool_followup(tmp_path, monkeypatch, cache_read_tokens): + (tmp_path / "facts.txt").write_text("The verified value is 42.\n", encoding="utf-8") + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + requests = [] + events = [] + + async def handle(request): + body = json.loads(request.content) + tool_results = [message["content"] for message in body["messages"] if message["role"] == "tool"] + requests.append(tool_results) + input_tokens, output_tokens = (315, 22) if tool_results else (270, 30) + assert body["max_tokens"] >= output_tokens + payload = completion_http_response(request, text="The value is 42.", output_tokens=output_tokens).json() + payload["usage"].update( + prompt_tokens=input_tokens, total_tokens=input_tokens + output_tokens, + prompt_tokens_details={"cached_tokens": 1}, + ) + if not tool_results: + choice = payload["choices"][0] + choice["finish_reason"] = "tool_calls" + choice["message"].update(content=None, tool_calls=[{ + "id": "read-facts", + "type": "function", + "function": {"name": "file_read", "arguments": json.dumps({"path": "facts.txt"})}, + }]) + return http_response(request, 200, json=payload) + + class CacheRedactingClient(LLMClient): + async def complete( + self, messages, *, on_admitted=None, before_request=None, on_attempt_usage=None, on_usage=None, **kwargs, + ): + controls = dict(on_admitted=on_admitted, before_request=before_request, + on_attempt_usage=on_attempt_usage, on_usage=on_usage) + kwargs.update((name, callback) for name, callback in controls.items() if callback is not None) + response = await super().complete(messages, **kwargs) + assert response.usage.cache_read_tokens == 1 + return replace(response, usage=replace(response.usage, cache_read_tokens=cache_read_tokens)) + + install_sdk_transport(monkeypatch, "openai", handle) + async with CacheRedactingClient( + api_key=__name__, base_url="https://test.invalid/v1", max_retries=0, + context_window=40_000, request_pool=str(tmp_path), request_capacity=1, + ) as client: + result = await OpenCollab(tmp_path, config={"llm_timeout": 3, "max_output_tokens": 900}).agent( + "Read facts.txt and report the verified value.", llm=client, tools=builtin_tools("file_read"), + budget=1_250, max_steps=4, trace=False, system_prompt="System.", + run_control=RunControl(on_event=events.append), + ) + + assert result.status == "completed", result.reason + assert result.output == "The value is 42." + assert result.tokens == 637 + assert len(requests) == 2 + assert requests[0] == [] + assert "The verified value is 42." in requests[1][0] + ledger = [json.loads(line) for line in (tmp_path / "usage.jsonl").read_text().splitlines()] + assert [entry["usage"]["total_tokens"] for entry in ledger] == [300, 337] + assert [event.data["total_tokens"] for event in events if event.type == "usage"] == [300, 337] From 8e965317dc35c97df300799a11094f0dda4c1c01 Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 21 Nov 2026 11:53:16 +0800 Subject: [PATCH 78/82] fix(llm): reconcile copied usage from settlement callbacks Associate callback usage with the native request owner so settlement retains the callback metadata and charges the request once. --- opencollab/application/_session_run_usage.py | 28 +++++++++++++------ tests/runtime/test_routed_completion_usage.py | 18 +++++++++--- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/opencollab/application/_session_run_usage.py b/opencollab/application/_session_run_usage.py index 7e10f5fe..dd6b385f 100644 --- a/opencollab/application/_session_run_usage.py +++ b/opencollab/application/_session_run_usage.py @@ -77,7 +77,8 @@ def __init__(self, runner: Any, *, protected_call: bool) -> None: self.protected_call = protected_call self.tokens = 0 self._attempt_tokens: dict[object | None, int] = {} - self._native_usages: list[tuple[Any, BaseException | None, object | None]] = [] + self._callback_usages: dict[object, Any] = {} + self._native_usages: list[tuple[Any, BaseException | None, object | None, Any]] = [] self._settled: list[tuple[Any, int]] = [] self._reported = False @@ -96,16 +97,23 @@ def record_attempt(self, usage: Any) -> None: self._attempt_tokens[owner] = self._attempt_tokens.get(owner, 0) + tokens def settle_provider(self, usage: Any) -> None: - self._settle(usage, self._attempt_tokens.pop(_native_usage_owner.get(), 0)) + owner = _native_usage_owner.get() + self._settle(usage, self._attempt_tokens.pop(owner, 0)) + if owner is not None: + self._callback_usages[owner] = usage def _observe_provider_response( self, usage: Any, error: BaseException | None, transport_timing: object | None, ) -> None: - self._native_usages.append((usage, error, transport_timing)) owner = _native_usage_owner.get() + # A callback may settle a redacted copy of this request's usage. + settled = self._callback_usages.pop(owner, usage) + if settled is not usage and not _same_token_counts(usage, settled): + settled = usage + self._native_usages.append((usage, error, transport_timing, settled)) # A router can forward the retry hook and keep the final callback for its chosen response. if owner is not None and owner in self._attempt_tokens: - self.settle_provider(usage) + self._settle(usage, self._attempt_tokens.pop(owner)) def _settle(self, usage: Any, attempt_tokens: int) -> None: tokens = _normalize_completion_usage(usage)[1] @@ -148,17 +156,19 @@ def finish( (item for item in reversed(self._native_usages) if _same_usage_measurements(item[0], usage)), None, ) - native_usage = usage if native is None else native[0] - if usage is not None and all(native_usage is not settled for settled, _tokens in self._settled): + reported_usage = usage if native is None else native[3] + if usage is not None and all(reported_usage is not settled for settled, _tokens in self._settled): attempt_tokens = self._attempt_tokens.pop(None, 0) if native is None else 0 - self._settle(native_usage, attempt_tokens) + self._settle(reported_usage, attempt_tokens) self._reported = True record_usage = getattr(self.runner, "_record_usage_event", None) if callable(record_usage): for settled, tokens in self._settled: - provider_error = next((failure for item, failure, _ in self._native_usages if item is settled), None) + provider_error = next( + (failure for _, failure, _, recorded in self._native_usages if recorded is settled), None, + ) record_usage(settled, tokens, purpose=purpose, late=late, - error=error if settled is native_usage and error is not None else provider_error) + error=error if settled is reported_usage and error is not None else provider_error) if late: self.record_late() return bool(self._settled) diff --git a/tests/runtime/test_routed_completion_usage.py b/tests/runtime/test_routed_completion_usage.py index b0f99ae1..bcdc88de 100644 --- a/tests/runtime/test_routed_completion_usage.py +++ b/tests/runtime/test_routed_completion_usage.py @@ -175,7 +175,8 @@ async def complete( @pytest.mark.parametrize("cache_read_tokens", [None, 0]) -async def test_cache_redaction_preserves_budget_for_tool_followup(tmp_path, monkeypatch, cache_read_tokens): +@pytest.mark.parametrize("redact_at", ["response", "settlement"]) +async def test_cache_redaction_preserves_budget_for_tool_followup(tmp_path, monkeypatch, cache_read_tokens, redact_at): (tmp_path / "facts.txt").write_text("The verified value is 42.\n", encoding="utf-8") monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") @@ -208,12 +209,18 @@ class CacheRedactingClient(LLMClient): async def complete( self, messages, *, on_admitted=None, before_request=None, on_attempt_usage=None, on_usage=None, **kwargs, ): + def redact_usage(usage): + on_usage(replace(usage, cache_read_tokens=cache_read_tokens)) + + settlement_callback = redact_usage if redact_at == "settlement" and on_usage is not None else on_usage controls = dict(on_admitted=on_admitted, before_request=before_request, - on_attempt_usage=on_attempt_usage, on_usage=on_usage) + on_attempt_usage=on_attempt_usage, on_usage=settlement_callback) kwargs.update((name, callback) for name, callback in controls.items() if callback is not None) response = await super().complete(messages, **kwargs) assert response.usage.cache_read_tokens == 1 - return replace(response, usage=replace(response.usage, cache_read_tokens=cache_read_tokens)) + if redact_at == "response": + return replace(response, usage=replace(response.usage, cache_read_tokens=cache_read_tokens)) + return response install_sdk_transport(monkeypatch, "openai", handle) async with CacheRedactingClient( @@ -234,4 +241,7 @@ async def complete( assert "The verified value is 42." in requests[1][0] ledger = [json.loads(line) for line in (tmp_path / "usage.jsonl").read_text().splitlines()] assert [entry["usage"]["total_tokens"] for entry in ledger] == [300, 337] - assert [event.data["total_tokens"] for event in events if event.type == "usage"] == [300, 337] + usage_events = [event.data for event in events if event.type == "usage"] + assert [event["total_tokens"] for event in usage_events] == [300, 337] + if redact_at == "settlement": + assert [event["cache_read_tokens"] for event in usage_events] == [cache_read_tokens] * 2 From 05b1a90b9380e240744cecf14fb66d50508130a5 Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 21 Nov 2026 12:03:19 +0800 Subject: [PATCH 79/82] fix(tools): use one socket for a single lock holder --- opencollab/adapters/tools/base.py | 79 ++++++++++++--------- tests/tools/test_local_file_lock_workers.py | 40 +++++++++++ 2 files changed, 86 insertions(+), 33 deletions(-) diff --git a/opencollab/adapters/tools/base.py b/opencollab/adapters/tools/base.py index 67c183d4..0043ab0a 100644 --- a/opencollab/adapters/tools/base.py +++ b/opencollab/adapters/tools/base.py @@ -84,6 +84,28 @@ async def _reap_lock_holders(processes: list[subprocess.Popen]) -> None: await asyncio.sleep(HOST_WRITE_LOCK_POLL_SECONDS) +async def _prepare_lock_holder(channel: socket.socket, paths: Sequence[str], env: Any) -> None: + loop = asyncio.get_running_loop() + channel.setblocking(False) + options = ( + [(path, _host_lock_path(path, env)) for path in paths], + HOST_WRITE_LOCK_TIMEOUT_SECONDS, HOST_WRITE_LOCK_POLL_SECONDS, + ) + await loop.sock_sendall(channel, json.dumps(options).encode() + b"\n") + response = bytearray() + while chunk := await loop.sock_recv(channel, 4096): + response.extend(chunk) + if chunk.endswith(b"\n"): + break + else: + raise RuntimeError("local write lock holder exited during acquisition") + status, detail = json.loads(response) + if status == "timeout": + raise TimeoutError(detail) + if status != "ready": + raise RuntimeError(f"local write lock holder failed: {detail}") + + @contextlib.asynccontextmanager async def _hold_lock_groups(paths: Sequence[str], env: Any, group_size: int) -> AsyncIterator[None]: owner = asyncio.current_task() @@ -101,49 +123,40 @@ async def watch_exit() -> None: return await asyncio.sleep(HOST_WRITE_LOCK_POLL_SECONDS) - # Startup sockets close before publication, leaving one descriptor for - # the lifetime of all lock holders. - read_end, write_end = os.pipe() + # One group can use its startup socket as the ownership signal. + shared = len(paths) > group_size + read_end, write_end = os.pipe() if shared else (-1, -1) + ownership_socket = None monitor = asyncio.create_task(watch_exit()) try: for start in range(0, len(paths), group_size): setup, child = socket.socketpair() - with setup, child: - # Isolate terminal interrupts and avoid per-child pidfds. - processes.append(subprocess.Popen( - [sys.executable, worker, str(read_end)], start_new_session=True, - stdin=child, stdout=child, pass_fds=(read_end,), - )) - child.close() - reader, writer = await asyncio.open_connection(sock=setup) - try: - options = ( - [(path, _host_lock_path(path, env)) for path in paths[start:start + group_size]], - HOST_WRITE_LOCK_TIMEOUT_SECONDS, HOST_WRITE_LOCK_POLL_SECONDS, - ) - writer.write(json.dumps(options).encode() + b"\n") - await writer.drain() - response = await reader.readline() - finally: - writer.close() - with contextlib.suppress(BrokenPipeError, ConnectionResetError): - await await_owned_operation(writer.wait_closed(), propagate_cancellation=True) - if not response: - raise RuntimeError("local write lock holder exited during acquisition") - status, detail = json.loads(response) - if status == "timeout": - raise TimeoutError(detail) - if status != "ready": - raise RuntimeError(f"local write lock holder failed: {detail}") + if not shared: + ownership_socket = setup + try: + with child: + # Isolate terminal interrupts and avoid per-child pidfds. + processes.append(subprocess.Popen( + [sys.executable, worker, str(read_end if shared else 0)], start_new_session=True, + stdin=child, stdout=child, pass_fds=(read_end,) if shared else (), + )) + await _prepare_lock_holder(setup, paths[start:start + group_size], env) + finally: + if shared: + setup.close() ready += 1 - os.close(read_end) - read_end = -1 + if shared: + os.close(read_end) + read_end = -1 yield finally: monitor.cancel() if read_end >= 0: os.close(read_end) - os.close(write_end) + if shared: + os.close(write_end) + if ownership_socket is not None: + ownership_socket.close() await await_owned_operation(_reap_lock_holders(processes), propagate_cancellation=True) diff --git a/tests/tools/test_local_file_lock_workers.py b/tests/tools/test_local_file_lock_workers.py index 740ab2ea..5a25ca1f 100644 --- a/tests/tools/test_local_file_lock_workers.py +++ b/tests/tools/test_local_file_lock_workers.py @@ -2,14 +2,20 @@ import asyncio import os +import resource import signal +import subprocess import sys +from contextlib import ExitStack +from pathlib import Path import pytest from filelock import FileLock, Timeout from opencollab.adapters.env import LocalEnvironment from opencollab.adapters.tools import base +from opencollab.adapters.tools.fs import FileWriteTool +from opencollab.application.tool_execution import ToolRuntime async def test_large_group_acquires_existing_flocks_in_path_order(tmp_path, monkeypatch): @@ -73,6 +79,40 @@ async def wait_until_first_is_held(): await env.cleanup() +def test_nested_file_write_with_four_free_descriptors(tmp_path): + completed = subprocess.run( + [sys.executable, "-c", + "import asyncio, sys; from tests.tools.test_local_file_lock_workers import _write_nested_file; " + "asyncio.run(_write_nested_file(sys.argv[1]))", str(tmp_path)], + cwd=Path(__file__).resolve().parents[2], + close_fds=True, capture_output=True, text=True, timeout=20, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + + +async def _write_nested_file(workspace): + target = Path(workspace, "nested/subdirectory/note.txt") + target.parent.mkdir(parents=True) + target.write_text("before\n") + env = LocalEnvironment(workspace) + try: + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + limit = 256 if soft == resource.RLIM_INFINITY else min(soft, 256) + resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) + opened = len(os.listdir("/dev/fd")) - 1 + with ExitStack() as descriptors: + for _ in range(limit - opened - 4): + descriptors.enter_context(open(os.devnull, "rb")) + await FileWriteTool().execute_with_runtime( + {"path": str(target.relative_to(workspace)), "mode": "str_replace", + "old_str": "before", "new_str": "after"}, + ToolRuntime(environment=env, safety_policy=None, permission_policy=None), + ) + assert target.read_text() == "after\n" + finally: + await env.cleanup() + + _OWNER = """ import asyncio import sys From cf76d8c698cbb64f430d39a3c6b7b10527b99328 Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 21 Nov 2026 11:57:57 +0800 Subject: [PATCH 80/82] fix(llm): reconcile usage settled by completion wrappers Match manual settlement callbacks to observed provider usage by request provenance. Preserve separate charges for separate requests and changed token measurements. --- opencollab/application/_session_run_usage.py | 64 +++++++++++++------ tests/runtime/test_routed_completion_usage.py | 21 ++++-- 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/opencollab/application/_session_run_usage.py b/opencollab/application/_session_run_usage.py index dd6b385f..29edaa21 100644 --- a/opencollab/application/_session_run_usage.py +++ b/opencollab/application/_session_run_usage.py @@ -7,6 +7,7 @@ from collections.abc import Iterator from contextlib import contextmanager from contextvars import ContextVar +from dataclasses import dataclass from typing import Any, Awaitable, Callable _native_usage_owner: ContextVar[object | None] = ContextVar("native_usage_owner", default=None) @@ -71,6 +72,14 @@ def _same_usage_measurements(native: Any, usage: Any) -> bool: ) +@dataclass +class _ProviderUsageObservation: + usage: Any + error: BaseException | None + transport_timing: object | None + reported_usage: Any + + class _CompletionUsageAccounting: def __init__(self, runner: Any, *, protected_call: bool) -> None: self.runner = runner @@ -78,7 +87,7 @@ def __init__(self, runner: Any, *, protected_call: bool) -> None: self.tokens = 0 self._attempt_tokens: dict[object | None, int] = {} self._callback_usages: dict[object, Any] = {} - self._native_usages: list[tuple[Any, BaseException | None, object | None, Any]] = [] + self._native_usages: list[_ProviderUsageObservation] = [] self._settled: list[tuple[Any, int]] = [] self._reported = False @@ -98,8 +107,15 @@ def record_attempt(self, usage: Any) -> None: def settle_provider(self, usage: Any) -> None: owner = _native_usage_owner.get() + native = self._find_native_usage(usage) if owner is None else None + if native is not None and native.usage is not usage and not _same_token_counts(native.usage, usage): + native = None + if native is not None and self._already_settled(native.reported_usage): + return self._settle(usage, self._attempt_tokens.pop(owner, 0)) - if owner is not None: + if native is not None: + native.reported_usage = usage + elif owner is not None: self._callback_usages[owner] = usage def _observe_provider_response( @@ -110,7 +126,7 @@ def _observe_provider_response( settled = self._callback_usages.pop(owner, usage) if settled is not usage and not _same_token_counts(usage, settled): settled = usage - self._native_usages.append((usage, error, transport_timing, settled)) + self._native_usages.append(_ProviderUsageObservation(usage, error, transport_timing, settled)) # A router can forward the retry hook and keep the final callback for its chosen response. if owner is not None and owner in self._attempt_tokens: self._settle(usage, self._attempt_tokens.pop(owner)) @@ -120,44 +136,50 @@ def _settle(self, usage: Any, attempt_tokens: int) -> None: self._charge(tokens - attempt_tokens) self._settled.append((usage, tokens)) - def finish( - self, usage: Any, *, purpose: str, late: bool, error: BaseException | None = None, - transport_timing: object | None = None, - ) -> bool: - if self._reported: - return False - # Middleware may copy the chosen response or return a different native call with equal counters. - native = next((item for item in self._native_usages if item[0] is usage), None) + def _already_settled(self, usage: Any) -> bool: + return any(usage is settled for settled, _tokens in self._settled) + + def _find_native_usage(self, usage: Any) -> _ProviderUsageObservation | None: + native = next((item for item in self._native_usages if item.usage is usage), None) origin = getattr(usage, "_native_usage_origin", None) if native is None and origin is not None: native = next( - (item for item in self._native_usages if getattr(item[0], "_native_usage_origin", None) is origin), + (item for item in self._native_usages if getattr(item.usage, "_native_usage_origin", None) is origin), None, ) if native is None and origin is not None: raw_usage = getattr(usage, "raw_usage", None) if raw_usage is not None: - # The usage payload takes precedence when a router carries timing from another call. native = next( (item for item in self._native_usages - if getattr(item[0], "raw_usage", None) is raw_usage and _same_token_counts(item[0], usage)), + if getattr(item.usage, "raw_usage", None) is raw_usage and _same_token_counts(item.usage, usage)), None, ) + return native + + def finish( + self, usage: Any, *, purpose: str, late: bool, error: BaseException | None = None, + transport_timing: object | None = None, + ) -> bool: + if self._reported: + return False + # A router can retain timing from a different call, so prefer the usage payload. + native = self._find_native_usage(usage) if native is None and transport_timing is not None: native = next( (item for item in self._native_usages - if item[2] is transport_timing and _same_token_counts(item[0], usage)), + if item.transport_timing is transport_timing and _same_token_counts(item.usage, usage)), None, ) if usage is not None and native is None: - native = next((item for item in reversed(self._native_usages) if item[0] == usage), None) - if native is None and origin is not None: + native = next((item for item in reversed(self._native_usages) if item.usage == usage), None) + if native is None and getattr(usage, "_native_usage_origin", None) is not None: native = next( - (item for item in reversed(self._native_usages) if _same_usage_measurements(item[0], usage)), + (item for item in reversed(self._native_usages) if _same_usage_measurements(item.usage, usage)), None, ) - reported_usage = usage if native is None else native[3] - if usage is not None and all(reported_usage is not settled for settled, _tokens in self._settled): + reported_usage = usage if native is None else native.reported_usage + if usage is not None and not self._already_settled(reported_usage): attempt_tokens = self._attempt_tokens.pop(None, 0) if native is None else 0 self._settle(reported_usage, attempt_tokens) self._reported = True @@ -165,7 +187,7 @@ def finish( if callable(record_usage): for settled, tokens in self._settled: provider_error = next( - (failure for _, failure, _, recorded in self._native_usages if recorded is settled), None, + (item.error for item in self._native_usages if item.reported_usage is settled), None, ) record_usage(settled, tokens, purpose=purpose, late=late, error=error if settled is reported_usage and error is not None else provider_error) diff --git a/tests/runtime/test_routed_completion_usage.py b/tests/runtime/test_routed_completion_usage.py index bcdc88de..b57270fe 100644 --- a/tests/runtime/test_routed_completion_usage.py +++ b/tests/runtime/test_routed_completion_usage.py @@ -27,7 +27,7 @@ "annotated-primary", "annotated-fallback", "annotated-primary-after-helper", "annotated-primary-after-helper-without-timing", "cache-redacted-primary-after-helper", "cache-redacted-primary-after-helper-without-timing", - "cache-redacted-fallback", "fallback-with-primary-timing", + "cache-redacted-fallback", "fallback-with-primary-timing", "manual-primary-settlement", "failed-primary", "failed-primary-attempt-callback-only", ]) async def test_routed_completion_accounts_for_each_provider( @@ -40,13 +40,14 @@ async def test_routed_completion_accounts_for_each_provider( cache_redacted = scenario.startswith("cache-redacted-") annotated = scenario.startswith("annotated-") or cache_redacted retains_primary_timing = scenario == "fallback-with-primary-timing" + manual_primary_settlement = scenario == "manual-primary-settlement" usable_primary = independent_helper or scenario in { "primary", "copied-primary", "scrubbed-primary", "replaced-primary", "annotated-primary", } equal_usage = independent_helper or retains_primary_timing or scenario in { "equal-fallback", "copied-fallback", "scrubbed-fallback", "replaced-fallback", - "annotated-fallback", "cache-redacted-fallback", + "annotated-fallback", "cache-redacted-fallback", "manual-primary-settlement", } failed_primary = scenario.startswith("failed-primary") monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) @@ -98,13 +99,15 @@ async def complete( controls = dict(on_admitted=on_admitted, before_request=before_request, on_attempt_usage=on_attempt_usage, on_usage=on_usage) primary_controls = dict(controls) - if scenario == "failed-primary-attempt-callback-only": + if manual_primary_settlement or scenario == "failed-primary-attempt-callback-only": primary_controls.pop("on_usage") try: first = await super().complete(messages, **kwargs, **primary_controls) except ResponsesProtocolError: assert failed_primary else: + if manual_primary_settlement and on_usage is not None: + on_usage(replace(first.usage, cache_read_tokens=None)) if first.content and first.content.strip(): if independent_helper: await fallback.complete(messages, **kwargs) @@ -175,7 +178,7 @@ async def complete( @pytest.mark.parametrize("cache_read_tokens", [None, 0]) -@pytest.mark.parametrize("redact_at", ["response", "settlement"]) +@pytest.mark.parametrize("redact_at", ["response", "settlement", "wrapper"]) async def test_cache_redaction_preserves_budget_for_tool_followup(tmp_path, monkeypatch, cache_read_tokens, redact_at): (tmp_path / "facts.txt").write_text("The verified value is 42.\n", encoding="utf-8") monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) @@ -212,12 +215,18 @@ async def complete( def redact_usage(usage): on_usage(replace(usage, cache_read_tokens=cache_read_tokens)) - settlement_callback = redact_usage if redact_at == "settlement" and on_usage is not None else on_usage + settlement_callback = on_usage + if redact_at == "settlement" and on_usage is not None: + settlement_callback = redact_usage + elif redact_at == "wrapper": + settlement_callback = None controls = dict(on_admitted=on_admitted, before_request=before_request, on_attempt_usage=on_attempt_usage, on_usage=settlement_callback) kwargs.update((name, callback) for name, callback in controls.items() if callback is not None) response = await super().complete(messages, **kwargs) assert response.usage.cache_read_tokens == 1 + if redact_at == "wrapper" and on_usage is not None: + redact_usage(response.usage) if redact_at == "response": return replace(response, usage=replace(response.usage, cache_read_tokens=cache_read_tokens)) return response @@ -243,5 +252,5 @@ def redact_usage(usage): assert [entry["usage"]["total_tokens"] for entry in ledger] == [300, 337] usage_events = [event.data for event in events if event.type == "usage"] assert [event["total_tokens"] for event in usage_events] == [300, 337] - if redact_at == "settlement": + if redact_at != "response": assert [event["cache_read_tokens"] for event in usage_events] == [cache_read_tokens] * 2 From 1909e16560ba90c24a01e976d19076da2e8cd23a Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 21 Nov 2026 12:03:55 +0800 Subject: [PATCH 81/82] fix(session): retain recovery timestamps on restore retry Reuse durable timestamps for rows synthesized while restoring an export when retrying its existing autosave sequence. Keep the locked checkpoint comparison for every source row and for the complete recovered state. --- opencollab/application/session.py | 20 ++++- .../test_session_mixed_tool_snapshot.py | 9 ++- tests/agents/test_session_snapshot_context.py | 5 ++ tests/runtime/test_exported_session_resume.py | 80 ++++++++++++++++++- 4 files changed, 111 insertions(+), 3 deletions(-) diff --git a/opencollab/application/session.py b/opencollab/application/session.py index 7556a668..db1aff4f 100644 --- a/opencollab/application/session.py +++ b/opencollab/application/session.py @@ -788,7 +788,25 @@ def _checkpoint_restored_auto_save_target( sequence = _snapshot_nonnegative_int(snapshot.get("_autosave_sequence")) if isinstance(self.store, JournalSnapshotStorePort): sequence += 1 - self.store.checkpoint_snapshot(target_path, messages, meta=meta, sequence=sequence) + try: + self.store.checkpoint_snapshot(target_path, messages, meta=meta, sequence=sequence) + except SnapshotSequenceConflict: + saved_message_count = len(snapshot.get("messages", ())) + if saved_message_count >= len(messages) or not isinstance(self.store, SnapshotStorePort): + raise + previous = self.store.load_snapshot(target_path, self.agent.system_prompt) + previous_messages = previous["messages"] + if previous.get("_autosave_sequence") != sequence or len(previous_messages) != len(messages): + raise + # A retry regenerates recovery rows with a new wall-clock time. + # The checkpoint still compares their content and every source row. + timestamps = [message.get("timestamp") for message in previous_messages[saved_message_count:]] + if not all(isinstance(timestamp, str) for timestamp in timestamps): + raise + for message, timestamp in zip(messages[saved_message_count:], timestamps): + message["timestamp"] = timestamp + self.store.checkpoint_snapshot(target_path, messages, meta=meta, sequence=sequence) + restored.message_timestamps[saved_message_count:] = timestamps else: meta["_autosave_sequence"] = sequence self.store.save(target_path, messages, meta=meta) diff --git a/tests/agents/test_session_mixed_tool_snapshot.py b/tests/agents/test_session_mixed_tool_snapshot.py index 3723c5eb..62bd108a 100644 --- a/tests/agents/test_session_mixed_tool_snapshot.py +++ b/tests/agents/test_session_mixed_tool_snapshot.py @@ -120,7 +120,14 @@ async def pause(event): source.save(str(saved)) restored_workspace = tmp_path / "restored" shutil.copytree(workspace, restored_workspace) - recovered = load_session(str(saved), agent=agent, env=LocalEnvironment(str(restored_workspace)), llm=model) + recovered = load_session( + str(saved), agent=agent, env=LocalEnvironment(str(restored_workspace)), + llm=model, auto_save_path=str(tmp_path / "restored-autosave.json") if deferred else None, + ) + if deferred: + recovered_messages = recovered.state.enriched_messages() + recovered.restore(str(saved)) + assert recovered.state.enriched_messages() == recovered_messages tool_messages = [row for row in recovered.messages if row["role"] == "tool"] assert [row["tool_call_id"] for row in tool_messages] == [ diff --git a/tests/agents/test_session_snapshot_context.py b/tests/agents/test_session_snapshot_context.py index decadc0b..7fe2f3f8 100644 --- a/tests/agents/test_session_snapshot_context.py +++ b/tests/agents/test_session_snapshot_context.py @@ -299,7 +299,12 @@ async def hold_submission(event): restored = load_session( str(saved), agent=agent, env=LocalEnvironment(str(restored_workspace)), llm=model, max_steps=max_steps, + auto_save_path=str(tmp_path / "restored-autosave.json") if event_type == "tool_end" else None, ) + if event_type == "tool_end": + recovered_messages = restored.state.enriched_messages() + restored.restore(str(saved)) + assert restored.state.enriched_messages() == recovered_messages for session in (branch, restored): assert await session.run_loop() == summary assert session.phase is SessionPhase.DONE diff --git a/tests/runtime/test_exported_session_resume.py b/tests/runtime/test_exported_session_resume.py index 53785128..2d28e702 100644 --- a/tests/runtime/test_exported_session_resume.py +++ b/tests/runtime/test_exported_session_resume.py @@ -10,10 +10,12 @@ from opencollab.adapters.env import LocalEnvironment from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.storage import SessionStore +from opencollab.application.ports import SnapshotSequenceConflict from opencollab.bootstrap import build_session, load_session +from opencollab.domain import session as session_module from opencollab.domain.agent import Agent from opencollab.tools import builtin_tools -from tests.support.provider_sdk_http import completion_http_response, install_sdk_transport +from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport from tests.support.session_run_test_support import FakeLLM, llm_response, tool_call @@ -218,6 +220,82 @@ async def test_old_anthropic_export_can_resume_an_existing_autosave_twice(tmp_pa finally: await environment.cleanup() + +@pytest.mark.parametrize("conflict", [None, "source-timestamp", "recovered-content"]) +async def test_interrupted_tool_export_retries_the_same_autosave(tmp_path, monkeypatch, conflict): + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + exported, target = tmp_path / "interrupted.json", tmp_path / "automatic.json" + (tmp_path / "archive.txt").write_text("The archive opened on April 18.") + model = "claude-sonnet-5-5" + agent = Agent(name="reviewer", system_prompt="Read the archive.", model=model, + provider="anthropic", tools=builtin_tools("file_read", headless=False)) + env = LocalEnvironment(str(tmp_path)) + requests = [] + + def peer(request): + requests.append(json.loads(request.content)) + if len(requests) > 1: + return completion_http_response(request, text="Done") + return http_response(request, 200, json={ + "id": "msg_read", "type": "message", "role": "assistant", "model": model, + "content": [{"type": "tool_use", "id": "read-archive", "name": "file_read", + "input": {"path": "archive.txt"}}], + "stop_reason": "tool_use", "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 5}, + }) + + async def export_at_tool_start(event): + if event.type == "tool_start": + original.save(str(exported)) + + install_sdk_transport(monkeypatch, "anthropic", peer) + try: + async with LLMClient( + model=model, provider="anthropic", max_retries=0, + api_key="test-placeholder", base_url="https://provider.invalid/v1", # pragma: allowlist secret + ) as client: + options = {"agent": agent, "llm": client, "env": env} + original = build_session(**options, event_sink=export_at_tool_start) + try: + await original.add_user_message("Look up the date.") + assert await original.run_loop() == "Done" + finally: + await original.aclose() + + first_stamp = "2026-01-01T00:00:00+00:00" + monkeypatch.setattr(session_module, "_now_iso", lambda: first_stamp) + first = load_session(str(exported), **options, auto_save_path=str(target)) + await first.aclose() + stored = json.loads(target.read_text()) + assert stored["_autosave_sequence"] == 1 + assert stored["messages"][-1]["timestamp"] == first_stamp + assert stored["messages"][-1]["content"] == "Tool execution interrupted by session restore." + + if conflict == "source-timestamp": + stored["messages"][1]["timestamp"] = "2026-01-01T00:00:02+00:00" + elif conflict == "recovered-content": + stored["messages"][-1]["content"] = "A different result was confirmed." + target.write_text(json.dumps(stored)) + monkeypatch.setattr(session_module, "_now_iso", lambda: "2026-01-01T00:00:01+00:00") + + if conflict: + with pytest.raises(SnapshotSequenceConflict, match="conflicts with persisted snapshot state"): + load_session(str(exported), **options, auto_save_path=str(target)) + else: + resumed = load_session(str(exported), **options, auto_save_path=str(target)) + try: + assert resumed.state.enriched_messages() == stored["messages"] + assert resumed.state.anthropic_history is not None + finally: + await resumed.aclose() + assert json.loads(target.read_text()) == stored + assert len(requests) == 2 + finally: + await env.cleanup() + + class SnapshotOnlyStore: """Exercise full-snapshot persistence without journal support.""" From 404a5cd6581d200d375eaccfe59aebe206a636ef Mon Sep 17 00:00:00 2001 From: Kai Date: Sat, 21 Nov 2026 12:10:12 +0800 Subject: [PATCH 82/82] fix(tools): allow concurrent local edits to sibling files --- .../adapters/tools/_file_lock_worker.py | 44 +++++++- opencollab/adapters/tools/base.py | 40 ++++--- tests/tools/test_local_file_lock_workers.py | 100 ++++++++++++++++++ 3 files changed, 164 insertions(+), 20 deletions(-) diff --git a/opencollab/adapters/tools/_file_lock_worker.py b/opencollab/adapters/tools/_file_lock_worker.py index 0ea53bc1..8ceb9e1c 100644 --- a/opencollab/adapters/tools/_file_lock_worker.py +++ b/opencollab/adapters/tools/_file_lock_worker.py @@ -1,15 +1,53 @@ """Hold one ordered group of existing local file locks for a parent process.""" import contextlib +import errno +import fcntl import json import math +import os import select import sys import time +from collections.abc import Iterator from filelock import FileLock, Timeout +@contextlib.contextmanager +def _try_file_lock(path: str, *, shared: bool = False) -> Iterator[FileLock | None]: + if shared: + try: + fd = os.open(path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o666) + except FileNotFoundError as exc: + if os.path.isdir(os.path.dirname(path)): + raise Timeout(path) from exc + raise + try: + try: + fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise Timeout(path) from exc + except OSError as exc: + if exc.errno != errno.ENOSYS: + raise + else: + try: + # FileLock can unlink its inode between open and flock. + if not os.fstat(fd).st_nlink: + raise Timeout(path) + yield None + finally: + fcntl.flock(fd, fcntl.LOCK_UN) + return + finally: + # Other shared holders still coordinate through this inode. + os.close(fd) + + with FileLock(path, timeout=0) as lock: + yield lock + + def _reply(status: str, detail: str = "") -> None: print(json.dumps((status, detail)), flush=True) @@ -23,12 +61,11 @@ def main() -> None: owner.register(int(sys.argv[1]), select.POLLIN) try: with contextlib.ExitStack() as held: - for path, lock_path in paths: - lock = FileLock(lock_path, timeout=0) + for path, lock_path, shared in paths: deadline = time.monotonic() + timeout while not owner.poll(0): try: - lock.acquire(timeout=0) + held.enter_context(_try_file_lock(lock_path, shared=shared)) except Timeout: remaining = deadline - time.monotonic() if remaining <= 0: @@ -37,7 +74,6 @@ def main() -> None: if owner.poll(math.ceil(min(poll, remaining) * 1000)): return else: - held.callback(lock.release) break else: return diff --git a/opencollab/adapters/tools/base.py b/opencollab/adapters/tools/base.py index 0043ab0a..7b661676 100644 --- a/opencollab/adapters/tools/base.py +++ b/opencollab/adapters/tools/base.py @@ -19,6 +19,7 @@ from filelock import FileLock, Timeout +from opencollab.adapters.tools._file_lock_worker import _try_file_lock from opencollab.application.async_timeout import await_owned_operation from opencollab.application.tool_execution import DeferredCall, ToolRuntime @@ -84,11 +85,13 @@ async def _reap_lock_holders(processes: list[subprocess.Popen]) -> None: await asyncio.sleep(HOST_WRITE_LOCK_POLL_SECONDS) -async def _prepare_lock_holder(channel: socket.socket, paths: Sequence[str], env: Any) -> None: +async def _prepare_lock_holder( + channel: socket.socket, paths: Sequence[str], env: Any, shared_paths: frozenset[str], +) -> None: loop = asyncio.get_running_loop() channel.setblocking(False) options = ( - [(path, _host_lock_path(path, env)) for path in paths], + [(path, _host_lock_path(path, env), path in shared_paths) for path in paths], HOST_WRITE_LOCK_TIMEOUT_SECONDS, HOST_WRITE_LOCK_POLL_SECONDS, ) await loop.sock_sendall(channel, json.dumps(options).encode() + b"\n") @@ -107,7 +110,9 @@ async def _prepare_lock_holder(channel: socket.socket, paths: Sequence[str], env @contextlib.asynccontextmanager -async def _hold_lock_groups(paths: Sequence[str], env: Any, group_size: int) -> AsyncIterator[None]: +async def _hold_lock_groups( + paths: Sequence[str], env: Any, group_size: int, shared_paths: frozenset[str], +) -> AsyncIterator[None]: owner = asyncio.current_task() assert owner is not None processes: list[subprocess.Popen] = [] @@ -140,7 +145,7 @@ async def watch_exit() -> None: [sys.executable, worker, str(read_end if shared else 0)], start_new_session=True, stdin=child, stdout=child, pass_fds=(read_end,) if shared else (), )) - await _prepare_lock_holder(setup, paths[start:start + group_size], env) + await _prepare_lock_holder(setup, paths[start:start + group_size], env, shared_paths) finally: if shared: setup.close() @@ -161,14 +166,16 @@ async def watch_exit() -> None: @contextlib.asynccontextmanager -async def _local_write_locks(paths: Sequence[str], env: Any) -> AsyncIterator[FileLock | None]: +async def _local_write_locks( + paths: Sequence[str], env: Any, *, shared_paths: frozenset[str] = frozenset(), +) -> AsyncIterator[FileLock | None]: capacity = _local_file_lock_capacity() if len(paths) == 1 or len(paths) <= _available_local_file_locks(capacity): async with contextlib.AsyncExitStack() as locks: target = None try: for path in paths: - target = await locks.enter_async_context(_target_write_lock(path, env)) + target = await locks.enter_async_context(_target_write_lock(path, env, shared=path in shared_paths)) except OSError as exc: if exc.errno != errno.EMFILE: raise @@ -177,7 +184,7 @@ async def _local_write_locks(paths: Sequence[str], env: Any) -> AsyncIterator[Fi return # Every holder uses the same per-path locks. Contention order stays the # same across groups, direct callers and older installations. - async with _hold_lock_groups(paths, env, min(512, capacity)): + async with _hold_lock_groups(paths, env, min(512, capacity), shared_paths): yield None @@ -206,12 +213,18 @@ async def host_write_locks(paths: Sequence[str], env: Any) -> AsyncIterator[Any] @contextlib.asynccontextmanager async def host_write_lock(path: str, env: Any) -> AsyncIterator[Any]: + if env is None or getattr(env, "local_filesystem", False): + paths = _write_lock_paths((path,), env) + # The single target sorts after its ancestors. + async with _local_write_locks(paths, env, shared_paths=frozenset(paths[:-1])) as target: + yield target + return async with host_write_locks((path,), env) as target: yield target @contextlib.asynccontextmanager -async def _target_write_lock(path: str, env: Any) -> AsyncIterator[Any]: +async def _target_write_lock(path: str, env: Any, *, shared: bool = False) -> AsyncIterator[Any]: if env is not None and not getattr(env, "local_filesystem", False): environment_lock = getattr(env, "file_write_lock", None) if callable(environment_lock): @@ -221,15 +234,13 @@ async def _target_write_lock(path: str, env: Any) -> AsyncIterator[Any]: yield None return - lock = FileLock(_host_lock_path(path, env), timeout=0) + lock_path = _host_lock_path(path, env) loop = asyncio.get_running_loop() deadline = loop.time() + HOST_WRITE_LOCK_TIMEOUT_SECONDS - acquired = False - try: + with contextlib.ExitStack() as held: while True: try: - lock.acquire(timeout=0) - acquired = True + lock = held.enter_context(_try_file_lock(lock_path, shared=shared)) break except Timeout: remaining = deadline - loop.time() @@ -237,9 +248,6 @@ async def _target_write_lock(path: str, env: Any) -> AsyncIterator[Any]: raise TimeoutError(f"timed out acquiring write lock for {path}") await asyncio.sleep(min(HOST_WRITE_LOCK_POLL_SECONDS, remaining)) yield lock - finally: - if acquired: - lock.release() class Tool: """Base class for all tools. Subclass and implement ``execute_with_runtime``. diff --git a/tests/tools/test_local_file_lock_workers.py b/tests/tools/test_local_file_lock_workers.py index 5a25ca1f..b18657de 100644 --- a/tests/tools/test_local_file_lock_workers.py +++ b/tests/tools/test_local_file_lock_workers.py @@ -1,6 +1,8 @@ from __future__ import annotations import asyncio +import errno +import fcntl import os import resource import signal @@ -14,10 +16,108 @@ from opencollab.adapters.env import LocalEnvironment from opencollab.adapters.tools import base +from opencollab.adapters.tools._file_lock_worker import _try_file_lock from opencollab.adapters.tools.fs import FileWriteTool from opencollab.application.tool_execution import ToolRuntime +@pytest.mark.parametrize("use_workers", [False, True], ids=["direct", "worker"]) +async def test_native_sibling_writes_share_ancestors_and_exclude_transactions(tmp_path, monkeypatch, use_workers): + if use_workers: + monkeypatch.setattr(base, "_local_file_lock_capacity", lambda: 2) + monkeypatch.setattr(base, "HOST_WRITE_LOCK_TIMEOUT_SECONDS", 0.25) + directory = tmp_path / "nested" + directory.mkdir() + for name in ("one.txt", "two.txt", "three.txt"): + (directory / name).write_text("before\n") + env = LocalEnvironment(str(tmp_path)) + runtime = ToolRuntime(environment=env, safety_policy=None, permission_policy=None) + transaction_paths = ("nested", "other.txt", "unrelated.txt") + started, finish = asyncio.Event(), asyncio.Event() + read_file = env.read_file + + async def pause_first_read(path): + if Path(path).name == "one.txt" and not started.is_set(): + started.set() + await finish.wait() + return await read_file(path) + + monkeypatch.setattr(env, "read_file", pause_first_read) + + async def write(name): + return await FileWriteTool().execute_with_runtime( + {"path": f"nested/{name}", "mode": "str_replace", "old_str": "before", "new_str": "after"}, runtime, + ) + + first = asyncio.create_task(write("one.txt")) + try: + try: + await asyncio.wait_for(started.wait(), 5) + await asyncio.wait_for(write("two.txt"), 5) + assert (directory / "two.txt").read_text() == "after\n" + assert (directory / "one.txt").read_text() == "before\n" + with pytest.raises(TimeoutError, match="one.txt"): + await write("one.txt") + with pytest.raises(TimeoutError, match="nested$"): + async with base.host_write_locks(transaction_paths, env): + pytest.fail("transaction acquired a directory with an active child edit") + finally: + finish.set() + await first + + assert (directory / "one.txt").read_text() == "after\n" + async with base.host_write_locks(transaction_paths, env): + with pytest.raises(TimeoutError, match="nested$"): + await write("three.txt") + assert (directory / "three.txt").read_text() == "before\n" + await write("three.txt") + assert (directory / "three.txt").read_text() == "after\n" + finally: + await env.cleanup() + + +def test_shared_flock_rejects_inode_unlinked_before_acquisition(tmp_path, monkeypatch): + path = tmp_path / "directory.lock" + acquire_exclusive = FileLock(path, timeout=0) + acquire_exclusive.acquire() + open_file = os.open + + def open_then_release(*args, **kwargs): + fd = open_file(*args, **kwargs) + path.unlink() + acquire_exclusive.release() + return fd + + try: + with monkeypatch.context() as patch: + patch.setattr(os, "open", open_then_release) + with pytest.raises(Timeout), _try_file_lock(str(path), shared=True): + pytest.fail("shared lock acquired an unlinked inode") + with _try_file_lock(str(path), shared=True): + with pytest.raises(Timeout), acquire_exclusive: + pytest.fail("legacy writer acquired an active shared inode") + finally: + acquire_exclusive.release() + + +def test_unavailable_shared_flock_uses_exclusive_file_lock(tmp_path, monkeypatch): + path = tmp_path / "directory.lock" + flock = fcntl.flock + + def exclusive_only(fd, mode): + if mode & fcntl.LOCK_SH: + raise OSError(errno.ENOSYS, "shared flock unavailable") + return flock(fd, mode) + + monkeypatch.setattr(fcntl, "flock", exclusive_only) + acquire_exclusive = FileLock(path, timeout=0) + with _try_file_lock(str(path), shared=True): + with pytest.raises(Timeout), acquire_exclusive: + pytest.fail("legacy writer acquired a fallback lock") + with acquire_exclusive: + pass + + async def test_large_group_acquires_existing_flocks_in_path_order(tmp_path, monkeypatch): monkeypatch.setattr(base, "_local_file_lock_capacity", lambda: 2) monkeypatch.setattr(base, "HOST_WRITE_LOCK_TIMEOUT_SECONDS", 0.2)