diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab43abe3..8259d51f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,8 @@ concurrency: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 40 + # Allow the full suite and the Python 3.12 DOM checks to finish on hosted runners. + timeout-minutes: 45 strategy: fail-fast: false matrix: diff --git a/docs/duo.md b/docs/duo.md index df36840a..b9591e64 100644 --- a/docs/duo.md +++ b/docs/duo.md @@ -204,6 +204,15 @@ Git candidates start with the source's current tracked files and untracked files that aren't ignored. Their delivered patches describe the subsequent candidate edits. Adoption applies that increment to the current source files and preserves independent user edits and the source index. A conflicting patch leaves the current source in place. +If a partially applied patch is rolled back, OpenCollab retains detached files that +another process may still be writing to. The error or warning gives the +`candidate-recovery-*` directory, typically in the common Git directory. +When shared Git metadata is read-only, OpenCollab prefers a writable location +on the same filesystem outside the worktree so the files survive worktree removal. +When present, `volumes/` records list additional recovery directories on affected +mounts. The adjacent `path` or `.path` record identifies each file's original +location. Close the writers and recover their data before removing the recovery +directories or worktree. Changes to ignore rules preserve files copied from the source and files added to the candidate index or commits. Newly created ignored files join delivery when explicitly added with `git add --force`. diff --git a/docs/duo/README.zh-CN.md b/docs/duo/README.zh-CN.md index 213f294a..58e22b6e 100644 --- a/docs/duo/README.zh-CN.md +++ b/docs/duo/README.zh-CN.md @@ -89,6 +89,8 @@ Duo 通过现有候选工作区接口完成隔离、变更捕获与采用。默 Git 候选从源目录当前受跟踪文件和未被忽略的未跟踪文件开始,交付补丁表示候选随后产生的增量。采用时将增量应用到当前源文件,保留独立用户修改和源索引,冲突会保留当前源目录。修改忽略规则后,已从源目录复制的文件和候选中已加入索引或提交的文件仍进入交付。新建的忽略文件经 `git add --force` 加入后进入交付。 +补丁部分应用后回滚时,OpenCollab 会保留其他进程可能仍在写入的文件。错误或警告会给出 `candidate-recovery-*` 目录,通常位于 Git 公共目录下。Git 公共目录只读时,OpenCollab 优先选择同一文件系统上工作树外的可写目录,让恢复文件在移除工作树后得以保留。如果包含 `volumes/`,该目录中的记录列出受影响挂载卷上的其他恢复目录。相邻的 `path` 或 `.path` 记录标明文件原来的位置。移除恢复目录或工作树前,请等待写入进程关闭文件,并恢复其中的数据。 + 候选运行期间源目录发生变化,运行时会报告错误并保留候选工作树。错误包含初始 Git tree,工作树中的 `refs/worktree/opencollab-source` 继续引用它,可以通过 `git show :` 恢复文件。后端提供源版本时,`CandidateRun.source_revision` 记录源 `HEAD`,采用前再次核对。源 diff 相同而 `HEAD` 已变化时,采用也会被拒绝。 本地 Git 候选初始化源目录中已有的干净子模块,包含选定工作区内的嵌套依赖。尚未初始化的可选子模块保持空目录。源子模块的变化会在候选获取前报告。候选修改子模块文件、提交或 gitlink 时,`CandidateCaptureError` 会保留完整工作树供恢复。仓库补丁后端交付父仓库的改动,子模块修改需要对应的交付后端。 diff --git a/docs/session-isolation.md b/docs/session-isolation.md index b1584e3c..524c5b29 100644 --- a/docs/session-isolation.md +++ b/docs/session-isolation.md @@ -241,3 +241,7 @@ to run its child agents in independent worktrees inside its own VM. Individual child calls can select `isolation="vm-session"` to request another guest. The [executable VM example](../examples/vm-authored-workflow/README.md) starts with a member writing source and finishes by collecting the native child result. + +The `budget` on `launch` is the child's total token allowance, shared across its +sessions. The host checks the remaining allowance before each model request, +including calls made with `over_budget_ok=True` inside the child. diff --git a/docs/session-isolation.zh-CN.md b/docs/session-isolation.zh-CN.md index e573dc4c..e55b72d1 100644 --- a/docs/session-isolation.zh-CN.md +++ b/docs/session-isolation.zh-CN.md @@ -174,3 +174,6 @@ Team 成员之间的通信继续遵循配置中的拓扑。 省略时继承成员策略。成员使用 `{"action":"launch","source":"generated.py","isolation":"worktree"}` 可让子 Agent 在自己的 VM 中使用独立工作树。单个子调用可通过 `isolation="vm-session"` 申请另一台来宾。 [可执行 VM 示例](../examples/vm-authored-workflow/README.zh-CN.md) 从成员写出源码开始,执行到取得原生子运行结果。 + +`launch` 的 `budget` 是子运行各 Session 共享的总额度。控制端在每个模型请求开始前检查剩余额度, +子代码使用 `over_budget_ok=True` 的调用也受该额度约束。 diff --git a/opencollab/adapters/_candidate_adoption.py b/opencollab/adapters/_candidate_adoption.py new file mode 100644 index 00000000..286bd25b --- /dev/null +++ b/opencollab/adapters/_candidate_adoption.py @@ -0,0 +1,813 @@ +"""Restore physical working files after a failed candidate publication.""" + +from __future__ import annotations + +import logging +import posixpath +import shlex +import sys +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from contextlib import AsyncExitStack, asynccontextmanager +from typing import Any + +from opencollab.adapters._env_base import ENV_FILE_WRITE_LIMIT_BYTES +from opencollab.adapters._git_patch_bytes import _complete, _patch_temporary +from opencollab.application.async_timeout import await_owned_operation +from opencollab.application.exception_notes import add_exception_note +from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError + +logger = logging.getLogger(__name__) + +_COMPARE_ENTRIES = r''' +present() { [ -e "$1" ] || [ -L "$1" ]; } +mode() { stat -c %a -- "$1" 2>/dev/null || stat -f %Lp -- "$1"; } + +same_entry() { + if [ -L "$1" ]; then + [ -L "$2" ] || return 1 + left=$(readlink "$1" && printf .) && right=$(readlink "$2" && printf .) && + [ "$left" = "$right" ] + elif [ -f "$1" ]; then + [ ! -L "$2" ] && [ -f "$2" ] || return 1 + left=$(mode "$1") || return 1 + right=${3:-} + if [ -z "$right" ]; then right=$(mode "$2") || return 1; fi + [ "$left" = "$right" ] || return 1 + if command -v cmp >/dev/null 2>&1; then + cmp -s -- "$1" "$2" + else + git diff --no-index --quiet --no-ext-diff --no-textconv -- "$1" "$2" + fi + else + ! present "$1" && ! present "$2" + fi +} + +conflict() { printf 'concurrent change at %s\n' "$1" >&2; return 2; } +''' + +_CHECK_PARENTS = r''' +directory() { [ -d "$1" ] && [ ! -L "$1" ]; } + +check_parents() { + if [ "$1" = "$workspace" ]; then directory "$workspace" || conflict "$workspace"; return; fi + parent=${1%/*} + while [ "$parent" != "$workspace" ] && [ -n "$parent" ]; do + directory "$parent" || { conflict "$parent"; return; } + parent=${parent%/*} + done + directory "$workspace" || conflict "$workspace" +} +''' + +_FILESYSTEM = r''' +filesystem() { stat -c '%d:%m' -- "$1" 2>/dev/null || stat -f %d -- "$1"; } +''' + +_RECOVERY_STORES = _FILESYSTEM + r''' +store_for() ( + parent=${1%/*} + filesystem_id=$(filesystem "$parent") && primary_id=$(filesystem "$recovery") || exit + if present "$1"; then + entry_id=$(filesystem "$1") || exit + if [ "$entry_id" != "$filesystem_id" ]; then + printf 'cannot hold the mounted entry %s\n' "$1" >&2 + exit 1 + fi + fi + if [ "$filesystem_id" = "$primary_id" ]; then printf '%s' "$recovery"; exit; fi + + # Prefer the mount root so Git can remove empty directories below it. + anchor= + while directory "$parent"; do + [ -w "$parent" ] && anchor=$parent + [ "$parent" != "$workspace" ] && [ "$parent" != / ] || break + parent=${parent%/*} + parent_id=$(filesystem "${parent:-/}") || exit + [ "$parent_id" = "$filesystem_id" ] || break + done + if [ -z "$anchor" ]; then + printf 'no writable recovery directory on the filesystem of %s\n' "$1" >&2 + exit 1 + fi + for location in "$recovery"/volumes/*; do + [ -f "$location" ] || continue + local_store=$(cat -- "$location" && printf .) || exit + local_store=${local_store%.} + [ "${local_store%/*}" = "$anchor" ] || continue + directory "$local_store" && [ "$(filesystem "$local_store")" = "$filesystem_id" ] || exit 1 + printf '%s' "$local_store" + exit + done + mkdir -p -m 700 -- "$recovery/volumes" || exit + location=$(mktemp "$recovery/volumes/volume-XXXXXXXX") || exit + local_store=$(mktemp -d "$anchor/.candidate-recovery-XXXXXXXX") || exit + printf '%s' "$local_store" > "$location" || { rmdir -- "$local_store"; exit 1; } + printf '%s' "$local_store" +) + +each_store() { + for storage_record in "$recovery"/volumes/*; do + [ -f "$storage_record" ] || continue + storage_path=$(cat -- "$storage_record" && printf .) || return 1 + storage_path=${storage_path%.} + [ -n "$storage_path" ] || continue + "$@" "$storage_path" || return $? + done + "$@" "$recovery" +} +''' + +_LINK_ORIGINALS = """\ +import os +import sys +from pathlib import Path + +for record in Path(sys.argv[1]).glob("[0-9]*.path"): + try: + os.link(record.read_bytes(), record.with_suffix(""), follow_symlinks=False) + except OSError: + sys.exit(1) +""" + +_HOLD_ORIGINALS = _COMPARE_ENTRIES + _CHECK_PARENTS + _RECOVERY_STORES + r''' +hold_original() { + if [ "$1" -ef "$2" ] || ln -P -- "$1" "$2" 2>/dev/null; then return 0; fi + store=$(store_for "$1") || return 1 + if [ "$store" != "$recovery" ]; then + mkdir -p -m 700 -- "$store/original-inodes" || return 1 + held="$store/original-inodes/${2##*/}" + printf '%s' "$1" > "$held.path" && rm -- "$2.path" || return 1 + if ln -P -- "$1" "$held" 2>/dev/null; then return 0; fi + fi + mkdir -p -m 700 -- "$store/replacements" && + cp -pP -- "$1" "$store/replacements/${2##*/}" || return 1 + needs_activation=1 +} + +hold_originals() { + index=0 + needs_activation=0 + mkdir -m 700 -- "$recovery/original-inodes" || return 1 + for path; do + source=$workspace_prefix$path + if check_parents "$source" 2>/dev/null && [ -f "$source" ] && [ ! -L "$source" ]; then + entry=$recovery/original-inodes/$index + printf '%s' "$source" > "$entry.path" || return 1 + if [ -z "$linker" ]; then hold_original "$source" "$entry" || return 1; fi + index=$((index + 1)) + fi + done + if [ -n "$linker" ] && ! "$linker" -I -S -c "$link_script" "$recovery/original-inodes"; then + for record in "$recovery"/original-inodes/*.path; do + [ -f "$record" ] || continue + source=$(cat -- "$record" && printf .) || return 1 + hold_original "${source%.}" "${record%.path}" || return 1 + done + fi + if [ "$needs_activation" -eq 1 ]; then printf 'activate-originals\n'; fi +} +''' + +_RESTORE_SAVED = r''' +restore_saved() { + if ! present "$1"; then return 0; fi + if present "$2"; then conflict "$2"; return; fi + mv -n -- "$1" "$2" || return 1 + if present "$1"; then conflict "$2"; fi +} +''' + +_ACTIVATE_ORIGINALS = _COMPARE_ENTRIES + _CHECK_PARENTS + _RESTORE_SAVED + _RECOVERY_STORES + r''' +activate_store() { + for replacement in "$1"/replacements/*; do + [ -f "$replacement" ] || continue + entry=$1/original-inodes/${replacement##*/} + live=$(cat -- "$entry.path" && printf .) || return 1 + live=${live%.} + check_parents "$live" && same_entry "$replacement" "$live" || { conflict "$live"; return; } + restore_saved "$live" "$entry" || return $? + if ! same_entry "$replacement" "$entry"; then + restore_saved "$entry" "$live" || return $? + conflict "$live" + return + fi + restore_saved "$replacement" "$live" || return $? + done +} +each_store activate_store +''' + +_UNDO_INCOMPLETE_ACTIVATION = _COMPARE_ENTRIES + _CHECK_PARENTS + _RESTORE_SAVED + _RECOVERY_STORES + r''' +undo_store() { + for replacement in "$1"/replacements/*; do + entry=$1/original-inodes/${replacement##*/} + [ -f "$replacement" ] && [ -f "$entry" ] || continue + live=$(cat -- "$entry.path" && printf .) || return 1 + live=${live%.} + check_parents "$live" && restore_saved "$entry" "$live" || return $? + done +} +each_store undo_store || exit +''' + +_DETACHED_FILES = r''' +detached=0 +parked=0 +collect_detached() { + for record in "$1"/original-inodes/*.path; do + [ -f "$record" ] || continue + path=$(cat -- "$record" && printf .) || return 1 + path=${path%.} + entry=${record%.path} + if ! present "$entry"; then + rm -- "$record" || return 1 + elif check_parents "$path" 2>/dev/null && [ ! -L "$path" ] && [ "$path" -ef "$entry" ]; then + rm -- "$record" "$entry" || return 1 + else + detached=1 + fi + done + for entry in "$1"/observed-*/entry; do + if [ -f "$entry" ] && [ ! -L "$entry" ]; then parked=1; break; fi + done +} +each_store collect_detached || exit +if [ "$parked" -eq 1 ]; then printf 'parked\n'; fi +if [ "$detached" -eq 1 ]; then printf 'original-detached\n'; fi +''' + +_RECOVERY_PARENT = _FILESYSTEM + r''' +writable_directory() { [ -d "$1" ] && [ -w "$1" ] && [ -x "$1" ]; } + +recovery_temporary() { + temporary=$(cd -P -- "${TMPDIR:-/tmp}" >/dev/null 2>&1 && pwd -P && printf .) || return 1 + temporary=${temporary%??} + case "$temporary/" in "${1%/}/"*|"${2%/}/"*) return 1;; esac + printf '%s\n' "$temporary" +} + +recovery_parent() { + worktree_device=$(filesystem .) && storage_device=$(filesystem "$1/") || return 1 + if [ "$worktree_device" = "$storage_device" ] && writable_directory "$1"; then + printf '%s\n' "$1" + return + fi + workspace=$(pwd -P && printf .) || return 1 + workspace=${workspace%??} + parent=$workspace + while [ "$parent" != / ]; do + parent=${parent%/*} + parent=${parent:-/} + parent_device=$(filesystem "$parent") || return 1 + [ "$worktree_device" = "$parent_device" ] || break + if writable_directory "$parent"; then + printf '%s\n' "$parent" + return + fi + done + if [ -n "${2:-}" ]; then + temporary=$(recovery_temporary "$workspace" "$2" && printf .) || temporary= + temporary=${temporary%??} + if writable_directory "$temporary" && [ "$(filesystem "$temporary")" = "$worktree_device" ]; then + printf '%s\n' "$temporary" + return + fi + fi + if writable_directory "$1"; then + printf '%s\n' "$1" + else + printf '%s\n' "${2:-$1}" + fi +} +''' + +_RECOVERY_GIT_DIRECTORIES = r''' +private=$(git rev-parse --absolute-git-dir && printf .) || exit +private=${private%??} +common=$(git rev-parse --git-common-dir && printf .) || exit +common=${common%??} +''' + + +async def _mounted_recovery_paths(environment: Any, workspace: str) -> tuple[str, ...]: + # Permissions may have changed since a recovery backup chose its parent. + command = f"cd -- {shlex.quote(workspace)} || exit\n" + _RECOVERY_PARENT + _RECOVERY_GIT_DIRECTORIES + r''' +workspace=$(pwd -P && printf .) || exit +workspace=${workspace%??} +prefix=${workspace%/}/ +collect_recovery_paths() { + for backup in "$1"/candidate-recovery-????????; do + [ -d "$backup" ] && [ ! -L "$backup" ] && [ -O "$backup" ] || continue + volumes=$backup/volumes + [ -d "$volumes" ] && [ ! -L "$volumes" ] && [ -O "$volumes" ] || continue + for receipt in "$volumes"/volume-????????; do + [ -f "$receipt" ] && [ ! -L "$receipt" ] && [ -O "$receipt" ] || continue + store=$(cat -- "$receipt" && printf .) || continue + store=${store%.} + case "$store" in "$prefix"*) ;; *) continue;; esac + case "${store##*/}" in .candidate-recovery-????????) ;; *) continue;; esac + [ -d "$store" ] && [ ! -L "$store" ] && [ -O "$store" ] || continue + anchor=${store%/*} + top=$(cd -P -- "${anchor:-/}" && [ "$PWD" = "${anchor:-/}" ] && + GIT_DISCOVERY_ACROSS_FILESYSTEM=1 git rev-parse --show-toplevel 2>/dev/null && + printf .) || continue + [ "${top%??}" = "$workspace" ] || continue + printf '%s\0' "${store#"$prefix"}" + done + done +} + +collect_recovery_paths "$common" +if [ ! "$private" -ef "$common" ]; then collect_recovery_paths "$private"; fi +parent=$workspace +while [ "$parent" != / ]; do + parent=${parent%/*} + parent=${parent:-/} + if [ ! "$parent" -ef "$common" ] && [ ! "$parent" -ef "$private" ]; then + collect_recovery_paths "$parent" + fi +done +if temporary=$(recovery_temporary "$workspace" "$private" && printf .); then + collect_recovery_paths "${temporary%??}" +fi +''' + result = _complete(await environment.exec_cmd(command, timeout=30), "mounted candidate recovery locations") + return tuple(dict.fromkeys(path for path in result.split("\0") if path)) + + +_RECOVER_PATHS = _COMPARE_ENTRIES + _CHECK_PARENTS + _RESTORE_SAVED + _RECOVERY_STORES + r''' + +check_candidate_descendants() { + prefix=${1#"$workspace_prefix"}; shift + for path; do + case "$path" in + "$prefix"/*) + target=$workspace_prefix$path + if check_parents "$target" 2>/dev/null && present "$target"; then + conflict "$target" + return + fi + ;; + esac + done +} + +prepare_parent() { + if [ "$1" != "$workspace" ]; then check_parents "$1" || return $?; fi + if ! present "$1"; then + if directory "$5"; then conflict "$1"; return; fi + mkdir -m 700 -- "$1" && printf '700 %s\n' "$2" > "$3" + elif ! directory "$1"; then + conflict "$1" + else + observed=$(mode "$1") || return 1 + restore=$observed + if directory "$5"; then + projected=$(mode "$5") || return 1 + if [ "$observed" = "$projected" ]; then restore=$2; fi + fi + temporary=$observed + if [ "$4" = write ] && [ ! -w "$1" ] && [ -O "$1" ]; then + chmod -- u+w "$1" && temporary=$(mode "$1") || return 1 + fi + if [ "$temporary" != "$restore" ]; then printf '%s %s\n' "$temporary" "$restore" > "$3"; fi + fi +} + +reset_parent_mode() { + if [ -f "$2" ]; then + check_parents "$1" && directory "$1" || { conflict "$1"; return; } + read -r temporary observed < "$2" || return 1 + current=$(mode "$1") || return 1 + if [ "$current" = "$temporary" ] && [ "$current" != "$observed" ]; then + chmod -- "$observed" "$1" + fi + fi +} + +park_candidate_entry() { + if ! present "$1"; then return 0; fi + parked=$(mktemp -d "$3/observed-XXXXXXXX") || return 1 + if [ -n "$4" ]; then printf '%s\n' "$4" > "$parked/mode" || return 1; fi + printf '%s' "$1" > "$parked/path" && mv -- "$1" "$parked/entry" || return 1 + if ! same_entry "$parked/entry" "$2" "$4"; then + restore_saved "$parked/entry" "$1" || return $? + conflict "$1" + fi +} + +project_file_mode() ( + source=$(mktemp "$recovery/mode-XXXXXXXX") || exit + expected_mode=$(mode "$1") && chmod -- "$expected_mode" "$source" || exit + sample=$(mktemp -d "$2/mode-XXXXXXXX") || exit + cp -pP -- "$source" "$sample/entry" && actual_mode=$(mode "$sample/entry") || exit + rm -- "$source" && rm -rf -- "$sample" || exit + printf '%s' "$actual_mode" +) + +restore_preimage() { + if ! present "$1"; then return 0; fi + store=${3:-} + if [ -z "$store" ]; then store=$(store_for "$2") || return 1; fi + if [ "$store" = "$recovery" ]; then restore_saved "$1" "$2"; return; fi + install=$(mktemp -d "$store/restore-XXXXXXXX") || return 1 + cp -pPR -- "$1" "$install/entry" && restore_saved "$install/entry" "$2" +} + +restore_entry() ( + old=$1 expected=$2 live=$3 + if same_entry "$old" "$expected"; then + # An unchanged replay entry cannot explain a changed patch target. + if candidate_path "$live" && ! same_entry "$old" "$live"; then conflict "$live"; return; fi + if directory "$live"; then candidate_descendants "$live"; return; fi + return 0 + fi + if directory "$live"; then + if ! directory "$old" && ! directory "$expected"; then + conflict "$live" + return + fi + result=0 + if directory "$expected"; then + for child in "$expected"/* "$expected"/.[!.]* "$expected"/..?*; do + present "$child" || continue + name=${child##*/} + if ! directory "$old" || ! present "$old/$name"; then + restore_entry "$absent" "$child" "$live/$name" || result=$? + fi + done + fi + if directory "$old"; then + for child in "$old"/* "$old"/.[!.]* "$old"/..?*; do + present "$child" || continue + name=${child##*/} + after=$absent + if directory "$expected"; then after=$expected/$name; fi + restore_entry "$child" "$after" "$live/$name" || result=$? + done + fi + [ "$result" -eq 0 ] || return "$result" + if ! directory "$old"; then + for child in "$live"/* "$live"/.[!.]* "$live"/..?*; do + present "$child" || continue + if present "$old"; then conflict "$live"; fi + return + done + rmdir -- "$live" || return 1 + restore_preimage "$old" "$live" + fi + return + fi + if same_entry "$live" "$old"; then return 0; fi + projected= + if same_entry "$live" "$expected"; then + store=$(store_for "$live") || return 1 + else + if [ ! -f "$live" ] || [ -L "$live" ] || [ ! -f "$expected" ] || [ -L "$expected" ]; then + conflict "$live" + return + fi + store=$(store_for "$live") || return 1 + if [ "$store" = "$recovery" ]; then conflict "$live"; return; fi + # A replay on the Git filesystem can represent modes unavailable on the live mount. + projected=$(project_file_mode "$expected" "$store") || return 1 + same_entry "$live" "$expected" "$projected" || { conflict "$live"; return; } + fi + park_candidate_entry "$live" "$expected" "$store" "$projected" || return $? + restore_preimage "$old" "$live" "$store" +) +''' + +_RECHECK_PARKED = _COMPARE_ENTRIES + r''' +verify_parked() { + for parked in "$1"/observed-*; do + [ -d "$parked" ] || continue + path=$(cat -- "$parked/path" && printf .) || return 1 + path=${path%.} + projected= + if [ -f "$parked/mode" ]; then read -r projected < "$parked/mode" || return 1; fi + same_entry "$parked/entry" "$staged/${path#"$workspace_prefix"}" "$projected" || { conflict "$path"; return; } + done +} +each_store verify_parked || exit +''' + + +@asynccontextmanager +async def _adoption_command( + environment: Any, workspace: str, command: str, *, repository: bool, +) -> AsyncIterator[str]: + size = len(command.encode("utf-8", errors="surrogateescape")) + if size <= 32_768: + yield command + return + resources = AsyncExitStack() + + async def stage(content: str) -> str: + return await resources.enter_async_context(_patch_temporary( + environment, workspace, content, prefix=".candidate-command-", suffix=".sh", + cleanup_best_effort=True, repository=repository, + )) + + try: + content = command + "\n" + if size < ENV_FILE_WRITE_LIMIT_BYTES: + script = await stage(content) + else: + # Keep each transfer within the file and bridge limits. Assemble + # the complete script before it can change the workspace. + chunk_size = ENV_FILE_WRITE_LIMIT_BYTES // 8 + script = await stage("") + for start in range(0, len(content), chunk_size): + fragment = await stage(content[start:start + chunk_size]) + _complete(await environment.exec_cmd( + f"cat -- {shlex.quote(fragment)} >> {shlex.quote(script)}", timeout=120, + ), "candidate command staging") + yield f"sh {shlex.quote(script)}" + finally: + try: + await await_owned_operation(resources.aclose()) + except Exception: + pass + + +@asynccontextmanager +async def _adoption_transaction( + environment: Any, workspace: str, paths: Sequence[str], *, repository: bool, + replay: Callable[[str, str | None, str | None], Awaitable[None]], + attributes_file: str | None = None, +) -> AsyncIterator[None]: + for path in paths: + if ( + not path or "\0" in path or posixpath.normpath(path) != path or posixpath.isabs(path) + or path in {".", ".."} or path.startswith("../") + or any(part.casefold() == ".git" for part in path.split("/")) + ): + raise RuntimeError(f"invalid path in candidate patch: {path!r}") + # Own a missing or non-directory ancestor too, so rollback also reverses + # directory creation and file/directory replacements without following links. + inspection = ( + f"cd -- {shlex.quote(workspace)} && " + f"for path in {shlex.join(paths)}; do root=$path; parent=${{path%/*}}; " + 'while [ "$parent" != "$path" ]; do ' + 'if [ ! -d "$parent" ] || [ -L "$parent" ]; then root=$parent; fi; ' + 'path=$parent; parent=${path%/*}; done; printf "%s\\0" "$root"; done' + ) + async with _adoption_command(environment, workspace, inspection, repository=repository) as command: + inspected = _complete(await environment.exec_cmd(command, timeout=120), "candidate adoption path inspection") + roots: list[str] = [] + for path in sorted(set(inspected.split("\0")) - {""}): + if not any(path.startswith(root + "/") for root in roots): + roots.append(path) + ancestors = {"."} + for path in roots: + parent = posixpath.dirname(path) + while parent: + ancestors.add(parent) + parent = posixpath.dirname(parent) + directory_modes: list[tuple[str, str]] = [] + if ancestors: + inspection = ( + f"cd -- {shlex.quote(workspace)} && " + f"for path in {shlex.join(sorted(ancestors, key=lambda path: (path.count('/'), path)))}; do " + 'if [ -d "$path" ] && [ ! -L "$path" ]; then ' + 'mode=$(stat -c %a -- "$path" 2>/dev/null) || ' + 'mode=$(stat -f %Lp -- "$path") || exit; ' + 'printf "%s\\0%s\\0" "$path" "$mode"; fi; done' + ) + async with _adoption_command(environment, workspace, inspection, repository=repository) as command: + metadata = _complete(await environment.exec_cmd( + command, timeout=120, + ), "candidate ancestor directory inspection").split("\0") + for path, mode in zip(metadata[::2], metadata[1::2]): + if not mode or len(mode) > 4 or any(digit not in "01234567" for digit in mode): + raise RuntimeError("candidate ancestor directory mode is invalid") + directory_modes.append((path, mode)) + git_directory = None + directory_script = f"cd -- {shlex.quote(workspace)} || exit\n" + _RECOVERY_PARENT + if repository: + # Git removes private metadata with the linked worktree. + directory_script += _RECOVERY_GIT_DIRECTORIES + r''' +printf '%s\0' "$private" || exit +recovery_parent "$common" "$private" +''' + else: + directory_script += 'recovery_parent "${TMPDIR:-/tmp}"' + recovery_parent = _complete(await environment.exec_cmd(directory_script, timeout=30), "candidate recovery storage") + if repository: + git_directory, recovery_parent = recovery_parent.split("\0") + storage = shlex.quote(posixpath.join(workspace, recovery_parent.removesuffix("\n"))) + backup = None + keep_backup = False + + async def allocate_backup() -> None: + nonlocal backup + backup = _complete(await environment.exec_cmd( + f'mktemp -d {storage}/candidate-recovery-XXXXXXXX', timeout=30, + ), "candidate recovery storage").removesuffix("\n") + + async def remove_backup(path: str, *, retain_detached: bool = False) -> None: + remove_entry = ( + 'remove_entry() { rm -rf -- "$1" || ' + '(find "$1" -type d ! -perm -u+w -exec chmod -- u+w {} + && rm -rf -- "$1"); }\n' + ) + removal = ( + f"recovery={shlex.quote(path)}; staged={shlex.quote(posixpath.join(path, 'replay'))}; " + f"workspace_prefix={shlex.quote(workspace.rstrip('/') + '/')};\n" + + _RECOVERY_STORES + ) + if retain_detached: + removal += remove_entry + _RECHECK_PARKED + r''' +prune_store() { + for entry in "$1"/*; do + case "${entry##*/}" in original-inodes|observed-*|volumes) continue;; esac + remove_entry "$entry" || return $? + done +} +each_store prune_store +''' + else: + removal = remove_entry + removal + "each_store remove_entry" + result = await environment.exec_cmd(removal, timeout=120) + if retain_detached: + _complete(result, "candidate recovery cleanup") + + try: + await await_owned_operation(allocate_backup(), propagate_cancellation=True) + assert backup is not None + staged = posixpath.join(backup, "replay") + stage_copies = [f"mkdir -m 700 -- {shlex.quote(staged)} || exit"] + stage_modes = [] + copies = [] + restores = [] + recreate_directories = [] + restore_directory_modes = [] + restore_parents = {posixpath.dirname(path) or "." for path in roots} + saved_attributes_file = None + if attributes_file is not None: + saved_attributes_file = posixpath.join(backup, "configured-attributes") + source_attributes = shlex.quote(attributes_file) + copies.append( + f"if [ -f {source_attributes} ] && [ -r {source_attributes} ]; then " + f"cat -- {source_attributes} > {shlex.quote(saved_attributes_file)} || exit; fi" + ) + saved_roots = set(roots) + # Git can prune empty ancestors. Make them writable until the saved files return. + for index, (path, mode) in enumerate(directory_modes): + source = shlex.quote(posixpath.normpath(posixpath.join(workspace, path))) + stage_path = shlex.quote(posixpath.normpath(posixpath.join(staged, path))) + stage_copies.append(f"mkdir -p -- {stage_path} || exit") + stage_modes.append(f"chmod -- {mode} {stage_path} || exit") + metadata_file = shlex.quote(posixpath.join(backup, f"directory-{index}.mode")) + temporary_mode = shlex.quote(posixpath.join(backup, f"directory-{index}.temporary")) + copies.append(f"printf '%s\\0%s\\0' {source} {mode} > {metadata_file} || exit") + # Attribute files inside affected roots are already in the file snapshot. + attributes_path = posixpath.normpath(posixpath.join(path, ".gitattributes")) + if repository and attributes_path not in saved_roots: + source_attributes = shlex.quote(posixpath.join(workspace, attributes_path)) + saved_attributes = shlex.quote(posixpath.join(backup, f"directory-{index}.attributes")) + staged_attributes = shlex.quote(posixpath.join(staged, attributes_path)) + copies.append( + f"if [ -f {source_attributes} ] && [ ! -L {source_attributes} ] " + f"&& [ -r {source_attributes} ]; then " + f"cat -- {source_attributes} > {saved_attributes} || exit; fi" + ) + stage_copies.append( + f"if [ -f {saved_attributes} ]; then " + f"cat -- {saved_attributes} > {staged_attributes} || exit; fi" + ) + permission = "write" if path in restore_parents else "read" + recreate_directories.append( + f"prepare_parent {source} {mode} {temporary_mode} {permission} {stage_path} || failed=$?" + ) + restore_directory_modes.append( + f"reset_parent_mode {source} {temporary_mode} || failed=$?" + ) + for index, path in enumerate(roots): + source = shlex.quote(posixpath.join(workspace, path)) + saved = shlex.quote(posixpath.join(backup, str(index))) + expected = shlex.quote(posixpath.join(staged, path)) + copies.append( + f"printf '%s' {source} > {saved}.path || exit; " + f"if [ -e {source} ] || [ -L {source} ]; then " + f"cp -pPR -- {source} {saved} || exit; fi" + ) + stage_copies.append( + f"if present {saved}; then cp -pPR -- {saved} {expected} || exit; fi" + ) + restores.append( + f"{{ check_parents {source} && restore_entry {saved} {expected} {source}; }} || failed=$?" + ) + replay_script = 'present() { [ -e "$1" ] || [ -L "$1" ]; }; ' + "; ".join(( + *stage_copies, *reversed(stage_modes), + )) + targets = [shlex.quote(path) for path in paths] + # Git can unlink the original before installing the candidate. + workspace_vars = ( + f"workspace={shlex.quote(workspace)}; recovery={shlex.quote(backup)};\n" + 'workspace_prefix="${workspace%/}/";\n' + ) + # Batch local links to avoid launching a process for every file. + linker = sys.executable if len(paths) >= 64 and getattr(environment, "local_filesystem", False) else "" + snapshot_script = ( + workspace_vars + f"linker={shlex.quote(linker)};\n" + + (f"link_script={shlex.quote(_LINK_ORIGINALS)};\n" if linker else "") + + _HOLD_ORIGINALS + "; ".join((*copies, f"hold_originals {' '.join(targets)} || exit")) + ) + target_test = f'case "${{1#"$workspace_prefix"}}" in {"|".join(targets)}) return 0;; esac; ' if targets else "" + recovery_script = ( + f"failed=0; {workspace_vars}" + f"absent={shlex.quote(posixpath.join(backup, 'absent'))};\n" + f"candidate_path() {{ {target_test}return 1; }}\n" + f"candidate_descendants() {{ check_candidate_descendants \"$1\" {' '.join(targets)}; }}\n" + ) + _RECOVER_PATHS + "; ".join(( + *recreate_directories, *restores, *reversed(restore_directory_modes), + )) + ';\nif [ "$failed" -ne 0 ]; then exit "$failed"; fi\n' + _DETACHED_FILES + async with AsyncExitStack() as commands: + snapshot_command = await commands.enter_async_context(_adoption_command( + environment, workspace, snapshot_script, repository=repository, + )) + # Store long recovery commands before publishing any source changes. + recovery_command = await commands.enter_async_context(_adoption_command( + environment, workspace, recovery_script, repository=repository, + )) + snapshot = _complete( + await await_owned_operation( + environment.exec_cmd(snapshot_command, timeout=900), propagate_cancellation=True, + ), + "candidate recovery snapshot", + ) + publication_started = False + try: + if "activate-originals" in snapshot.splitlines(): + # Filesystems without hard links keep the original by + # moving it aside and giving Git an equivalent working copy. + _complete(await environment.exec_cmd( + workspace_vars + _ACTIVATE_ORIGINALS, timeout=900, + ), "candidate recovery preparation") + publication_started = True + yield + except BaseException as failure: + async def recover() -> tuple[bool, bool]: + command = recovery_command + if publication_started: + async with _adoption_command( + environment, workspace, replay_script, repository=repository, + ) as copy_command: + _complete(await environment.exec_cmd(copy_command, timeout=900), + "candidate recovery replay copy") + await replay(staged, git_directory, saved_attributes_file) + else: + command = workspace_vars + _UNDO_INCOMPLETE_ACTIVATION + _DETACHED_FILES + + shell_commands = getattr(environment, "_shell_commands", None) + active = shell_commands.active if shell_commands is not None else 0 + last_started = shell_commands.last_started if shell_commands is not None else None + receipt = _complete(await environment.exec_cmd( + command, timeout=900, + ), "candidate adoption recovery") + shell_overlapped = shell_commands is not None and ( + active > 0 or last_started is not shell_commands.last_started + ) + entries = receipt.splitlines() + parked = "parked" in entries + return parked or "original-detached" in entries, parked and shell_overlapped + + try: + detached, parked_during_shell = await await_owned_operation(recover()) + except BaseException as recovery: + keep_backup = True + revoke = getattr(environment, "revoke", None) + if callable(revoke): + revoke() + raise CandidateAdoptionRecoveryError( + f"candidate adoption failed ({failure}); recovery failed ({recovery}); " + f"original files retained in {backup}" + ) from recovery + if detached: + keep_backup = True + if parked_during_shell: + # A command can still own a detached inode after its original path is restored. + raise CandidateAdoptionRecoveryError( + f"candidate adoption failed ({failure}); a concurrent shell command may have written " + f"to parked files retained in {backup}" + ) from failure + assert backup is not None + try: + await await_owned_operation(remove_backup(backup, retain_detached=True)) + except BaseException as cleanup: + raise CandidateAdoptionRecoveryError( + f"candidate adoption failed ({failure}); cleanup failed ({cleanup}); " + f"recovery location {backup}" + ) from cleanup + detail = f"Detached candidate files retained in {backup}" + add_exception_note(failure, detail) + logger.warning("%s", detail) + raise + finally: + if backup is not None and not keep_backup: + try: + await await_owned_operation(remove_backup(backup)) + except Exception: + pass diff --git a/opencollab/adapters/_candidate_increment.py b/opencollab/adapters/_candidate_increment.py index 4272f226..b2dd2401 100644 --- a/opencollab/adapters/_candidate_increment.py +++ b/opencollab/adapters/_candidate_increment.py @@ -5,11 +5,15 @@ import posixpath import shlex from collections.abc import Sequence +from contextlib import AsyncExitStack from typing import Any -from opencollab.adapters._candidate_physical import _apply_physical +from opencollab.adapters._candidate_adoption import _adoption_transaction +from opencollab.adapters._candidate_physical import _physical_git +from opencollab.adapters._env_docker import DockerEnvironment from opencollab.adapters._git_patch_bytes import _complete, _patch_temporary from opencollab.adapters.git_patch import _git_command_and_config_guard +from opencollab.adapters.tools.base import host_write_locks from opencollab.patches import patch_paths @@ -30,12 +34,20 @@ async def _apply_increment( if not isinstance(patch, str) or not patch.strip(): raise ValueError("candidate patch must be non-empty") preserved = tuple(_safe_path(path) for path in preserve_paths) + delegate = getattr(environment, "_delegate", environment) + docker = isinstance(delegate, DockerEnvironment) + lock_root = ( + workspace if docker or getattr(environment, "local_filesystem", False) + else getattr(environment, "workspace", workspace) + ) + # Directory copies can sit below an unrelated Git worktree. + git_prefix = "git" if repository else "GIT_DIR=/dev/null git" async with _patch_temporary( environment, workspace, patch, prefix=".candidate-adopt-", suffix=".patch", cleanup_best_effort=True, repository=repository, ) as candidate_file: output = _complete(await environment.exec_cmd( - f"git -C {shlex.quote(workspace)} apply --numstat -z -- {shlex.quote(candidate_file)}", + f"{git_prefix} -C {shlex.quote(workspace)} apply --numstat -z -- {shlex.quote(candidate_file)}", timeout=30, ), "candidate path inspection") candidate_paths = { @@ -47,19 +59,76 @@ async def _apply_increment( for protected in preserved for path in candidate_paths ): raise ValueError("candidate patch overlaps a preserved path") - git = "git -c core.filemode=true" - guard = "" - if strict and repository: - git, guard = _git_command_and_config_guard() - if physical_paths and repository: - if guard: - _complete(await environment.exec_cmd( - f"cd -- {shlex.quote(workspace)} && {guard}:", timeout=120, - ), "physical candidate configuration inspection") - await _apply_physical(environment, workspace, candidate_file, physical_paths, strict=strict) - return - _complete(await environment.exec_cmd( - f"cd -- {shlex.quote(workspace)} && {guard}" - f"{git} apply --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}", - timeout=120, - ), "candidate adoption") + targets = (posixpath.join(lock_root, ".candidate-adoption"), + *(posixpath.join(workspace, path) for path in sorted(candidate_paths))) + locks = ( + delegate.file_write_locks(targets, _repository_root=workspace) + if docker else host_write_locks(targets, environment) + ) + async with locks: + git = f"{git_prefix} -c core.filemode=true" + guard = "" + if strict and repository: + git, guard = _git_command_and_config_guard() + async with AsyncExitStack() as resources: + command = f"cd -- {shlex.quote(workspace)} && {guard}{git}" + operation = "candidate adoption" + if physical_paths and repository: + if guard: + _complete(await environment.exec_cmd( + f"cd -- {shlex.quote(workspace)} && {guard}:", timeout=120, + ), "physical candidate configuration inspection") + command, _ = await resources.enter_async_context(_physical_git( + environment, workspace, physical_paths, strict=strict, + )) + operation = "physical candidate adoption" + # Let Git reject unsafe path aliases before a recursive backup can + # encounter repository metadata. Publication can still fail later. + check = f"{command} apply --check --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}" + configured_rules = repository and not strict and not physical_paths + if configured_rules: + check += f' && ({git} config --path --get core.attributesFile || [ "$?" -eq 1 ])' + checked = _complete(await environment.exec_cmd(check, timeout=120), operation) + attributes_file = None + if configured_rules and (configured := checked.removesuffix("\n")): + attributes_file = posixpath.join(workspace, configured) + + async def replay( + staged: str, git_directory: str | None, saved_attributes_file: str | None, + ) -> None: + async with AsyncExitStack() as metadata: + replay_git = f"cd -- {shlex.quote(workspace)} && {guard}{git}" + if physical_paths and repository: + replay_git, git_directory = await metadata.enter_async_context(_physical_git( + environment, workspace, physical_paths, strict=strict, + )) + if git_directory is not None: + replay_git += ( + f" --git-dir={shlex.quote(git_directory)} --work-tree={shlex.quote(staged)}" + f" -c safe.directory={shlex.quote(staged)}" + ) + if saved_attributes_file is not None: + replay_git += f" -c core.attributesFile={shlex.quote(saved_attributes_file)}" + replay_git += f" -C {shlex.quote(staged)}" + _complete(await environment.exec_cmd( + f"{replay_git} apply --whitespace=nowarn --check --binary" + f" -- {shlex.quote(candidate_file)}", timeout=120, + ), "candidate recovery replay check") + # A failed replay still records the files Git was able to write. + result = await environment.exec_cmd( + f"{replay_git} apply --whitespace=nowarn --binary" + f" -- {shlex.quote(candidate_file)}", timeout=120, + ) + _complete(result, "candidate recovery replay", allowed=(0, 128)) + + async with _adoption_transaction( + environment, workspace, sorted(candidate_paths), repository=repository, + replay=replay, attributes_file=attributes_file, + ): + try: + _complete(await environment.exec_cmd( + f"{command} apply --binary --whitespace=nowarn -- {shlex.quote(candidate_file)}", + timeout=120, + ), operation) + finally: + await resources.aclose() diff --git a/opencollab/adapters/_candidate_source.py b/opencollab/adapters/_candidate_source.py index 302209e3..6ecf7f55 100644 --- a/opencollab/adapters/_candidate_source.py +++ b/opencollab/adapters/_candidate_source.py @@ -25,7 +25,6 @@ class _SourceSnapshot: index_revision: str patch: str excluded: tuple[str, ...] - untracked_excluded: tuple[str, ...] physical_tree: str | None modules: tuple[tuple[str, str], ...] initialized: tuple[str, ...] @@ -61,12 +60,13 @@ async def changed(self, patch: str | None) -> bool: ): return True environment = self.owner._environment + untracked_excluded = (*await self.owner._source_untracked_paths(self.root), *self.excluded) initialized = await _initialized_submodule_paths(environment, self.root) if self._included(initialized) != self._included(self.initialized): return True current_modules = dict(await _physical_submodule_trees( environment, self.root, include_paths=tuple(path for path, _tree in self.modules), - exclude_paths=self.untracked_excluded, exclude_tree_paths=self.excluded, + exclude_paths=untracked_excluded, exclude_tree_paths=self.excluded, module_paths=initialized, )) original_modules = dict(self.modules) @@ -85,7 +85,7 @@ async def changed(self, patch: str | None) -> bool: return True if self.physical_tree is None and patch == self.patch and not await _has_worktree_conversions( environment, self.root, exclude_paths=self.excluded, - exclude_untracked_paths=self.untracked_excluded, + exclude_untracked_paths=untracked_excluded, ): return False # A plain source's saved Git patch contains its physical bytes. Build @@ -93,7 +93,7 @@ async def changed(self, patch: str | None) -> bool: original = self.physical_tree or await self._original_tree() current = await _physical_tree( environment, self.root, self.revision, - exclude_paths=self.untracked_excluded, exclude_tree_paths=self.excluded, + exclude_paths=untracked_excluded, exclude_tree_paths=self.excluded, ) return current != original @@ -105,7 +105,7 @@ async def _source_snapshot(owner: Any, patch: str | None, excluded: tuple[str, . environment = owner._environment revision = await owner.source_revision() index_revision = await owner.source_index_revision() - untracked_excluded = (*owner._owned_untracked_paths(root), *excluded) + untracked_excluded = (*await owner._source_untracked_paths(root), *excluded) initialized = await _initialized_submodule_paths(environment, root) modules = await _physical_submodule_trees( environment, root, exclude_paths=untracked_excluded, exclude_tree_paths=excluded, @@ -119,6 +119,6 @@ async def _source_snapshot(owner: Any, patch: str | None, excluded: tuple[str, . environment, root, revision, exclude_paths=untracked_excluded, exclude_tree_paths=excluded, ) return _SourceSnapshot( - owner, root, revision, index_revision, patch, excluded, untracked_excluded, + owner, root, revision, index_revision, patch, excluded, physical_tree, modules, initialized, ) diff --git a/opencollab/adapters/_docker_snapshot.py b/opencollab/adapters/_docker_snapshot.py index aaa10778..fe04cd99 100644 --- a/opencollab/adapters/_docker_snapshot.py +++ b/opencollab/adapters/_docker_snapshot.py @@ -44,12 +44,6 @@ async def _copy_to_host(environment: Any, source: str, target: Path, *, director async def _copy_artifact_to_host(environment: Any, source: str, target: Path) -> None: - relative = ( - posixpath.relpath(source, environment.workspace) if posixpath.isabs(source) else posixpath.normpath(source) - ) - if relative == ".." or relative.startswith("../"): - raise PermissionError("artifact path escapes workspace") - source = posixpath.join(environment.workspace, relative) delegate = environment._delegate if isinstance(environment, _TeamCandidateEnvironment) else environment if isinstance(delegate, WorktreeEnvironment): delegate._ensure_active() @@ -65,6 +59,12 @@ def copy() -> None: await delegate._run_file_operation(copy) return + relative = ( + posixpath.relpath(source, environment.workspace) if posixpath.isabs(source) else posixpath.normpath(source) + ) + if relative == ".." or relative.startswith("../"): + raise PermissionError("artifact path escapes workspace") + source = posixpath.join(environment.workspace, source) if isinstance(delegate, DockerEnvironment): source = await delegate._contained_file_path(source) if isinstance(delegate, (DockerEnvironment, VMEnvironment)): diff --git a/opencollab/adapters/_env_base.py b/opencollab/adapters/_env_base.py index d36ab371..cc6c28e8 100644 --- a/opencollab/adapters/_env_base.py +++ b/opencollab/adapters/_env_base.py @@ -2,6 +2,8 @@ from __future__ import annotations +from collections.abc import Iterator +from contextlib import AbstractContextManager, contextmanager, nullcontext from dataclasses import dataclass, field from opencollab.adapters._safe_file_common import _normalize_newlines, _split_lines @@ -28,6 +30,26 @@ class TextFileRange: has_more: bool chars_truncated: bool = False +class _ShellCommands: + def __init__(self) -> None: + self.active = 0 + self.last_started: object | None = None + + @contextmanager + def running(self) -> Iterator[None]: + self.active += 1 + self.last_started = object() + try: + yield + finally: + self.active -= 1 + + +def _track_shell_command(environment: object) -> AbstractContextManager[None]: + commands = getattr(environment, "_shell_commands", None) + return commands.running() if commands is not None else nullcontext() + + class Environment: """Abstract execution environment used by tools and workflows.""" @@ -39,6 +61,7 @@ class Environment: def __init__(self) -> None: self._aborted = False + self._shell_commands = _ShellCommands() @property def revoked(self) -> bool: diff --git a/opencollab/adapters/_env_docker.py b/opencollab/adapters/_env_docker.py index 4996aa14..ca53c580 100644 --- a/opencollab/adapters/_env_docker.py +++ b/opencollab/adapters/_env_docker.py @@ -13,7 +13,7 @@ import stat import threading import uuid -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncIterator, Callable, Sequence from dataclasses import dataclass, field from typing import NoReturn @@ -30,6 +30,7 @@ timed_out_result, ) from opencollab.adapters._safe_file_common import _normalize_newlines, _split_lines +from opencollab.adapters.tools.base import _write_lock_paths from opencollab.application.async_timeout import await_owned_operation from opencollab.application.exception_notes import add_exception_note from opencollab.domain.docker import DockerOptions @@ -210,6 +211,7 @@ class DockerEnvironment(Environment): """ process_isolated = True + remote_filesystem = True def __init__( self, @@ -691,6 +693,71 @@ async def write_file(self, path: str, content: str) -> None: async def write_file_with_change(self, path: str, content: str) -> bool | None: return await self._write_file(path, content, observe_change=True) + @contextlib.asynccontextmanager + async def file_write_locks( + self, paths: Sequence[str], *, _repository_root: str | None = None, + ) -> AsyncIterator[None]: + self._ensure_active() + await self._bind_attached() + if self._container_id is None: + raise RuntimeError("Container not started. Call setup() first.") + ordered = await self._grouped_write_targets(paths, repository_root=_repository_root) + while True: + async with contextlib.AsyncExitStack() as locks: + for target in ordered: + await locks.enter_async_context(self._resolved_file_write_lock(target)) + # An earlier publisher may have changed a directory alias while + # we waited. Release the group before acquiring any new target. + current = await self._grouped_write_targets(paths, repository_root=_repository_root) + if current == ordered: + yield None + return + ordered = current + + async def _grouped_write_targets( + self, paths: Sequence[str], *, repository_root: str | None = None, + ) -> list[str]: + requested = {self._validate_container_path(path) for path in paths} + if not requested: + return [] + # The first path sorts before any of its descendants, so it can + # establish the physical workspace using ordinary parent resolution. + parent_cache: dict[tuple[str, str, tuple[str, ...]], tuple[str, str]] = {} + first = min(requested) + workspace = repository_root or self.workspace + targets = {await self._resolve_write_target( + first, parent_cache=parent_cache, repository_root=repository_root, + )} + physical_workspace = parent_cache[(workspace, posixpath.dirname(first), ())][0] + ancestors_by_path: dict[str, tuple[str, ...]] = {} + for path in sorted(requested): + ancestors = [] + parent = posixpath.dirname(path) + while parent not in ("", ".", "/", workspace, physical_workspace): + if posixpath.isabs(parent) and not any( + _within_directory(root, posixpath.normpath(parent)) + for root in (workspace, physical_workspace) + ): + break + if posixpath.basename(parent) not in (".", ".."): + ancestors.append(parent) + parent = posixpath.dirname(parent) + ancestors_by_path[path] = tuple(ancestors) + resolve = dict(ancestors_by_path) + for ancestors in ancestors_by_path.values(): + for index, ancestor in enumerate(ancestors): + if ancestor not in resolve: + resolve[ancestor] = ancestors[index + 1:] + resolve.pop(first) + targets.update({ + await self._resolve_write_target( + path, replaceable_parents=tuple(parent for parent in ancestors if parent in requested), + parent_cache=parent_cache, repository_root=repository_root, + ) + for path, ancestors in resolve.items() + }) + return _write_lock_paths(targets, self, workspace=physical_workspace) + @contextlib.asynccontextmanager async def file_write_lock( self, @@ -701,12 +768,16 @@ async def file_write_lock( """Hold the target lock across a native edit, including its final write.""" self._ensure_active() await self._bind_attached() - container_id = self._container_id - if container_id is None: + if self._container_id is None: raise RuntimeError("Container not started. Call setup() first.") target = self._validate_container_path(path) target = await self._resolve_write_target(target, containment_root=containment_root) - key = f"{container_id}\0{target}" + async with self._resolved_file_write_lock(target): + yield target + + @contextlib.asynccontextmanager + async def _resolved_file_write_lock(self, target: str) -> AsyncIterator[str]: + key = f"{self._container_id}\0{target}" task = asyncio.current_task() assert task is not None # Native tools already hold this lock when they call write_file. Only @@ -757,6 +828,8 @@ async def _write_file( f"docker file exceeds write limit of {ENV_FILE_WRITE_LIMIT_BYTES} bytes: {path}" ) async with self.file_write_lock(path, containment_root=containment_root) as target: + if containment_root == "/tmp": + self._temporary_files.add(path) return await self._write_file_atomic(target, payload, observe_change=observe_change) async def _resolve_write_target( @@ -764,15 +837,43 @@ async def _resolve_write_target( target: str, *, containment_root: str | None = None, + replaceable_parents: Sequence[str] = (), + parent_cache: dict[tuple[str, str, tuple[str, ...]], tuple[str, str]] | None = None, + repository_root: str | None = None, ) -> str: - # Resolve the parent through the file execution wrapper, including its - # cwd and command prefix. Keep the final component so atomic writes - # replace a file symlink rather than modifying its referent. directory, filename = posixpath.split(target) + if containment_root not in (None, "/tmp"): + raise ValueError("container temporary files must stay under /tmp") + root = containment_root or repository_root or self.workspace + key = (root, directory, tuple(replaceable_parents)) + parent = parent_cache.get(key) if parent_cache is not None else None + if parent is None: + parent = await self._resolve_write_parent(directory, root, replaceable_parents) + if parent_cache is not None: + parent_cache[key] = parent + resolved_root, resolved = parent + # Owned temporary files do not change the cached worktree root. + if containment_root is None and repository_root is None: + self._resolved_workspace = resolved_root + # Keep the final component for atomic replacement of file symlinks. + resolved_target = posixpath.normpath(posixpath.join(resolved, filename)) + if not _within_directory(resolved_root, resolved_target): + boundary = "workspace" if containment_root is None else "temporary directory" + raise PermissionError(f"container file path escapes {boundary}: {target}") + return resolved_target + + async def _resolve_write_parent( + self, directory: str, root: str, replaceable_parents: Sequence[str], + ) -> tuple[str, str]: + # A patch may replace an existing file or symlink with a directory. command = ( - 'directory=$1; workspace=$2; suffix=; ' + 'directory=$1; workspace=$2; shift 2; suffix=; ' 'while [ ! -d "$directory" ]; do ' - '[ ! -e "$directory" ] && [ ! -L "$directory" ] || exit 66; ' + 'case "${directory##*/}" in ..) exit 66;; esac; ' + 'if [ -e "$directory" ] || [ -L "$directory" ]; then ' + '[ -f "$directory" ] || [ -L "$directory" ] || exit 66; ' + 'for parent do [ "$parent" != "$directory" ] || break; done; ' + '[ "$#" -gt 0 ] && [ "$parent" = "$directory" ] || exit 66; fi; ' 'suffix="/${directory##*/}$suffix"; directory=${directory%/*}; ' '[ -n "$directory" ] || directory=/; done; ' 'cd -P -- "$directory" || exit 66; ' @@ -782,13 +883,11 @@ async def _resolve_write_target( parent = directory or "." if not posixpath.isabs(parent): parent = "./" + parent - if containment_root not in (None, "/tmp"): - raise ValueError("container temporary files must stay under /tmp") - root = self.workspace if containment_root is None else containment_root + arguments = [parent, root, *( + path if posixpath.isabs(path) else "./" + path for path in replaceable_parents + )] result = await self._exec( - "bash -c " - f"{shlex.quote(command)} opencollab-file-parent {shlex.quote(parent)} " - f"{shlex.quote(root)}", + f"bash -c {shlex.quote(command)} opencollab-file-parent {shlex.join(arguments)}", timeout=DOCKER_CONTROL_TIMEOUT_SECONDS, login=False, ) @@ -802,16 +901,7 @@ async def _resolve_write_target( or not posixpath.isabs(resolved) ): raise OSError("cannot resolve container file execution directory") - resolved_root = posixpath.normpath(resolved_root) - # Owned /tmp files must not replace the cached workspace root. A later - # workspace write would then treat the temporary directory as the project. - if containment_root is None: - self._resolved_workspace = resolved_root - resolved_target = posixpath.normpath(posixpath.join(resolved, filename)) - if not _within_directory(resolved_root, resolved_target): - boundary = "workspace" if containment_root is None else "temporary directory" - raise PermissionError(f"container file path escapes {boundary}: {target}") - return resolved_target + return posixpath.normpath(resolved_root), resolved async def _physical_workspace(self) -> str: if self._resolved_workspace is not None: @@ -865,6 +955,16 @@ async def _contained_file_path(self, path: str) -> str: raise PermissionError(f"container file path escapes workspace: {path}") return referent + async def resolve_search_path(self, path: str) -> str: + if not isinstance(path, str) or not path or "\0" in path: + raise ValueError("container search path must be non-empty text without NUL bytes") + target = await self._symlink_referent(posixpath.join(self.workspace, path)) + if target is None: + raise FileNotFoundError(f"cannot resolve container search path: {path}") + if not _within_directory(await self._physical_workspace(), target): + raise PermissionError(f"container search path escapes workspace: {path}") + return target + @staticmethod def _validate_container_path(path: str) -> str: if not isinstance(path, str) or not path or "\0" in path: @@ -989,7 +1089,6 @@ async def write_temp_file( # Patch staging owns this path. It is outside the project on purpose; # workspace containment would reject it and drop the binary patch. await self._write_file(path, content, observe_change=False, containment_root="/tmp") - self._temporary_files.add(path) return path async def remove_file(self, path: str) -> None: diff --git a/opencollab/adapters/_env_local.py b/opencollab/adapters/_env_local.py index dc227f02..c5aa1cee 100644 --- a/opencollab/adapters/_env_local.py +++ b/opencollab/adapters/_env_local.py @@ -54,7 +54,7 @@ class LocalEnvironment(Environment): def __init__(self, workspace: str = ".") -> None: super().__init__() - self.workspace = os.path.realpath(os.path.abspath(workspace)) + self.workspace = os.path.realpath(workspace) if not os.path.isdir(self.workspace): raise NotADirectoryError(self.workspace) self._workspace_fd: int | None = open_directory_anchor(self.workspace) @@ -72,17 +72,12 @@ def _relative_path(self, path: str) -> str: candidate = canonicalize_system_path(path) workspace = canonicalize_system_path(self.workspace) try: - inside = os.path.commonpath((workspace, candidate)) == os.fspath(workspace) - except ValueError: - inside = False - if not inside: - raise PermissionError(f"path escapes local workspace: {path}") - normalized = os.path.relpath(candidate, workspace) + relative = os.fspath(candidate.relative_to(workspace)) + except ValueError as exc: + raise PermissionError(f"path escapes local workspace: {path}") from exc else: - normalized = os.path.normpath(path) - if normalized == ".." or normalized.startswith(f"..{os.sep}"): - raise PermissionError(f"relative path escapes local workspace: {path}") - return normalized + relative = path + return relative def _root_fd(self) -> int: if self._workspace_fd is None: @@ -229,7 +224,8 @@ async def write_temp_file( async def remove_file(self, path: str) -> None: self._ensure_active() target = self._relative_path(path) - if target not in self._temporary_files: + owned_target = os.path.normpath(target) + if owned_target not in self._temporary_files: raise OSError(f"refusing to remove unowned local temporary file: {path}") await self._run_file_operation( unlink_regular_file_durable_at, @@ -237,7 +233,7 @@ async def remove_file(self, path: str) -> None: self.workspace, target, ) - self._temporary_files.discard(target) + self._temporary_files.discard(owned_target) async def _cleanup_files(self) -> None: while self._file_operations: diff --git a/opencollab/adapters/_env_vm.py b/opencollab/adapters/_env_vm.py index c83a5da3..21d46d3b 100644 --- a/opencollab/adapters/_env_vm.py +++ b/opencollab/adapters/_env_vm.py @@ -57,6 +57,8 @@ class VMEnvironment(Environment): """ process_isolated = True + remote_filesystem = True + file_write_lock_scope = "workspace" def __init__(self, connection: VMConnection, workspace: str, *, state_directory: str, run_id: str | None = None): super().__init__() @@ -281,6 +283,9 @@ async def recover_operation(self, operation_id: str) -> Any: async def read_file(self, path: str) -> str: return await self._operate("read", {"path": path}) + async def resolve_search_path(self, path: str) -> str: + return await self._operate("resolve_search_path", {"path": path}) + async def read_text_range(self, path: str, *, offset: int, limit: int, max_chars: int) -> TextFileRange: return TextFileRange(**await self._operate("range", {"path": path, "offset": offset, "limit": limit, "max_chars": max_chars})) diff --git a/opencollab/adapters/_vm_remote.py b/opencollab/adapters/_vm_remote.py index e15b1d11..21f3b2d2 100644 --- a/opencollab/adapters/_vm_remote.py +++ b/opencollab/adapters/_vm_remote.py @@ -34,6 +34,7 @@ unlink_regular_file_durable_at, write_regular_bytes_atomic_at, ) +from opencollab.adapters.safety import SandboxInterceptor from opencollab.adapters.vm_config import validate_vm_path _MAX_MESSAGE = 32 * 1024 * 1024 @@ -205,6 +206,8 @@ async def _perform(request: dict[str, Any], directory: Path) -> Any: return result if action == "read": return await env.read_file(args["path"]) + if action == "resolve_search_path": + return SandboxInterceptor(env.workspace).check_path(args["path"]) if action == "range": return asdict(await env.read_text_range(**args)) if action == "write": diff --git a/opencollab/adapters/candidate_workspace.py b/opencollab/adapters/candidate_workspace.py index ed8e0e3c..c5bc3f20 100644 --- a/opencollab/adapters/candidate_workspace.py +++ b/opencollab/adapters/candidate_workspace.py @@ -13,6 +13,7 @@ from dataclasses import dataclass from typing import Any +from opencollab.adapters._candidate_adoption import _mounted_recovery_paths from opencollab.adapters._candidate_increment import _apply_increment as _apply_increment from opencollab.adapters._candidate_increment import _safe_path from opencollab.adapters._candidate_physical import ( @@ -85,19 +86,20 @@ async def _raw_diff_at( ), "candidate tracked diff", environment, workspace, base_revision, index_file, ) + untracked_paths = (*excluded, *(_safe_path(path) + "/" for path in untracked_exclude_paths)) + untracked_pathspec = " -- . " + shlex.join( + f":(exclude,literal){path}" for path in untracked_paths + ) if untracked_paths else "" untracked = _complete( await environment.exec_cmd( f"{git} ls-files --others --exclude-standard -z" - + pathspec, + + untracked_pathspec, timeout=CANDIDATE_WORKSPACE_GIT_TIMEOUT_SECONDS, ), "candidate untracked listing", ) parts = [tracked] if tracked.strip() else [] - untracked_excluded = tuple(_safe_path(path) + "/" for path in untracked_exclude_paths) for path in (item for item in untracked.split("\0") if item): - if path.startswith(untracked_excluded): - continue patch = await _complete_patch( await environment.exec_cmd( "git -c core.quotePath=true -C " @@ -206,17 +208,21 @@ async def source_contents_changed(self) -> bool: async def physical_source_changed(self, exclude_paths: Sequence[str] = (), *, force: bool = False) -> bool: excluded = _repository_exclusions(self.workspace_prefix, exclude_paths) + untracked_exclusions = ( + *self.source_untracked_exclusions, + *await _mounted_recovery_paths(self.base_environment, self.source_workspace), + ) if await self._submodule_contents_changed( self.source_workspace, self.physical_source_submodules, initialized=self.initialized_source_submodules, - excluded=excluded, untracked_exclusions=self.source_untracked_exclusions, + excluded=excluded, untracked_exclusions=untracked_exclusions, ): return True original_tree = self.physical_base_revision if original_tree is None: if not force and not await _has_worktree_conversions( self.base_environment, self.source_workspace, strict=self.strict_capture, - exclude_paths=excluded, exclude_untracked_paths=self.source_untracked_exclusions, + exclude_paths=excluded, exclude_untracked_paths=untracked_exclusions, ): return False # Without initial conversions, the ordinary source tree already @@ -228,7 +234,7 @@ async def physical_source_changed(self, exclude_paths: Sequence[str] = (), *, fo ), "candidate original source tree").strip() current = await _physical_tree( self.base_environment, self.source_workspace, self.base_revision, - exclude_paths=self.source_untracked_exclusions, strict=self.strict_capture, + exclude_paths=untracked_exclusions, strict=self.strict_capture, ) if not exclude_paths: return current != original_tree @@ -422,6 +428,12 @@ def _owned_untracked_paths(self, repository_root: str) -> tuple[str, ...]: return () return (os.path.relpath(self._owned_save_dir, repository_root).replace(os.sep, "/"),) + async def _source_untracked_paths(self, repository_root: str) -> tuple[str, ...]: + return ( + *self._owned_untracked_paths(repository_root), + *await _mounted_recovery_paths(self._environment, repository_root), + ) + async def _repository_root(self) -> str: return _complete( await self._environment.exec_cmd( @@ -443,6 +455,7 @@ async def _workspace_prefix(self) -> str: async def _source_patch( self, repository_root: str, base_revision: str, exclude_paths: Sequence[str] = (), + *, untracked_exclusions: Sequence[str] = (), ) -> str: # Keep staged additions in a temporary index without its skip hints, # then stage the current tracked bytes without changing the source index. @@ -472,13 +485,13 @@ async def _source_patch( await _assert_patchable_submodules( self._environment, repository_root, base_revision, index_file=source_index, reject_ignored_untracked=False, - exclude_untracked_paths=self._owned_untracked_paths(repository_root), + exclude_untracked_paths=untracked_exclusions, exclude_paths=exclude_paths, ) return await _raw_diff_at( self._environment, repository_root, exclude_paths, base_revision=base_revision, index_file=source_index, - untracked_exclude_paths=self._owned_untracked_paths(repository_root), + untracked_exclude_paths=untracked_exclusions, ) finally: async def remove_index() -> None: @@ -535,6 +548,14 @@ async def _acquire( ), "candidate base revision", ).strip() + source_owned = ( + *self._owned_untracked_paths(repository_root), + *(_safe_path(path) for path in strict_untracked_exclusions), + ) + source_untracked = ( + *source_owned, + *await _mounted_recovery_paths(self._environment, repository_root), + ) if strict_capture: source_patch = await _complete_patch( await self._environment.exec_cmd( @@ -542,10 +563,7 @@ async def _acquire( + guarded_staged_diff_command( base_revision=base_revision, reject_ignored_untracked=False, - exclude_untracked_paths=( - *self._owned_untracked_paths(repository_root), - *(_safe_path(path) for path in strict_untracked_exclusions), - ), + exclude_untracked_paths=source_untracked, ), timeout=CANDIDATE_WORKSPACE_GIT_TIMEOUT_SECONDS, ), @@ -555,28 +573,28 @@ async def _acquire( async with AsyncExitStack() as capture_resources: converted = await _has_worktree_conversions( self._environment, repository_root, strict=strict_capture, + exclude_untracked_paths=source_untracked, ) source_capture = physical_context = None if converted: if not strict_capture: await _assert_patchable_submodules( self._environment, repository_root, base_revision, reject_ignored_untracked=False, - exclude_untracked_paths=self._owned_untracked_paths(repository_root), + exclude_untracked_paths=source_untracked, ) physical_context = await capture_resources.enter_async_context(_physical_git( self._environment, repository_root, strict=strict_capture, )) source_capture = await _capture_physical( self._environment, repository_root, base_revision, - exclude_paths=( - *self._owned_untracked_paths(repository_root), - *strict_untracked_exclusions, - ), + exclude_paths=source_untracked, strict=strict_capture, metadata=physical_context, ) source_patch = source_capture.patch elif not strict_capture: - source_patch = await self._source_patch(repository_root, base_revision) + source_patch = await self._source_patch( + repository_root, base_revision, untracked_exclusions=source_untracked, + ) initialized_source = await _initialized_submodule_paths( self._environment, repository_root, strict=strict_capture, source_prefix=workspace_prefix.rstrip("/"), @@ -584,7 +602,7 @@ async def _acquire( source_modules = await _physical_submodule_trees( self._environment, repository_root, strict=strict_capture, source_prefix=workspace_prefix.rstrip("/"), - exclude_paths=(*self._owned_untracked_paths(repository_root), *strict_untracked_exclusions), + exclude_paths=source_untracked, module_paths=initialized_source, ) lease = _CandidateLease( @@ -596,10 +614,7 @@ async def _acquire( workspace_prefix=workspace_prefix, physical_base_revision=source_capture.tree if source_capture is not None else None, strict_capture=strict_capture, - source_untracked_exclusions=( - *self._owned_untracked_paths(repository_root), - *strict_untracked_exclusions, - ), + source_untracked_exclusions=source_owned, physical_source_submodules=source_modules, initialized_source_submodules=initialized_source, ) @@ -774,24 +789,28 @@ async def source_diff(self, exclude_paths: Sequence[str] = ()) -> str: repository_root = await self._repository_root() excluded = _repository_exclusions(await self._workspace_prefix(), exclude_paths) if exclude_paths else () base_revision = await self.source_revision() - return await self._source_patch(repository_root, base_revision, excluded) + return await self._source_patch( + repository_root, base_revision, excluded, + untracked_exclusions=await self._source_untracked_paths(repository_root), + ) async def physical_source_revision(self, exclude_paths: Sequence[str] = ()) -> str | None: """Read conversion-aware source trees with repository-relative exclusions.""" root = await self._repository_root() excluded = tuple(_safe_path(path) for path in exclude_paths) + owned = await self._source_untracked_paths(root) modules = await _physical_submodule_trees( - self._environment, root, exclude_paths=(*self._owned_untracked_paths(root), *excluded), + self._environment, root, exclude_paths=(*owned, *excluded), exclude_tree_paths=excluded, ) revision = None if await _has_worktree_conversions( self._environment, root, exclude_paths=excluded, - exclude_untracked_paths=self._owned_untracked_paths(root), + exclude_untracked_paths=owned, ): revision = await _physical_tree( self._environment, root, await self.source_revision(), - exclude_paths=(*self._owned_untracked_paths(root), *excluded), + exclude_paths=(*owned, *excluded), exclude_tree_paths=excluded, ) return json.dumps((revision, modules)) if modules else revision @@ -805,7 +824,7 @@ async def _source_changed(self, exclude_paths: Sequence[str] = ()) -> bool: _repository_exclusions(await self._workspace_prefix(), exclude_paths) if exclude_paths else () ) - owned = self._owned_untracked_paths(repository_root) + owned = await self._source_untracked_paths(repository_root) def relative_paths(paths: Sequence[str], prefix: str) -> tuple[str, ...]: if not prefix: diff --git a/opencollab/adapters/hooks.py b/opencollab/adapters/hooks.py index 4db25095..09032782 100644 --- a/opencollab/adapters/hooks.py +++ b/opencollab/adapters/hooks.py @@ -11,6 +11,7 @@ from collections.abc import Awaitable, Callable from typing import Any +from opencollab.adapters._env_base import _track_shell_command from opencollab.adapters._env_process import ProcessCleanupError, run_process from opencollab.application.ports import HookPort from opencollab.domain.hooks import TOOL_SCOPED_EVENTS, HookOutcome, HookSpec, match_hooks @@ -94,7 +95,8 @@ async def _run_command(self, spec: HookSpec, payload: dict[str, Any]) -> None: f"printf '%s' {shlex.quote(stdin_bytes.decode())} | " f"{assignments} sh -c {shlex.quote(spec.command)}" ) - result = await execution_environment.exec_cmd(command, timeout=timeout) + with _track_shell_command(execution_environment): + result = await execution_environment.exec_cmd(command, timeout=timeout) if getattr(result, "timed_out", False): logger.warning("hook command timed out after %.2fs: %s", timeout, spec.command) return diff --git a/opencollab/adapters/llm/anthropic_provider.py b/opencollab/adapters/llm/anthropic_provider.py index 6a1fc61e..a0f09108 100644 --- a/opencollab/adapters/llm/anthropic_provider.py +++ b/opencollab/adapters/llm/anthropic_provider.py @@ -10,8 +10,9 @@ import inspect import json import re -from typing import Any +from typing import Any, Callable +from opencollab.adapters.llm._attempt_usage import _combine_attempt_usage from opencollab.adapters.llm.first_token import NOT_STREAMED, begin_attempt from opencollab.adapters.llm.retry import RetryTimeBudget, with_retry from opencollab.adapters.llm.tool_contracts import ( @@ -385,6 +386,8 @@ async def complete_anthropic( max_output_tokens: int | None = None, reasoning_effort: str | None = None, provider_error_time_budget: RetryTimeBudget | None = None, + before_attempt: Callable[[], int | None] | None = None, + on_attempt_usage: Callable[[Usage], None] | None = None, ) -> LLMResponse: """Single-shot completion against the Anthropic API.""" kwargs = _build_request_kwargs( @@ -400,18 +403,44 @@ async def complete_anthropic( reasoning_effort, ) - async def request_once() -> Any: - # Record the non-streamed attempt's start and unavailable first-token timing. - begin_attempt(streamed=False, unavailable_reason=NOT_STREAMED) - create = client.messages.create - return await create(**_sdk_sampling_kwargs(create, kwargs)) + failed_usages: list[Usage | None] = [] - resp = await with_retry( - request_once, - max_retries=max_retries, - retry_time_budget=provider_error_time_budget, - ) - return _parse_response(resp) + def prepare_attempt() -> None: + if before_attempt is not None and (limit := before_attempt()) is not None: + kwargs["max_tokens"] = int(limit or DEFAULT_MAX_OUTPUT_TOKENS) + + def report_failure(_exc: BaseException) -> None: + if failed_usages and (usage := failed_usages[-1]) is not None and on_attempt_usage is not None: + on_attempt_usage(usage) + + async def request_once() -> LLMResponse: + resp = None + try: + begin_attempt(streamed=False, unavailable_reason=NOT_STREAMED) + create = client.messages.create + resp = await create(**_sdk_sampling_kwargs(create, kwargs)) + return _parse_response(resp) + except BaseException as exc: + usage = _parse_usage(resp.usage) if resp is not None else getattr(exc, "usage", None) + failed_usages.append(usage) + raise + + try: + response = await with_retry( + request_once, + max_retries=max_retries, + retry_time_budget=provider_error_time_budget, + before_attempt=prepare_attempt, + on_failure=report_failure, + ) + except BaseException as exc: + usage = _combine_attempt_usage(failed_usages) + if usage is not None: + exc.usage = usage + raise + if failed_usages: + response.usage = _combine_attempt_usage([*failed_usages, response.usage]) + return response def _sdk_sampling_kwargs(create: Any, kwargs: dict[str, Any]) -> dict[str, Any]: """Preserve supported sampling on SDKs that moved it to extra_body.""" diff --git a/opencollab/adapters/llm/client.py b/opencollab/adapters/llm/client.py index 67ece79f..1fcc1be2 100644 --- a/opencollab/adapters/llm/client.py +++ b/opencollab/adapters/llm/client.py @@ -9,6 +9,7 @@ import threading import time import uuid +from contextlib import AsyncExitStack from typing import Any, Callable from urllib.parse import urlsplit @@ -38,7 +39,14 @@ from opencollab.adapters.llm.retry import RetryTimeBudget, _scoped_native_retry_budget from opencollab.adapters.llm.types import LLMResponse, model_context_window from opencollab.adapters.llm.usage_ledger import record_api_usage +from opencollab.application._session_run_usage import _observe_native_usage, _own_native_usage from opencollab.application.async_timeout import await_owned_operation +from opencollab.application.ports import ( + _NATIVE_COMPLETION_CAPABILITY, + _NATIVE_REQUEST_ADMISSION, + _LegacyCompletionMessage, + _native_completion_controls, +) from opencollab.domain.token_estimation import estimate_request_tokens # Native data-residency endpoints share the OpenAI model parameter rules. @@ -69,6 +77,7 @@ def _locked_record() -> None: except Exception: return + class LLMClient: """Client for OpenAI-compatible endpoints and native Anthropic requests. @@ -76,6 +85,11 @@ class LLMClient: OpenAI-format history per request. """ + supports_request_revalidation = _NATIVE_COMPLETION_CAPABILITY + supports_request_admission = _NATIVE_REQUEST_ADMISSION + supports_attempt_usage = _NATIVE_COMPLETION_CAPABILITY + supports_usage_settlement = _NATIVE_COMPLETION_CAPABILITY + def __init__( self, model: str = "gpt-4o", @@ -121,7 +135,6 @@ def __init__( raise ValueError("request_pool must be an absolute local directory path") self.request_pool = request_pool self.request_capacity = request_capacity - self.supports_request_admission = request_pool is not None self.model = model self.provider = provider self.max_retries = max(0, max_retries) @@ -290,6 +303,9 @@ async def complete( max_output_tokens: int | None = None, response_session_id: str | None = None, on_admitted: Callable[[], None] | None = None, + before_request: Callable[[], int] | None = None, + on_usage: Callable[[Any], None] | None = None, + on_attempt_usage: Callable[[Any], None] | None = None, ) -> LLMResponse: """Single-shot completion. Returns full response. @@ -304,18 +320,63 @@ async def complete( ``top_p`` is ``None`` by default (provider default); when set it is sent as the nucleus-sampling knob. Unset == byte-identical request as today. + + ``before_request`` runs before each physical attempt. Failed attempts + report their known usage through ``on_attempt_usage`` before retry + backoff. ``on_usage`` receives the final aggregate before slot release. """ + scope = _native_completion_controls.get() + if scope is not None and scope.callbacks and scope.matches(messages, on_admitted): + controls = scope.callbacks + if on_admitted is None: + on_admitted = controls.get("on_admitted") + if before_request is None: + before_request = controls.get("before_request") + if on_attempt_usage is None: + on_attempt_usage = controls.get("on_attempt_usage") + if on_usage is None: + on_usage = controls.get("on_usage") + controls.clear() + if scope is not None and scope.admission_callback is None: + messages = [ + dict(message) if isinstance(message, _LegacyCompletionMessage) else message for message in messages + ] + start = time.monotonic() response = None + provider_error: BaseException | None = None + settlement_error: BaseException | None = None + settled_usage = None + first_attempt = True + initial_limit: int | None = None + + def request_limit() -> int | None: + limit = max_output_tokens + if before_request is not None: + current_limit = before_request() + limit = current_limit if limit is None else min(limit, current_limit) + return limit + + def prepare_attempt() -> int | None: + nonlocal first_attempt + if first_attempt: + first_attempt = False + return initial_limit + return request_limit() + retry_budget = ( self.provider_retry_budget if self._retry_budget_explicit else _scoped_native_retry_budget(self.provider_retry_budget) ) - # Measure here so every provider path records the call timing. - with recording() as first_token: - try: - async with request_slot(self.request_pool, self.request_capacity): + try: + async with AsyncExitStack() as request_owner: + if on_usage is not None or on_attempt_usage is not None: + request_owner.enter_context(_own_native_usage()) + first_token = request_owner.enter_context(recording()) + try: + await request_owner.enter_async_context(request_slot(self.request_pool, self.request_capacity)) + initial_limit = request_limit() if on_admitted is not None: on_admitted() if self._anthropic: @@ -330,9 +391,11 @@ async def complete( thinking_params=thinking_params, tool_choice=tool_choice, top_p=top_p, - max_output_tokens=max_output_tokens, + max_output_tokens=initial_limit, reasoning_effort=reasoning_effort, provider_error_time_budget=retry_budget, + before_attempt=prepare_attempt, + on_attempt_usage=on_attempt_usage, ) elif self.wire_protocol == RESPONSES: response = await complete_responses( @@ -346,7 +409,7 @@ async def complete( thinking_params=thinking_params, tool_choice=tool_choice, top_p=top_p, - max_output_tokens=max_output_tokens, + max_output_tokens=initial_limit, reasoning_effort=reasoning_effort, prompt_cache_namespace=self._responses_prompt_cache_namespace, response_session_id=response_session_id or uuid.uuid4().hex, @@ -358,6 +421,8 @@ async def complete( stream_idle_timeout=self.stream_idle_timeout, round_timeout=self.request_timeout, provider_error_time_budget=retry_budget, + before_attempt=prepare_attempt, + on_attempt_usage=on_attempt_usage, ) else: response = await complete_openai( @@ -371,51 +436,56 @@ async def complete( thinking_params=thinking_params, tool_choice=tool_choice, top_p=top_p, - max_output_tokens=max_output_tokens, + max_output_tokens=initial_limit, reasoning_effort=reasoning_effort, provider_error_time_budget=retry_budget, + before_attempt=prepare_attempt, + on_attempt_usage=on_attempt_usage, stream=self.stream_chat, first_event_timeout=self.first_event_timeout, stream_idle_timeout=self.stream_idle_timeout, ) - transport_timing = first_token.snapshot() - try: - response.transport_timing = transport_timing - except (AttributeError, TypeError): - # Custom provider responses may reject new attributes. The usage ledger - # still records their timing. - pass + except BaseException as exc: + provider_error = exc + raise + finally: + settled_usage = getattr(response if response is not None else provider_error, "usage", None) + transport_timing = first_token.snapshot() + latency_s = time.monotonic() - start + if response is not None: + try: + response.transport_timing = transport_timing + except (AttributeError, TypeError): + pass + if settled_usage is not None: + try: + if on_usage is not None: + on_usage(settled_usage) + _observe_native_usage( + settled_usage, provider_error, transport_timing if response is not None else None, + ) + except BaseException as exc: + exc.usage = settled_usage + settlement_error = exc + finally: + try: await _record_api_usage_async( provider=self.provider, model=self.model, wire_protocol=self.wire_protocol, reasoning_effort=reasoning_effort, base_url=self.base_url, - latency_s=time.monotonic() - start, - status="success", + latency_s=latency_s, + status="success" if response is not None else "error", response=response, + error=provider_error, transport_timing=transport_timing, ) - return response except BaseException as exc: - if response is not None: - exc.usage = response.usage - else: - try: - await _record_api_usage_async( - provider=self.provider, - model=self.model, - wire_protocol=self.wire_protocol, - reasoning_effort=reasoning_effort, - base_url=self.base_url, - latency_s=time.monotonic() - start, - status="error", - error=exc, - transport_timing=first_token.snapshot(), - ) - except asyncio.CancelledError as cancellation: - usage = getattr(exc, "usage", None) - if usage is not None: - cancellation.usage = usage - raise + if settled_usage is not None: + exc.usage = settled_usage raise + finally: + if settlement_error is not None: + raise settlement_error + return response diff --git a/opencollab/adapters/llm/openai_provider.py b/opencollab/adapters/llm/openai_provider.py index 5bb015f9..51aef665 100644 --- a/opencollab/adapters/llm/openai_provider.py +++ b/opencollab/adapters/llm/openai_provider.py @@ -7,7 +7,7 @@ from __future__ import annotations import re -from typing import Any +from typing import Any, Callable from opencollab.adapters.llm._attempt_usage import _combine_attempt_usage from opencollab.adapters.llm._chat_response import _parse_response, _reported_chat_usage @@ -229,6 +229,8 @@ async def complete_openai( stream: bool = False, first_event_timeout: float | None = 180.0, stream_idle_timeout: float | None = 180.0, + before_attempt: Callable[[], int | None] | None = None, + on_attempt_usage: Callable[[Usage], None] | None = None, ) -> LLMResponse: """Single-shot completion against an OpenAI-compatible endpoint. @@ -248,36 +250,41 @@ async def complete_openai( max_output_tokens, reasoning_effort, ) - if not stream: + if stream: + kwargs.update(_STREAM_REQUEST_FIELDS) + failed_usages: list[Usage | None] = [] - async def unstreamed_once() -> Any: - # Record the non-streamed attempt's start and unavailable first-token timing. - begin_attempt(streamed=False, unavailable_reason=NOT_STREAMED) - return await client.chat.completions.create(**kwargs) + def prepare_attempt() -> None: + if before_attempt is not None and (limit := before_attempt()) is not None: + field = "max_completion_tokens" if _uses_reasoning_request_fields(model) else "max_tokens" + kwargs[field] = limit - resp = await with_retry( - unstreamed_once, - max_retries=max_retries, - retry_time_budget=provider_error_time_budget, - ) - return _parse_response(resp, kwargs["messages"], kwargs.get("tools")) - - stream_kwargs = {**kwargs, **_STREAM_REQUEST_FIELDS} - failed_usages: list[Usage | None] = [] + def report_failure(_exc: BaseException) -> None: + if failed_usages and (usage := failed_usages[-1]) is not None and on_attempt_usage is not None: + on_attempt_usage(usage) async def request_once() -> LLMResponse: # Retry stream creation and draining together. Creation returns at headers, # so its success alone cannot cover a later stream failure. state = None + resp = None try: + if not stream: + begin_attempt(streamed=False, unavailable_reason=NOT_STREAMED) + resp = await client.chat.completions.create(**kwargs) + return _parse_response(resp, kwargs["messages"], kwargs.get("tools")) state = await _create_and_consume_chat_stream( - client, stream_kwargs, first_event_timeout, stream_idle_timeout + client, kwargs, first_event_timeout, stream_idle_timeout ) return _stream_state_to_response( - state, stream_kwargs["messages"], stream_kwargs.get("tools") + state, kwargs["messages"], kwargs.get("tools") ) except BaseException as exc: - usage = _reported_chat_usage(state.usage_object) if state is not None else getattr(exc, "usage", None) + usage = ( + _reported_chat_usage(state.usage_object) if state is not None + else _reported_chat_usage(resp.usage) if resp is not None + else getattr(exc, "usage", None) + ) failed_usages.append(usage) raise @@ -286,6 +293,8 @@ async def request_once() -> LLMResponse: request_once, max_retries=max_retries, retry_time_budget=provider_error_time_budget, + before_attempt=prepare_attempt, + on_failure=report_failure, ) except BaseException as exc: usage = _combine_attempt_usage(failed_usages) diff --git a/opencollab/adapters/llm/responses_provider.py b/opencollab/adapters/llm/responses_provider.py index acc11ed7..f125edb6 100644 --- a/opencollab/adapters/llm/responses_provider.py +++ b/opencollab/adapters/llm/responses_provider.py @@ -7,7 +7,7 @@ import logging import os from dataclasses import dataclass, field -from typing import Any +from typing import Any, Callable from opencollab.adapters.llm._responses_instructions import _check_instructions_echo from opencollab.adapters.llm._responses_output import ( @@ -685,6 +685,8 @@ async def complete_responses( round_timeout: float | None = None, provider_error_time_budget: RetryTimeBudget | None = None, stream: bool = True, + before_attempt: Callable[[], int | None] | None = None, + on_attempt_usage: Callable[[Usage], None] | None = None, ) -> LLMResponse: """Run one locally replayable Responses request and require typed completion.""" external_isolation = os.environ.get("OPENCOLLAB_EXTERNAL_PROVIDER_ISOLATION") == "1" @@ -710,6 +712,25 @@ async def complete_responses( capabilities = model_capabilities(model) stream = stream and capabilities.supports_responses_streaming failed_usages: list[Usage | None] = [] + callback_timeout: asyncio.TimeoutError | None = None + + def prepare_attempt() -> None: + nonlocal callback_timeout + try: + if before_attempt is not None and (limit := before_attempt()) is not None: + kwargs["max_output_tokens"] = limit + except asyncio.TimeoutError as exc: + callback_timeout = exc + raise + + def report_failure(_exc: BaseException) -> None: + nonlocal callback_timeout + try: + if failed_usages and (usage := failed_usages[-1]) is not None and on_attempt_usage is not None: + on_attempt_usage(usage) + except asyncio.TimeoutError as exc: + callback_timeout = exc + raise async def request_once() -> LLMResponse: terminal = None @@ -740,7 +761,8 @@ async def request_once() -> LLMResponse: converted_tools, ) except BaseException as exc: - failed_usages.append(_reported_responses_usage(terminal) or getattr(exc, "usage", None)) + usage = _reported_responses_usage(terminal) or getattr(exc, "usage", None) + failed_usages.append(usage) raise if retry_budget is not None: @@ -757,8 +779,12 @@ async def run() -> LLMResponse: if retry_budget is not None: return await with_retry( bounded_request_once, max_retries=retry_limit, retry_time_budget=retry_budget, + before_attempt=prepare_attempt, + on_failure=report_failure, ) - return await with_retry(request_once, max_retries=retry_limit) + return await with_retry( + request_once, max_retries=retry_limit, before_attempt=prepare_attempt, on_failure=report_failure, + ) try: try: @@ -767,6 +793,8 @@ async def run() -> LLMResponse: else: response = await asyncio.wait_for(run(), timeout=round_timeout) except asyncio.TimeoutError as exc: + if exc is callback_timeout or round_timeout is None: + raise raise ResponsesProtocolError(f"Responses round deadline exceeded after {round_timeout:g}s") from exc except BaseException as exc: usage = _combine_responses_usage(failed_usages) diff --git a/opencollab/adapters/llm/retry.py b/opencollab/adapters/llm/retry.py index c6bf204e..a72f7e01 100644 --- a/opencollab/adapters/llm/retry.py +++ b/opencollab/adapters/llm/retry.py @@ -139,6 +139,8 @@ async def with_retry( max_retries: int | None, *, retry_time_budget: RetryTimeBudget | None = None, + before_attempt: Callable[[], None] | None = None, + on_failure: Callable[[BaseException], None] | None = None, ) -> Any: """Retry transient provider errors with exponential backoff. @@ -146,14 +148,19 @@ async def with_retry( """ attempt = 0 while True: + if before_attempt is not None: + before_attempt() started = time.monotonic() try: return await call_factory() - except Exception as e: - if not is_retryable_error(e): + except BaseException as e: + elapsed = time.monotonic() - started + if on_failure is not None: + on_failure(e) + if not isinstance(e, Exception) or not is_retryable_error(e): raise retry_time_available = retry_time_budget is None or retry_time_budget.consume( - time.monotonic() - started + elapsed ) if max_retries is not None and attempt >= max_retries: raise diff --git a/opencollab/adapters/llm/types.py b/opencollab/adapters/llm/types.py index 96ef7033..2d26bcf7 100644 --- a/opencollab/adapters/llm/types.py +++ b/opencollab/adapters/llm/types.py @@ -13,6 +13,13 @@ ) +class _NativeUsageOrigin: + __slots__ = () + + def __deepcopy__(self, _memo: dict[int, Any]) -> _NativeUsageOrigin: + return self + + @dataclass class Usage: """Token accounting for one completion. @@ -23,6 +30,9 @@ class Usage: added again to compute the budget total. """ + # Retain request identity through copies without adding it to usage telemetry. + __slots__ = ("__dict__", "__weakref__", "_native_usage_origin") + input_tokens: int = 0 output_tokens: int = 0 cache_read_tokens: int | None = 0 @@ -39,6 +49,9 @@ class Usage: context_output_tokens: int | None = None context_tokens_estimated: bool | None = None + def __post_init__(self) -> None: + self._native_usage_origin = _NativeUsageOrigin() + @property def total_tokens(self) -> int: return self.input_tokens + self.output_tokens diff --git a/opencollab/adapters/safe_anchored_files.py b/opencollab/adapters/safe_anchored_files.py index aeff6f22..7c674dfe 100644 --- a/opencollab/adapters/safe_anchored_files.py +++ b/opencollab/adapters/safe_anchored_files.py @@ -71,12 +71,14 @@ def _relative_components( value = os.fspath(path) if not value or "\0" in value or os.path.isabs(value): raise ValueError("descriptor-relative path must be non-empty relative text") - normalized = os.path.normpath(value) - if normalized in {"", ".", ".."} or normalized.startswith(f"..{os.sep}"): - raise PermissionError(f"path escapes anchored directory: {value}") - parts = normalized.split(os.sep) - if any(part in {"", ".", ".."} for part in parts): + parts = [part for part in value.split(os.sep) if part not in {"", "."}] + if not parts or parts[-1] == "..": raise PermissionError(f"path escapes anchored directory: {value}") + depth = 0 + for part in parts[:-1]: + depth += -1 if part == ".." else 1 + if depth < 0: + raise PermissionError(f"path escapes anchored directory: {value}") return parts[:-1], parts[-1] def open_parent_beneath( @@ -89,9 +91,22 @@ def open_parent_beneath( """Open a target parent without resolving a component by pathname.""" _require_anchor_path(root_fd, root_path) parents, name = _relative_components(path) - current_fd = os.dup(root_fd) + # A directory followed by .. must exist and keep its parent descriptor open. + existing_only: set[int] = set() + parent_steps = 0 + for index in range(len(parents) - 1, -1, -1): + if parents[index] == "..": + parent_steps += 1 + elif parent_steps: + existing_only.add(index) + parent_steps -= 1 + opened = [os.dup(root_fd)] try: - for component in parents: + for index, component in enumerate(parents): + if component == "..": + os.close(opened.pop()) + continue + current_fd = opened[-1] try: next_fd = os.open( component, @@ -99,7 +114,7 @@ def open_parent_beneath( dir_fd=current_fd, ) except FileNotFoundError: - if not create: + if not create or index in existing_only: raise try: os.mkdir(component, 0o700, dir_fd=current_fd) @@ -110,12 +125,13 @@ def open_parent_beneath( _directory_open_flags(), dir_fd=current_fd, ) - os.close(current_fd) - current_fd = next_fd - return current_fd, name - except BaseException: - os.close(current_fd) - raise + if index not in existing_only: + os.close(opened.pop()) + opened.append(next_fd) + return opened.pop(), name + finally: + for fd in opened: + os.close(fd) @contextmanager def open_regular_read_at( diff --git a/opencollab/adapters/safe_paths.py b/opencollab/adapters/safe_paths.py index ac706cda..43d8d3e7 100644 --- a/opencollab/adapters/safe_paths.py +++ b/opencollab/adapters/safe_paths.py @@ -28,4 +28,4 @@ def canonicalize_system_path(path: str | os.PathLike[str]) -> Path: value = os.fspath(path) if not value or "\0" in value: raise ValueError("path must be non-empty text without NUL bytes") - return _canonicalize_system_alias(Path(os.path.abspath(value))) + return _canonicalize_system_alias(Path(value).absolute()) diff --git a/opencollab/adapters/safety.py b/opencollab/adapters/safety.py index 5691346b..3d4b8bee 100644 --- a/opencollab/adapters/safety.py +++ b/opencollab/adapters/safety.py @@ -14,8 +14,10 @@ from __future__ import annotations import os +import posixpath import re import shlex +import stat from typing import Awaitable, Callable # Commands that warrant human confirmation (not outright blocked) @@ -97,10 +99,13 @@ class SandboxInterceptor: await interceptor.check_cmd_interactive("rm -rf ./build", confirm_fn) # asks user """ - def __init__(self, workspace_root: str, allowed_dirs: list[str] | None = None): - self.root = os.path.abspath(workspace_root) + def __init__( + self, workspace_root: str, allowed_dirs: list[str] | None = None, *, remote_filesystem: bool = False, + ): + self.root = os.path.realpath(workspace_root) + self._remote_workspace = workspace_root if remote_filesystem and posixpath.isabs(workspace_root) else None # Additional allowed directories (e.g., /tmp for scratch) - self.allowed_roots = [self.root] + [os.path.abspath(d) for d in (allowed_dirs or [])] + self.allowed_roots = [self.root] + [os.path.realpath(d) for d in (allowed_dirs or [])] def check_path(self, target_path: str) -> str: """Resolve and validate a file path. Returns absolute safe path or raises. @@ -109,9 +114,14 @@ def check_path(self, target_path: str) -> str: """ # Resolve relative to workspace if not os.path.isabs(target_path): - resolved = os.path.abspath(os.path.join(self.root, target_path)) + resolved = os.path.join(self.root, target_path) else: - resolved = os.path.abspath(target_path) + resolved = target_path + prefix = os.sep + for component in resolved.split(os.sep): + if component == ".." and not stat.S_ISDIR(os.stat(prefix).st_mode): + raise NotADirectoryError(prefix) + prefix = os.path.join(prefix, component) resolved = os.path.realpath(resolved) # Check against all allowed roots @@ -128,6 +138,17 @@ def check_path(self, target_path: str) -> str: f"(allowed roots: {self.allowed_roots})" ) + def check_remote_file_path(self, target_path: str) -> str | None: + workspace = self._remote_workspace + if workspace is None: + return None + path = posixpath.join(workspace, target_path) + root = posixpath.normpath(workspace) + if posixpath.commonpath((root, posixpath.normpath(path))) != root: + raise PermissionError(f"Path escapes workspace: '{target_path}' (workspace: {workspace})") + # Native file operations validate the actual parent and symlink target. + return path + def check_cmd(self, cmd: str) -> None: """Hard-block destructive commands. Raises PermissionError.""" for pattern in BLOCKED_RE: diff --git a/opencollab/adapters/single2_safety.py b/opencollab/adapters/single2_safety.py index 6c373306..4a159fbc 100644 --- a/opencollab/adapters/single2_safety.py +++ b/opencollab/adapters/single2_safety.py @@ -28,6 +28,10 @@ def __init__(self, policy: SafetyPolicyPort, workspace: str): def check_path(self, target_path: str) -> str: return self._policy.check_path(target_path) + def check_remote_file_path(self, target_path: str) -> str | None: + check = getattr(self._policy, "check_remote_file_path", None) + return check(target_path) if callable(check) else None + def check_cmd(self, cmd: str) -> None: self._policy.check_cmd(cmd) self._check_container_search(cmd) diff --git a/opencollab/adapters/storage.py b/opencollab/adapters/storage.py index e2f60f41..83c040f0 100644 --- a/opencollab/adapters/storage.py +++ b/opencollab/adapters/storage.py @@ -20,6 +20,9 @@ write_regular_bytes_atomic, write_regular_file_atomic, ) +from opencollab.application.ports import SnapshotCapacityError, SnapshotSequenceConflict +from opencollab.application.session_snapshot import decode_anthropic_history +from opencollab.domain.session import same_snapshot_value MAX_SESSION_SNAPSHOT_BYTES = 64 * 1024 * 1024 _JSON_WRITE_CHUNK_CHARS = 64 * 1024 @@ -127,7 +130,7 @@ def write(self, text: str) -> int: payload = text[offset : offset + _JSON_WRITE_CHUNK_CHARS].encode("utf-8") next_size = self._bytes_written + len(payload) if next_size > MAX_SESSION_SNAPSHOT_BYTES: - raise ValueError( + raise SnapshotCapacityError( "session snapshot exceeds " f"{MAX_SESSION_SNAPSHOT_BYTES} UTF-8 bytes while writing: " f"{self._path}" @@ -183,7 +186,7 @@ def load_snapshot(self, path: str, system_prompt: str) -> dict[str, Any]: ) def _load_snapshot_unlocked( - self, path: str, system_prompt: str, *, read_journal: bool = True, + self, path: str, fallback_system_prompt: str | None, *, read_journal: bool = True, ) -> dict[str, Any]: """Load a locked snapshot or a standalone read-only export.""" try: @@ -282,7 +285,8 @@ def _load_snapshot_unlocked( raise ValueError( "Invalid session snapshot: expected at least one message" ) - messages = [{"role": "system", "content": system_prompt}] + if fallback_system_prompt is not None: + messages = [{"role": "system", "content": fallback_system_prompt}] snapshot["messages"] = messages return snapshot @@ -446,6 +450,72 @@ def append_snapshot_delta( seen_result_hashes_added: list[str] | None = None, ) -> None: """Durably append one absolute, idempotent autosave delta.""" + record = self._snapshot_delta_record( + sequence=sequence, + replace_from=replace_from, + messages=messages, + meta=meta, + seen_result_hashes_reset=seen_result_hashes_reset, + seen_result_hashes_added=seen_result_hashes_added, + ) + self._append_journal_record(path, record) + + def append_snapshot_replacement( + self, + path: str, + messages: list[dict[str, Any]], + *, + meta: dict[str, Any], + sequence: int, + reject_conflicts: bool = False, + ) -> bool: + state = meta.get("session_state") + seen_result_hashes = state.get("seen_result_hashes", []) if isinstance(state, dict) else [] + record = self._snapshot_delta_record( + sequence=sequence, + replace_from=0, + messages=messages, + meta=meta, + seen_result_hashes_reset=True, + seen_result_hashes_added=seen_result_hashes, + ) + with _journal_operation_lock(path): + try: + persisted = self._load_snapshot_unlocked(path, fallback_system_prompt=None) + except FileNotFoundError: + pass + else: + persisted_sequence = self._snapshot_sequence(persisted) + if sequence <= persisted_sequence: + if reject_conflicts: + candidate = {**meta, _AUTOSAVE_SEQUENCE_KEY: sequence, "messages": messages} + if not self._checkpoint_states_equal(persisted, candidate): + raise SnapshotSequenceConflict( + f"refusing replacement sequence {sequence}; " + f"it conflicts with persisted state at sequence {persisted_sequence}" + ) + return False + stored_messages = persisted["messages"] + replace_from = min(len(stored_messages), len(messages)) + for index in range(replace_from): + if not same_snapshot_value(stored_messages[index], messages[index]): + replace_from = index + break + record["replace_from"] = replace_from + record["messages"] = messages[replace_from:] + self._append_journal_record_unlocked(path, record) + return True + + def _snapshot_delta_record( + self, + *, + sequence: int, + replace_from: int, + messages: list[dict[str, Any]], + meta: dict[str, Any], + seen_result_hashes_reset: bool, + seen_result_hashes_added: list[str] | None, + ) -> dict[str, Any]: if ( isinstance(sequence, bool) or not isinstance(sequence, int) @@ -474,7 +544,7 @@ def append_snapshot_delta( journal_state = dict(raw_state) journal_state.pop("seen_result_hashes", None) journal_meta["session_state"] = journal_state - record = { + return { "journal_version": _AUTOSAVE_JOURNAL_VERSION, "sequence": sequence, "replace_from": replace_from, @@ -484,7 +554,6 @@ def append_snapshot_delta( "seen_result_hashes_reset": seen_result_hashes_reset, "seen_result_hashes_added": seen_added, } - self._append_journal_record(path, record) def checkpoint_snapshot( self, @@ -501,6 +570,7 @@ def checkpoint_snapshot( or sequence < 0 ): raise ValueError("autosave sequence must be a non-negative integer") + self._validate_messages(messages) self._ensure_parent(path) obj = { **meta, @@ -510,7 +580,7 @@ def checkpoint_snapshot( with _journal_operation_lock(path): persisted_sequence = self._newest_persisted_sequence(path) if persisted_sequence > sequence: - raise ValueError( + raise SnapshotSequenceConflict( "refusing stale checkpoint sequence " f"{sequence}; persisted sequence {persisted_sequence} is newer" ) @@ -524,23 +594,23 @@ def checkpoint_snapshot( # Multiple records at one cursor are ambiguous: replay skips # the second and later records, so accepting this checkpoint # would clear data that cannot be proven equivalent. - raise ValueError( + raise SnapshotSequenceConflict( "refusing checkpoint sequence " f"{sequence}; persisted journal contains duplicate sequence records" ) - if same_sequence_records: - # A same-sequence journal record may belong to another writer. - # Only compact it when the proposed checkpoint is exactly the - # state obtained by replaying the durable base and journal; - # otherwise retaining the journal is safer than silently - # discarding a peer's update. + if persisted_sequence == sequence > 0: + # A peer can own this sequence in either the base or the journal. persisted = self._load_snapshot_unlocked(path, "") if not self._checkpoint_states_equal(persisted, obj): - raise ValueError( + source = "journal" if same_sequence_records else "snapshot" + raise SnapshotSequenceConflict( "refusing checkpoint sequence " - f"{sequence}; it conflicts with persisted journal state" + f"{sequence}; it conflicts with persisted {source} state" ) - self._atomic_json_write(path, obj) + try: + self._atomic_json_write(path, obj) + except SnapshotCapacityError: + self._atomic_json_write(path, obj, indent=None) write_regular_bytes_atomic( self._journal_path(path), b"", @@ -564,47 +634,50 @@ def _encode_journal_record(record: dict[str, Any]) -> str: def _append_journal_record(self, path: str, record: dict[str, Any]) -> None: with _journal_operation_lock(path): - journal_path = self._journal_path(path) - payload = self._encode_journal_record(record) - payload_size = len(payload.encode("utf-8")) - if payload_size > MAX_SESSION_SNAPSHOT_BYTES: - raise ValueError( - "autosave journal record exceeds " - f"{MAX_SESSION_SNAPSHOT_BYTES} UTF-8 bytes while writing: " - f"{journal_path}" + self._append_journal_record_unlocked(path, record) + + def _append_journal_record_unlocked(self, path: str, record: dict[str, Any]) -> None: + journal_path = self._journal_path(path) + payload = self._encode_journal_record(record) + payload_size = len(payload.encode("utf-8")) + if payload_size > MAX_SESSION_SNAPSHOT_BYTES: + raise ValueError( + "autosave journal record exceeds " + f"{MAX_SESSION_SNAPSHOT_BYTES} UTF-8 bytes while writing: " + f"{journal_path}" + ) + append_start: int | None = None + try: + with open_regular_text_append(journal_path, readable=True) as handle: + current_size = os.fstat(handle.fileno()).st_size + complete_size = self._complete_journal_size( + handle.fileno(), + current_size, ) - append_start: int | None = None - try: - with open_regular_text_append(journal_path, readable=True) as handle: - current_size = os.fstat(handle.fileno()).st_size - complete_size = self._complete_journal_size( - handle.fileno(), - current_size, + if complete_size < current_size: + os.ftruncate(handle.fileno(), complete_size) + current_size = complete_size + if current_size + payload_size > MAX_SESSION_SNAPSHOT_BYTES: + raise ValueError( + "autosave journal exceeds " + f"{MAX_SESSION_SNAPSHOT_BYTES} UTF-8 bytes while writing: " + f"{journal_path}" ) - if complete_size < current_size: - os.ftruncate(handle.fileno(), complete_size) - current_size = complete_size - if current_size + payload_size > MAX_SESSION_SNAPSHOT_BYTES: - raise ValueError( - "autosave journal exceeds " - f"{MAX_SESSION_SNAPSHOT_BYTES} UTF-8 bytes while writing: " - f"{journal_path}" - ) - append_start = current_size - write_locked_text(handle, payload) - handle.flush() - os.fsync(handle.fileno()) - except BaseException as append_error: - if append_start is not None: - try: - # Close the append handle before truncating so a buffered - # close cannot write the failed record back afterward. - with open_regular_text_append(journal_path, readable=True) as handle: - os.ftruncate(handle.fileno(), append_start) - os.fsync(handle.fileno()) - except BaseException as rollback_error: - raise append_error from rollback_error - raise + append_start = current_size + write_locked_text(handle, payload) + handle.flush() + os.fsync(handle.fileno()) + except BaseException as append_error: + if append_start is not None: + try: + # Close the append handle before truncating so a buffered + # close cannot write the failed record back afterward. + with open_regular_text_append(journal_path, readable=True) as handle: + os.ftruncate(handle.fileno(), append_start) + os.fsync(handle.fileno()) + except BaseException as rollback_error: + raise append_error from rollback_error + raise @staticmethod def _complete_journal_size(fd: int, size: int) -> int: @@ -679,26 +752,33 @@ def _newest_persisted_sequence(self, path: str) -> int: def _checkpoint_states_equal( persisted: dict[str, Any], candidate: dict[str, Any] ) -> bool: - """Compare a replayed state with a checkpoint after replay normalization. - - Journal replay materializes ``session_state.seen_result_hashes`` and - sorts it, while a caller's full checkpoint payload may omit that empty - field or retain insertion order. Those representations are - semantically identical; comparing the raw dictionaries would reject a - legitimate idempotent compaction and leave the journal unbounded. - """ - def normalize(value: dict[str, Any]) -> dict[str, Any]: normalized = dict(value) raw_state = normalized.get("session_state") state = dict(raw_state) if isinstance(raw_state, dict) else {} + # Journal replay materializes and sorts the seen-result set. seen = state.get("seen_result_hashes", []) if isinstance(seen, list) and all(isinstance(item, str) for item in seen): state["seen_result_hashes"] = sorted(set(seen)) normalized["session_state"] = state return normalized - return normalize(persisted) == normalize(candidate) + persisted = normalize(persisted) + candidate = normalize(candidate) + if same_snapshot_value(persisted, candidate): + history = persisted["session_state"].get("anthropic_history") + return history is None or decode_anthropic_history(history, persisted["messages"]) is not None + + for snapshot in (persisted, candidate): + state = snapshot["session_state"] + history = state.get("anthropic_history") + if history is None: + return False + decoded = decode_anthropic_history(history, snapshot["messages"]) + if decoded is None: + return False + state["anthropic_history"] = {**history, **decoded} + return same_snapshot_value(persisted, candidate) @staticmethod def _snapshot_sequence(snapshot: dict[str, Any]) -> int: @@ -742,11 +822,14 @@ def _read_snapshot_text(path: str) -> str: ).decode("utf-8") @staticmethod - def _atomic_json_write(path: str, value: Any) -> None: + def _atomic_json_write(path: str, value: Any, *, indent: int | None = 2) -> None: """Durably replace one JSON file without exposing a partial snapshot.""" def serialize(handle: BinaryIO) -> None: writer = _BoundedUTF8Writer(handle, path=path) - json.dump(value, writer, ensure_ascii=False, indent=2) + if indent is None: + json.dump(value, writer, ensure_ascii=False, separators=(",", ":")) + else: + json.dump(value, writer, ensure_ascii=False, indent=indent) write_regular_file_atomic( path, diff --git a/opencollab/adapters/tools/_file_lock_worker.py b/opencollab/adapters/tools/_file_lock_worker.py new file mode 100644 index 00000000..8ceb9e1c --- /dev/null +++ b/opencollab/adapters/tools/_file_lock_worker.py @@ -0,0 +1,89 @@ +"""Hold one ordered group of existing local file locks for a parent process.""" + +import contextlib +import errno +import fcntl +import json +import math +import os +import select +import sys +import time +from collections.abc import Iterator + +from filelock import FileLock, Timeout + + +@contextlib.contextmanager +def _try_file_lock(path: str, *, shared: bool = False) -> Iterator[FileLock | None]: + if shared: + try: + fd = os.open(path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o666) + except FileNotFoundError as exc: + if os.path.isdir(os.path.dirname(path)): + raise Timeout(path) from exc + raise + try: + try: + fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise Timeout(path) from exc + except OSError as exc: + if exc.errno != errno.ENOSYS: + raise + else: + try: + # FileLock can unlink its inode between open and flock. + if not os.fstat(fd).st_nlink: + raise Timeout(path) + yield None + finally: + fcntl.flock(fd, fcntl.LOCK_UN) + return + finally: + # Other shared holders still coordinate through this inode. + os.close(fd) + + with FileLock(path, timeout=0) as lock: + yield lock + + +def _reply(status: str, detail: str = "") -> None: + print(json.dumps((status, detail)), flush=True) + + +def main() -> None: + setup = sys.stdin.buffer.readline() + if not setup: + return + paths, timeout, poll = json.loads(setup) + owner = select.poll() + owner.register(int(sys.argv[1]), select.POLLIN) + try: + with contextlib.ExitStack() as held: + for path, lock_path, shared in paths: + deadline = time.monotonic() + timeout + while not owner.poll(0): + try: + held.enter_context(_try_file_lock(lock_path, shared=shared)) + except Timeout: + remaining = deadline - time.monotonic() + if remaining <= 0: + _reply("timeout", f"timed out acquiring write lock for {path}") + return + if owner.poll(math.ceil(min(poll, remaining) * 1000)): + return + else: + break + else: + return + _reply("ready") + owner.poll() + except BrokenPipeError: + return + except OSError as exc: + _reply("error", str(exc)) + + +if __name__ == "__main__": + main() diff --git a/opencollab/adapters/tools/_paths.py b/opencollab/adapters/tools/_paths.py index b71560f3..bb3c9785 100644 --- a/opencollab/adapters/tools/_paths.py +++ b/opencollab/adapters/tools/_paths.py @@ -1,11 +1,4 @@ -"""Shared path-jail helper for filesystem tools. - -Every filesystem tool resolves a user-supplied path through the sandbox's -``check_path`` before touching the environment, so no path can escape the -workspace (ref: ``adapters/safety.py``). This module is the single copy of that -guard; ``fs.py`` and ``apply_patch.py`` delegate here instead of repeating the -``policy and policy.check_path(...)`` dance. -""" +"""Check workspace paths for filesystem tools.""" from __future__ import annotations @@ -15,13 +8,17 @@ from opencollab.application.tool_execution import ToolRuntime -def checked_path(runtime: ToolRuntime, path: str) -> str: - """Jail ``path`` to the workspace via the runtime's safety policy. +def _checked_remote_path(runtime: ToolRuntime, path: str) -> str | None: + check = getattr(runtime.safety_policy, "check_remote_file_path", None) + return check(path) if callable(check) else None + - Returns the resolved absolute path, or ``path`` unchanged when no policy is - wired (yolo / tests). Raises ``PermissionError`` if the path escapes the - workspace — callers that prefer a soft error catch it and return a string. - """ +def checked_path(runtime: ToolRuntime, path: str, *, native_file: bool = False) -> str: + """Apply the runtime policy and preserve remote components for native file operations.""" + if native_file: + remote_path = _checked_remote_path(runtime, path) + if remote_path is not None: + return remote_path policy = runtime.safety_policy checked = policy.check_path(path) if policy else path environment = runtime.environment @@ -32,3 +29,12 @@ def checked_path(runtime: ToolRuntime, path: str) -> str: if os.path.isabs(checked) and os.path.commonpath((host_root, checked)) == host_root: checked = posixpath.join(workspace, os.path.relpath(checked, host_root)) return checked + + +async def checked_search_path(runtime: ToolRuntime, path: str) -> str: + resolve = getattr(runtime.environment, "resolve_search_path", None) + if callable(resolve): + remote_path = _checked_remote_path(runtime, path) + if remote_path is not None: + return await resolve(remote_path) + return checked_path(runtime, path) diff --git a/opencollab/adapters/tools/apply_patch.py b/opencollab/adapters/tools/apply_patch.py index 20497067..cee8f016 100644 --- a/opencollab/adapters/tools/apply_patch.py +++ b/opencollab/adapters/tools/apply_patch.py @@ -108,7 +108,7 @@ async def execute_with_runtime( if env is None: return "Error: no execution environment available." - path = checked_path(runtime, path) + path = checked_path(runtime, path, native_file=True) try: async with host_write_lock(path, env): diff --git a/opencollab/adapters/tools/base.py b/opencollab/adapters/tools/base.py index 59c5c33f..7b661676 100644 --- a/opencollab/adapters/tools/base.py +++ b/opencollab/adapters/tools/base.py @@ -4,14 +4,23 @@ import asyncio import contextlib +import errno import hashlib +import json import os +import posixpath +import resource +import socket +import subprocess +import sys import tempfile -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Sequence from typing import Any from filelock import FileLock, Timeout +from opencollab.adapters.tools._file_lock_worker import _try_file_lock +from opencollab.application.async_timeout import await_owned_operation from opencollab.application.tool_execution import DeferredCall, ToolRuntime HOST_WRITE_LOCK_TIMEOUT_SECONDS = 10.0 @@ -29,9 +38,193 @@ def _host_lock_path(path: str, env: Any) -> str: digest = hashlib.sha256(identity.encode("utf-8", errors="surrogatepass")).hexdigest() return os.path.join(_host_lock_root(), f"{digest}.lock") +def _write_lock_paths( + paths: Sequence[str], env: Any, *, workspace: str | None = None, +) -> list[str]: + local = bool(getattr(env, "local_filesystem", False)) + path_module = os.path if local else posixpath + workspace = path_module.abspath(workspace or getattr(env, "workspace", "") or ".") + if local: + workspace = os.path.realpath(workspace) + targets = set() + for path in paths: + target = path_module.normpath(path_module.join(workspace, path)) + if local: + target = os.path.join(os.path.realpath(os.path.dirname(target)), os.path.basename(target)) + targets.add(target) + parent = path_module.dirname(target) + while parent != workspace and parent.startswith(workspace.rstrip("/") + "/"): + targets.add(parent) + parent = path_module.dirname(parent) + return sorted(targets) + + +def _local_file_lock_capacity() -> int: + limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + if limit == resource.RLIM_INFINITY: + return sys.maxsize + # Leave descriptors for the environment's subprocesses and pending I/O. + return max(1, limit - max(16, limit // 4)) + + +def _available_local_file_locks(capacity: int) -> int: + try: + return capacity - len(os.listdir("/dev/fd")) + except OSError as exc: + return 0 if exc.errno == errno.EMFILE else capacity + + +async def _reap_lock_holders(processes: list[subprocess.Popen]) -> None: + loop = asyncio.get_running_loop() + deadline = loop.time() + 2 + while pending := [process for process in processes if process.poll() is None]: + if loop.time() >= deadline: + for process in pending: + process.kill() + deadline = float("inf") + await asyncio.sleep(HOST_WRITE_LOCK_POLL_SECONDS) + + +async def _prepare_lock_holder( + channel: socket.socket, paths: Sequence[str], env: Any, shared_paths: frozenset[str], +) -> None: + loop = asyncio.get_running_loop() + channel.setblocking(False) + options = ( + [(path, _host_lock_path(path, env), path in shared_paths) for path in paths], + HOST_WRITE_LOCK_TIMEOUT_SECONDS, HOST_WRITE_LOCK_POLL_SECONDS, + ) + await loop.sock_sendall(channel, json.dumps(options).encode() + b"\n") + response = bytearray() + while chunk := await loop.sock_recv(channel, 4096): + response.extend(chunk) + if chunk.endswith(b"\n"): + break + else: + raise RuntimeError("local write lock holder exited during acquisition") + status, detail = json.loads(response) + if status == "timeout": + raise TimeoutError(detail) + if status != "ready": + raise RuntimeError(f"local write lock holder failed: {detail}") + + +@contextlib.asynccontextmanager +async def _hold_lock_groups( + paths: Sequence[str], env: Any, group_size: int, shared_paths: frozenset[str], +) -> AsyncIterator[None]: + owner = asyncio.current_task() + assert owner is not None + processes: list[subprocess.Popen] = [] + ready = 0 + worker = os.path.join(os.path.dirname(__file__), "_file_lock_worker.py") + + async def watch_exit() -> None: + while True: + for index in range(ready): + returncode = processes[index].poll() + if returncode is not None: + owner.cancel(f"local write lock holder exited unexpectedly ({returncode})") + return + await asyncio.sleep(HOST_WRITE_LOCK_POLL_SECONDS) + + # One group can use its startup socket as the ownership signal. + shared = len(paths) > group_size + read_end, write_end = os.pipe() if shared else (-1, -1) + ownership_socket = None + monitor = asyncio.create_task(watch_exit()) + try: + for start in range(0, len(paths), group_size): + setup, child = socket.socketpair() + if not shared: + ownership_socket = setup + try: + with child: + # Isolate terminal interrupts and avoid per-child pidfds. + processes.append(subprocess.Popen( + [sys.executable, worker, str(read_end if shared else 0)], start_new_session=True, + stdin=child, stdout=child, pass_fds=(read_end,) if shared else (), + )) + await _prepare_lock_holder(setup, paths[start:start + group_size], env, shared_paths) + finally: + if shared: + setup.close() + ready += 1 + if shared: + os.close(read_end) + read_end = -1 + yield + finally: + monitor.cancel() + if read_end >= 0: + os.close(read_end) + if shared: + os.close(write_end) + if ownership_socket is not None: + ownership_socket.close() + await await_owned_operation(_reap_lock_holders(processes), propagate_cancellation=True) + + +@contextlib.asynccontextmanager +async def _local_write_locks( + paths: Sequence[str], env: Any, *, shared_paths: frozenset[str] = frozenset(), +) -> AsyncIterator[FileLock | None]: + capacity = _local_file_lock_capacity() + if len(paths) == 1 or len(paths) <= _available_local_file_locks(capacity): + async with contextlib.AsyncExitStack() as locks: + target = None + try: + for path in paths: + target = await locks.enter_async_context(_target_write_lock(path, env, shared=path in shared_paths)) + except OSError as exc: + if exc.errno != errno.EMFILE: + raise + else: + yield target + return + # Every holder uses the same per-path locks. Contention order stays the + # same across groups, direct callers and older installations. + async with _hold_lock_groups(paths, env, min(512, capacity), shared_paths): + yield None + + +@contextlib.asynccontextmanager +async def host_write_locks(paths: Sequence[str], env: Any) -> AsyncIterator[Any]: + """Own edited paths and their ancestors through publication or recovery.""" + if env is None or getattr(env, "local_filesystem", False): + async with _local_write_locks(_write_lock_paths(paths, env), env) as target: + yield target + return + grouped = getattr(env, "file_write_locks", None) + if callable(grouped): + async with grouped(paths) as target: + yield target + return + if getattr(env, "file_write_lock_scope", None) == "workspace": + async with _target_write_lock(paths[0], env) as target: + yield target + return + async with contextlib.AsyncExitStack() as locks: + target = None + for path in _write_lock_paths(paths, env): + target = await locks.enter_async_context(_target_write_lock(path, env)) + yield target + + @contextlib.asynccontextmanager async def host_write_lock(path: str, env: Any) -> AsyncIterator[Any]: - """Hold the environment's file lock across the complete native edit.""" + if env is None or getattr(env, "local_filesystem", False): + paths = _write_lock_paths((path,), env) + # The single target sorts after its ancestors. + async with _local_write_locks(paths, env, shared_paths=frozenset(paths[:-1])) as target: + yield target + return + async with host_write_locks((path,), env) as target: + yield target + + +@contextlib.asynccontextmanager +async def _target_write_lock(path: str, env: Any, *, shared: bool = False) -> AsyncIterator[Any]: if env is not None and not getattr(env, "local_filesystem", False): environment_lock = getattr(env, "file_write_lock", None) if callable(environment_lock): @@ -41,15 +234,13 @@ async def host_write_lock(path: str, env: Any) -> AsyncIterator[Any]: yield None return - lock = FileLock(_host_lock_path(path, env), timeout=0) + lock_path = _host_lock_path(path, env) loop = asyncio.get_running_loop() deadline = loop.time() + HOST_WRITE_LOCK_TIMEOUT_SECONDS - acquired = False - try: + with contextlib.ExitStack() as held: while True: try: - lock.acquire(timeout=0) - acquired = True + lock = held.enter_context(_try_file_lock(lock_path, shared=shared)) break except Timeout: remaining = deadline - loop.time() @@ -57,9 +248,6 @@ async def host_write_lock(path: str, env: Any) -> AsyncIterator[Any]: raise TimeoutError(f"timed out acquiring write lock for {path}") await asyncio.sleep(min(HOST_WRITE_LOCK_POLL_SECONDS, remaining)) yield lock - finally: - if acquired: - lock.release() class Tool: """Base class for all tools. Subclass and implement ``execute_with_runtime``. diff --git a/opencollab/adapters/tools/bash.py b/opencollab/adapters/tools/bash.py index 44305288..63ef8799 100644 --- a/opencollab/adapters/tools/bash.py +++ b/opencollab/adapters/tools/bash.py @@ -4,6 +4,7 @@ from typing import Any +from opencollab.adapters._env_base import _track_shell_command from opencollab.adapters.tools._output import require_positive_int, truncate from opencollab.adapters.tools.base import Tool from opencollab.application.tool_execution import ToolRuntime @@ -116,7 +117,8 @@ async def execute_with_runtime( if safety_policy: await safety_policy.check_cmd_interactive(cmd, runtime.confirm_fn()) - result = await env.exec_cmd(cmd, timeout=timeout) + with _track_shell_command(env): + result = await env.exec_cmd(cmd, timeout=timeout) observations = getattr(runtime, "observations", None) if observations is not None: observations.record_execution( diff --git a/opencollab/adapters/tools/fs.py b/opencollab/adapters/tools/fs.py index ad1ec24b..b32754a8 100644 --- a/opencollab/adapters/tools/fs.py +++ b/opencollab/adapters/tools/fs.py @@ -11,7 +11,7 @@ from opencollab.adapters._safe_file_common import _normalize_newlines, _split_lines from opencollab.adapters.tools._output import require_positive_int, truncate from opencollab.adapters.tools._parameters import integer_parameter -from opencollab.adapters.tools._paths import checked_path +from opencollab.adapters.tools._paths import checked_path, checked_search_path from opencollab.adapters.tools.apply_patch_engine import ( _MIXED_NEWLINES, _detect_newline_style, @@ -90,7 +90,7 @@ async def execute_with_runtime( if env is None: return "Error: no execution environment available." - path = checked_path(runtime, path) + path = checked_path(runtime, path, native_file=True) try: range_reader = getattr(env, "read_text_range", None) @@ -250,7 +250,7 @@ async def execute_with_runtime( if mode == "create" and "content" not in params: return "Error: content is required for create mode." - path = checked_path(runtime, path) + path = checked_path(runtime, path, native_file=True) try: async with host_write_lock(path, env): @@ -427,15 +427,15 @@ async def execute_with_runtime( if env is None: return "Error: no execution environment available." - try: - search_path = checked_path(runtime, search_path) - except PermissionError as exc: - return f"Error: {exc}" try: max_results = int(raw_max_results) except (TypeError, ValueError): return "Error: max_results must be an integer." max_results = max(1, min(max_results, 500)) + try: + search_path = await checked_search_path(runtime, search_path) + except OSError as exc: + return f"Error: {exc}" quoted_pattern = shlex.quote(pattern) quoted_search_path = shlex.quote(search_path) diff --git a/opencollab/application/_anthropic_history.py b/opencollab/application/_anthropic_history.py index 52e2cf51..64330ac2 100644 --- a/opencollab/application/_anthropic_history.py +++ b/opencollab/application/_anthropic_history.py @@ -3,9 +3,13 @@ from __future__ import annotations import copy +from collections import Counter +from collections.abc import Iterator +from itertools import chain from typing import Any -from opencollab.domain.session import SessionState +from opencollab.application.shaping.pipeline import _group_spans, is_complete_tool_exchange +from opencollab.domain.session import SessionState, same_snapshot_value _THINKING_TYPES = frozenset({"thinking", "redacted_thinking"}) @@ -25,32 +29,181 @@ def _strip_anthropic_thinking(messages: list[dict[str, Any]], start: int = 0) -> def _edited_anthropic_history(before: list[dict], after: list[dict]) -> list[dict]: """Drop thinking bound to a changed prefix, keeping the valid older run.""" - if [message for message in before if message.get("role") == "system" and not message.get("compacted")] != [ - message for message in after if message.get("role") == "system" and not message.get("compacted") - ]: + before_system = [message for message in before if message.get("role") == "system" and not message.get("compacted")] + after_system = [message for message in after if message.get("role") == "system" and not message.get("compacted")] + if not same_snapshot_value(before_system, after_system): return _strip_anthropic_thinking(after) for index, (old, new) in enumerate(zip(before, after)): - if old != new: + if not same_snapshot_value(old, new): return _strip_anthropic_thinking(after, index) return after +def _anthropic_source_changed(state: SessionState) -> bool: + history = state.anthropic_history + source = history.get("source_messages") if history is not None else None + return source is not None and not same_snapshot_value(state.messages[:len(source)], source) + +def _anchor_key(group: list[dict]) -> tuple: + message = group[0] + calls = message.get("tool_calls") + if ( + isinstance(calls, list) and all(isinstance(call, dict) for call in calls) + and is_complete_tool_exchange(group, (0, len(group))) + ): + return ("exchange", tuple(str.__str__(call["id"]) for call in calls)) + role = message.get("role") + content = message.get("tool_call_id", message.get("content")) + if isinstance(content, list) and content and isinstance(content[0], dict): + content = content[0].get("text") + return ( + "message", str.__str__(role) if isinstance(role, str) else None, + str.__str__(content) if isinstance(content, str) else None, + ) + +def _ordered_anchors(candidates: list[tuple[int, int]]) -> Iterator[tuple[int, int]]: + # A moved group can cross several candidates. Keep the anchors on both sides + # of the move and let the caller rebuild the reordered region from the source. + highest = -1 + increasing = set() + for old, new in candidates: + if new > highest: + increasing.add(old) + highest = new + lowest = float("inf") + ordered = [] + for old, new in reversed(candidates): + if new < lowest and old in increasing: + ordered.append((old, new)) + lowest = min(lowest, new) + yield from reversed(ordered) + +def _align_groups( + before: list[list[dict]], after: list[list[dict]], *, projected: bool = False, +) -> Iterator[tuple[int, int]]: + old_keys = list(map(_anchor_key, before)) + new_keys = list(map(_anchor_key, after)) + old_counts, new_counts = Counter(old_keys), Counter(new_keys) + + def matches(old: int, new: int) -> bool: + key = old_keys[old] + if key != new_keys[new]: + return False + if same_snapshot_value(before[old], after[new]): + return True + if not projected: + return False + if key[0] == "exchange": + return old_counts[key] == new_counts[key] == 1 + return same_snapshot_value(_strip_anthropic_thinking(before[old]), after[new]) + + def equal_edges(old: int, new: int, old_stop: int, new_stop: int) -> tuple[int, int]: + size = min(old_stop - old, new_stop - new) + prefix = suffix = 0 + while prefix < size and matches(old + prefix, new + prefix): + prefix += 1 + while suffix < size - prefix and matches(old_stop - suffix - 1, new_stop - suffix - 1): + suffix += 1 + return prefix, suffix + + # Matching runs at the ends can anchor repeated text without a unique key. + prefix, suffix = equal_edges(0, 0, len(before), len(after)) + yield from ((index, index) for index in range(prefix)) + old_stop, new_stop = len(before) - suffix, len(after) - suffix + old_middle, new_middle = old_keys[prefix:old_stop], new_keys[prefix:new_stop] + old_counts_middle, new_counts_middle = Counter(old_middle), Counter(new_middle) + new_positions = { + key: index for index, key in enumerate(new_middle, prefix) if new_counts_middle[key] == 1 + } + candidates = [ + (old, new_positions[key]) for old, key in enumerate(old_middle, prefix) + if old_counts_middle[key] == 1 and key in new_positions and matches(old, new_positions[key]) + ] + + old = new = prefix + for next_old, next_new in chain(_ordered_anchors(candidates), ((old_stop, new_stop),)): + head, tail = equal_edges(old, new, next_old, next_new) + for offset in range(head): + yield old + offset, new + offset + if next_old - old == next_new - new: + for offset in range(head, next_old - old - tail): + if matches(old + offset, new + offset): + yield old + offset, new + offset + for offset in range(-tail, 0): + yield next_old + offset, next_new + offset + if next_old < old_stop: + yield next_old, next_new + old, new = next_old + 1, next_new + 1 + yield from ((old_stop + offset, new_stop + offset) for offset in range(suffix)) + +def _rebase_anthropic_history(history: dict, messages: list[dict]) -> list[dict]: + view = history["messages"] + source, projection, current = ( + [rows[start:stop] for start, stop in _group_spans(rows)] + for rows in (history["source_messages"], view, messages) + ) + unchanged = dict(_align_groups(source, current)) + first_edit = 0 + while unchanged.get(first_edit) == first_edit: + first_edit += 1 + end_edit = len(source) + last_match = unchanged.get(end_edit - 1) + while end_edit > first_edit and last_match is not None and unchanged.get(end_edit - 1) == last_match: + end_edit -= 1 + last_match -= 1 + source_counts = Counter(map(_anchor_key, source)) + projection_counts = Counter(map(_anchor_key, projection)) + + def projected_groups() -> Iterator[tuple[int, int, list[list[dict]]]]: + old = seen = 0 + anchors = _align_groups(source, projection, projected=True) + for next_old, next_seen in chain(anchors, ((len(source), len(projection)),)): + if old < next_old: + gap = projection[seen:next_seen] + # The intact ends can retain whole slices. An isolated rewritten + # exchange inside the edits needs a unique call-ID anchor. + if next_old <= first_edit or old >= end_edit or all( + key[0] != "exchange" or source_counts[key] == projection_counts[key] == 1 + for key in map(_anchor_key, gap) + ): + yield old, next_old, gap + if next_old < len(source): + yield next_old, next_old + 1, [projection[next_seen]] + old, seen = next_old + 1, next_seen + 1 + + rebased: list[dict] = [] + taken = 0 + for start, stop, replacement in projected_groups(): + start_new = unchanged.get(start) + # A summary covers its whole source interval, including possible insertions. + if start_new is None or any(unchanged.get(index) != start_new + index - start for index in range(start, stop)): + continue + rebased.extend(chain.from_iterable(current[taken:start_new])) + rebased.extend(chain.from_iterable(replacement)) + taken = start_new + stop - start + rebased.extend(chain.from_iterable(current[taken:])) + return _edited_anthropic_history(view, rebased) + def _anthropic_history_input(state: SessionState) -> list[dict]: - """Append fresh transcript messages to the last native provider view.""" history = state.anthropic_history if history is None: return state.messages + if _anthropic_source_changed(state): + return copy.deepcopy(_rebase_anthropic_history(history, state.messages)) return copy.deepcopy([*history["messages"], *state.messages[history["source_count"]:]]) -def _remember_anthropic_history(state: SessionState, messages: list[dict], tools: list[dict] | None) -> list[dict]: +def _remember_anthropic_history( + state: SessionState, messages: list[dict], tools: list[dict] | None, +) -> list[dict]: """Persist edits so removed blocks and compacted content stay removed.""" history = state.anthropic_history messages = _edited_anthropic_history(_anthropic_history_input(state), messages) if history is not None: messages = _edited_anthropic_history(history["messages"], messages) - if history["tools"] != tools: + if not same_snapshot_value(history["tools"], tools): messages = _strip_anthropic_thinking(messages) state.anthropic_history = copy.deepcopy({ "messages": messages, "source_count": len(state.messages), "tools": tools, + "source_messages": state.messages, }) return messages diff --git a/opencollab/application/_scheduler_run.py b/opencollab/application/_scheduler_run.py index 8d838fa7..dcf9f3de 100644 --- a/opencollab/application/_scheduler_run.py +++ b/opencollab/application/_scheduler_run.py @@ -191,13 +191,7 @@ async def _run_turn_exclusive(self, aid: int, user_message: str) -> str: break await self._wait_for_scheduler_progress(aid) continue - # The waiter joins the wait, never the liveness question: it - # never completes on its own, so counting it as pending work - # would keep a quiescent team's turn running until someone - # cancelled it. - if cancel_waiter is not None: - pending.add(cancel_waiter) - await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + await self._wait_for_active_scheduler_tasks(pending, cancel_waiter) finally: if cancel_waiter is not None: cancel_waiter.cancel() @@ -244,9 +238,7 @@ async def _wait_for_restored_turn(self, aid: int, turn_start: int) -> None: if not pending: await self._wait_for_scheduler_progress(aid) continue - if cancel_waiter is not None: - pending.add(cancel_waiter) - await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + await self._wait_for_active_scheduler_tasks(pending, cancel_waiter) except asyncio.CancelledError: # The public caller performs bounded team cleanup after propagating # cancellation. A suspended recovery driver has already returned, @@ -381,6 +373,19 @@ async def _settle_cancelled_suspended_turn(self, aid: int) -> None: ) self._write_manifest() + def _message_budget_cleanup_tasks(self) -> set[asyncio.Task]: + if not any( + inbox and aid not in self._cancelled_turn_inbox_holds + for aid, inbox in self._message_inbox.items() + ): + return set() + # Any member's deferred grant can hold the shared budget an inbox needs. + return { + task + for _aid, _grant, _baseline, tasks, finished in self._pending_provider_budget_leases.values() + for task in tasks - finished + } + def _active_scheduler_tasks(self) -> set[asyncio.Task]: """Return live scheduler-owned producers, excluding the current waiter.""" current = asyncio.current_task() @@ -401,20 +406,54 @@ def _active_scheduler_tasks(self) -> set[asyncio.Task]: *self._startup_tasks.values(), *self._message_delivery_tasks.values(), *message_cleanup, + *self._message_budget_cleanup_tasks(), ) if not task.done() and task is not current } + async def _wait_for_active_scheduler_tasks( + self, pending: set[asyncio.Task], cancel_waiter: asyncio.Task | None = None, + ) -> None: + budget_owners = self._message_budget_cleanup_tasks() + budget_owners.discard(asyncio.current_task()) + waiting = pending | budget_owners + if cancel_waiter is not None: + waiting.add(cancel_waiter) + done, _ = await asyncio.wait(waiting, return_when=asyncio.FIRST_COMPLETED) + if done & budget_owners: + await self._drain_ready_message_inboxes() + async def _wait_for_scheduler_progress(self, aid: int) -> None: """Block for a producer completion or report a state with no producer.""" - await self._drain_ready_message_inboxes() - scb = self.table.get(aid) - if scb is not None and scb.state.phase.is_terminal() and self._quiescent(): - return - producers = self._active_scheduler_tasks() - if not producers: - raise SchedulerStalledError(aid) - await asyncio.wait(producers, return_when=asyncio.FIRST_COMPLETED) + settled_budget_owners: set[asyncio.Task] = set() + while True: + budget_owners = self._message_budget_cleanup_tasks() + await self._drain_ready_message_inboxes() + scb = self.table.get(aid) + if scb is not None and scb.state.phase.is_terminal() and self._quiescent(): + return + producers = self._active_scheduler_tasks() + if producers: + await self._wait_for_active_scheduler_tasks(producers) + return + budget_owners.update(self._message_budget_cleanup_tasks()) + callbacks = budget_owners - settled_budget_owners + if not callbacks: + raise SchedulerStalledError(aid) + # A done task can still hold its grant until the registered usage + # and reservation callbacks run. Follow them with an owned callback. + callback_tail = asyncio.get_running_loop().create_future() + remaining = set(callbacks) + + def after_budget_callback(task: asyncio.Task) -> None: + remaining.discard(task) + if not remaining and not callback_tail.done(): + callback_tail.set_result(None) + + for task in callbacks: + task.add_done_callback(after_budget_callback) + await callback_tail + settled_budget_owners.update(callbacks) def _quiescent(self) -> bool: """True when no session is mid-flight: none is awaiting events, none has diff --git a/opencollab/application/_session_run_completion.py b/opencollab/application/_session_run_completion.py index 7f696175..ca2e7f2a 100644 --- a/opencollab/application/_session_run_completion.py +++ b/opencollab/application/_session_run_completion.py @@ -28,15 +28,23 @@ _TokenBudgetStop, ) from opencollab.application._session_run_trace import _SessionRunTraceMixin -from opencollab.application._session_run_usage import _complete_with_error_usage, _normalize_completion_usage +from opencollab.application._session_run_usage import ( + _complete_with_error_usage, + _CompletionUsageAccounting, + _normalize_completion_usage, +) from opencollab.application._tool_loop_execution import _apply_completed_prefix_progress from opencollab.application.async_timeout import CallerTimeoutError, abandon_on_timeout from opencollab.application.ports import ( + _NATIVE_COMPLETION_CAPABILITY, + _NATIVE_REQUEST_ADMISSION, CompletionResponse, ContextTokenEstimatorPort, RequestOutputRequirementPort, RequestTokenEstimatorPort, ThinkingPrefixPort, + _native_completion_controls, + _NativeCompletionControls, ) from opencollab.application.shaping import ShaperPipeline from opencollab.application.shaping.pipeline import context_token_estimator @@ -55,6 +63,49 @@ logger = logging.getLogger(__name__) +async def _with_native_completion_controls( + operation: Awaitable[CompletionResponse], + scope: _NativeCompletionControls | None, +) -> CompletionResponse: + token = _native_completion_controls.set(scope) + try: + return await operation + finally: + if scope is not None: + scope.callbacks.clear() + _native_completion_controls.reset(token) + +def _completion_keyword_parameters( + complete: Callable[..., Awaitable[Any]], +) -> tuple[set[str] | None, bool, set[str], Callable[..., Any]]: + keywords: set[str] = set() + outer_accepts_kwargs = False + outer_keywords: set[str] = set() + target = complete + + def has_fixed_keywords(target: Callable[..., Any]) -> bool: + nonlocal outer_accepts_kwargs + parameters = inspect.signature(target, follow_wrapped=False).parameters.values() + names = { + parameter.name for parameter in parameters + if parameter.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY) + } + keywords.update(names) + accepts_kwargs = any(parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters) + if target is complete: + outer_accepts_kwargs = accepts_kwargs + outer_keywords.update(names) + return not accepts_kwargs + + # Follow variadic decorators through to the first fixed signature. + try: + target = inspect.unwrap(complete, stop=has_fixed_keywords) + if not hasattr(target, "__wrapped__") and not has_fixed_keywords(target): + return None, outer_accepts_kwargs, outer_keywords, target + except (TypeError, ValueError): + pass + return keywords, outer_accepts_kwargs, outer_keywords, target + def _selector_names_tool_choice(selector: Any) -> bool: """Return whether a structured error selector targets ``tool_choice``.""" values = selector if isinstance(selector, (list, tuple)) else [selector] @@ -623,8 +674,6 @@ async def _complete_with_choice( max_output_tokens = self._request_output_limit( messages, tools, thinking=thinking, thinking_params=params, ) - if max_output_tokens != DEFAULT_MAX_TOKENS_PER_STEP or self._completion_accepts_output_limit(): - extra["max_output_tokens"] = max_output_tokens reasoning_effort = getattr(self.agent, "reasoning_effort", None) if reasoning_effort is not None: extra["reasoning_effort"] = reasoning_effort @@ -635,6 +684,7 @@ async def _complete_with_choice( messages=messages, tools=tools, temperature=self.agent.temperature, + max_output_tokens=max_output_tokens, **extra, ) except Exception as exc: @@ -651,8 +701,6 @@ async def _complete_with_choice( max_output_tokens = self._request_output_limit( messages, tools, thinking=thinking, thinking_params=params, ) - if max_output_tokens != DEFAULT_MAX_TOKENS_PER_STEP or self._completion_accepts_output_limit(): - extra["max_output_tokens"] = max_output_tokens payload = { "reason": "prefix_binding_mismatch", "removed_blocks": removed, "retry": 1, "error_type": rejected_by, "status_code": 400, @@ -671,6 +719,7 @@ async def _complete_with_choice( messages=messages, tools=tools, temperature=self.agent.temperature, + max_output_tokens=max_output_tokens, **extra, ) @@ -759,22 +808,80 @@ def remaining_budget_tokens(self) -> int | None: return None return min(finite) - int(self.state.used_tokens) - sum(reserve() for reserve in self.budget_reservations) - def _completion_accepts_output_limit(self) -> bool: - """Include the default limit when the injected completion accepts it.""" - try: - parameters = inspect.signature(self.llm.complete).parameters.values() - except (TypeError, ValueError): + def _add_completion_controls( + self, + complete: Callable[..., Awaitable[CompletionResponse]], + kwargs: dict[str, Any], + *, + max_output_tokens: int, + usage_accounting: _CompletionUsageAccounting, + revalidate: Callable[[], int], + ) -> tuple[asyncio.Event | None, _NativeCompletionControls | None]: + keywords, outer_accepts_kwargs, outer_keywords, wrapped = _completion_keyword_parameters(complete) + owner = getattr(complete, "__self__", None) + wraps_base_method = owner is not None and any( + vars(base).get("complete") is wrapped for base in type(owner).__mro__[1:] + ) + # A metadata-only signature leaves admission on the outer router's kwargs. + accepts_admission = outer_accepts_kwargs and not wraps_base_method + scoped: dict[str, Callable[..., Any]] = {} + + def accepts(name: str) -> bool: + return keywords is None or name in keywords + + def add_control( + name: str, + capability: str, + callback: Callable[..., Any], + *, + native_support: object = _NATIVE_COMPLETION_CAPABILITY, + allow_scope: bool = True, + ) -> bool: + supported = getattr(self.llm, capability, False) + if not supported: + return False + if name in outer_keywords or (name == "on_admitted" and (accepts(name) or accepts_admission)): + kwargs[name] = callback + return True + # A native marker also appears on subclasses whose wrappers forward + # only legacy parameters. Supply their controls at native dispatch. + inherited = inspect.getattr_static(owner, capability, None) + declared = inspect.getattr_static(self.llm, capability, inherited) + if declared is native_support: + if allow_scope and inherited is native_support: + scoped[name] = callback + return True + return False + if declared is None and native_support is _NATIVE_COMPLETION_CAPABILITY: + if keywords is None or name not in keywords: + return False + # Explicit middleware opt-ins can consume controls before forwarding. + if accepts(name) or (outer_accepts_kwargs and (supported is True or name == "on_admitted")): + kwargs[name] = callback + return True return False - return any( - parameter.kind == inspect.Parameter.VAR_KEYWORD - or ( - parameter.name == "max_output_tokens" - and parameter.kind in { - inspect.Parameter.POSITIONAL_OR_KEYWORD, - inspect.Parameter.KEYWORD_ONLY, - } - ) - for parameter in parameters + + if max_output_tokens != DEFAULT_MAX_TOKENS_PER_STEP or accepts("max_output_tokens"): + kwargs["max_output_tokens"] = max_output_tokens + add_control("on_attempt_usage", "supports_attempt_usage", usage_accounting.record_attempt) + add_control("on_usage", "supports_usage_settlement", usage_accounting.settle_provider) + add_control("before_request", "supports_request_revalidation", revalidate) + admitted = asyncio.Event() + has_admission = add_control( + "on_admitted", "supports_request_admission", admitted.set, + native_support=_NATIVE_REQUEST_ADMISSION, + allow_scope=self._per_call_timeout is not None and outer_accepts_kwargs, + ) + scope = _NativeCompletionControls(kwargs.get("on_admitted"), scoped) if scoped else None + return admitted if has_admission else None, scope + + def _revalidate_request( + self, messages: list[dict], tools: list[dict] | None, *, thinking: bool = False, + thinking_params: dict[str, Any] | None = None, + ) -> int: + self._raise_if_run_cancelled() + return self._request_output_limit( + messages, tools, thinking=thinking, thinking_params=thinking_params, ) async def _invoke_summary( @@ -792,13 +899,22 @@ async def _invoke_summary( ) if _summary_output_limit is not None: max_output_tokens = min(max_output_tokens, _summary_output_limit(messages)) - if max_output_tokens != DEFAULT_MAX_TOKENS_PER_STEP or self._completion_accepts_output_limit(): - kwargs["max_output_tokens"] = max_output_tokens start = time.monotonic() protected_call = self.state.wind_down_done - admitted = asyncio.Event() if getattr(self.llm, "supports_request_admission", False) else None - if admitted is not None: - kwargs["on_admitted"] = admitted.set + usage_accounting = _CompletionUsageAccounting(self, protected_call=protected_call) + + def revalidate() -> int: + limit = self._revalidate_request( + messages, None, thinking=thinking, thinking_params=thinking_params, + ) + return min(limit, _summary_output_limit(messages)) if _summary_output_limit is not None else limit + + admitted, scope = self._add_completion_controls( + self.llm.complete, kwargs, max_output_tokens=max_output_tokens, + usage_accounting=usage_accounting, revalidate=revalidate, + ) + if scope is not None: + messages = scope.prepare_messages(messages) abandoned = False if self.tracer: self.tracer.log_step(step_type="llm_call_started", payload={ @@ -808,16 +924,13 @@ async def _invoke_summary( }) def account_response(response: CompletionResponse) -> None: - input_tokens, total_tokens = _normalize_completion_usage(response.usage) - self.state.add_used_tokens(total_tokens) - self._record_usage_event(response.usage, total_tokens, purpose="summary", late=abandoned) - self._mark_budget_reserve_consumed(protected_call=protected_call) + input_tokens, _total_tokens = _normalize_completion_usage(response.usage) + usage_accounting.finish( + response.usage, purpose="summary", late=abandoned, + transport_timing=getattr(response, "transport_timing", None), + ) self.state.add_markup_recovered(getattr(response.usage, "markup_recovered", 0)) self.state.set_context_tokens(input_tokens) - if abandoned: - self._late_provider_usage += (total_tokens,) - if self.late_provider_usage_checkpoint is not None: - self.late_provider_usage_checkpoint() self.record_llm_trace(response, time.monotonic() - start, purpose="summary", thinking=thinking) async def complete_owned() -> CompletionResponse: @@ -825,14 +938,17 @@ async def complete_owned() -> CompletionResponse: if self._run_control is not None: self._raise_if_run_cancelled() return await _complete_with_error_usage( - self, complete, messages, protected_call=protected_call, + self, complete, messages, usage_accounting=usage_accounting, usage_purpose="summary", on_response=account_response, **kwargs, ) finally: self._draining_provider_tasks.discard(asyncio.current_task()) self._finish_run_control_if_quiesced() - owner = asyncio.create_task(complete_owned()) + operation = complete_owned() + if scope is not None or _native_completion_controls.get() is not None: + operation = _with_native_completion_controls(operation, scope) + owner = asyncio.create_task(operation) self._track_provider_task(owner) try: if self._per_call_timeout is None: @@ -857,7 +973,7 @@ async def complete_owned() -> CompletionResponse: self._mark_provider_task_draining(owner) raise - async def _invoke_llm(self, **kwargs: Any) -> CompletionResponse: + async def _invoke_llm(self, *, max_output_tokens: int, **kwargs: Any) -> CompletionResponse: """Bound one provider generation by ``_per_call_timeout``. ``None`` disables the ceiling. Expiry raises ``GenerationTimeoutError`` @@ -867,20 +983,52 @@ async def _invoke_llm(self, **kwargs: Any) -> CompletionResponse: if self._run_control is not None: self._raise_if_run_cancelled() protected_call = self.state.wind_down_done - admitted = asyncio.Event() if getattr(self.llm, "supports_request_admission", False) else None - if admitted is not None: - kwargs["on_admitted"] = admitted.set - operation = _complete_with_error_usage(self, self.llm.complete, protected_call=protected_call, **kwargs) + usage_accounting = _CompletionUsageAccounting(self, protected_call=protected_call) + complete = self.llm.complete + admitted, scope = self._add_completion_controls( + complete, kwargs, max_output_tokens=max_output_tokens, + usage_accounting=usage_accounting, + revalidate=lambda: self._revalidate_request( + kwargs["messages"], kwargs.get("tools"), thinking=kwargs.get("thinking", False), + thinking_params=kwargs.get("thinking_params"), + ), + ) + if scope is not None: + kwargs["messages"] = scope.prepare_messages(kwargs["messages"]) + late_accounted = False + + async def complete_owned() -> CompletionResponse: + nonlocal late_accounted + try: + response = await _complete_with_error_usage( + self, complete, usage_accounting=usage_accounting, **kwargs, + ) + late_accounted = asyncio.current_task() in self._draining_provider_tasks + usage_accounting.finish( + response.usage, purpose="completion", late=late_accounted, + transport_timing=getattr(response, "transport_timing", None), + ) + return response + finally: + self._draining_provider_tasks.discard(asyncio.current_task()) + self._finish_run_control_if_quiesced() + + def record_abandoned(task: asyncio.Task[Any]) -> None: + if late_accounted: + self._provider_task_done(task) + else: + self._record_late_provider_result(task, accounting=usage_accounting) + + operation = complete_owned() + if scope is not None or _native_completion_controls.get() is not None: + operation = _with_native_completion_controls(operation, scope) try: response = await operation if self._per_call_timeout is None else await abandon_on_timeout( operation, self._per_call_timeout, task_tracker=self._track_provider_task, late_task_tracker=self._mark_provider_task_draining, - late_result_handler=lambda task: self._record_late_provider_result( - task, - protected_call=protected_call, - ), + late_result_handler=record_abandoned, start_event=admitted, ) except CallerTimeoutError as exc: diff --git a/opencollab/application/_session_run_usage.py b/opencollab/application/_session_run_usage.py index 49477bec..29edaa21 100644 --- a/opencollab/application/_session_run_usage.py +++ b/opencollab/application/_session_run_usage.py @@ -4,8 +4,32 @@ import asyncio import time +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass from typing import Any, Awaitable, Callable +_native_usage_owner: ContextVar[object | None] = ContextVar("native_usage_owner", default=None) +_native_usage_observer: ContextVar[Callable[[Any, BaseException | None, object | None], None] | None] = ContextVar( + "native_usage_observer", default=None, +) + + +@contextmanager +def _own_native_usage() -> Iterator[None]: + token = _native_usage_owner.set(object()) + try: + yield + finally: + _native_usage_owner.reset(token) + + +def _observe_native_usage(usage: Any, error: BaseException | None, transport_timing: object | None) -> None: + observer = _native_usage_observer.get() + if observer is not None: + observer(usage, error, transport_timing) + def _nonnegative_usage_int(value: Any, field: str) -> int: if isinstance(value, bool) or not isinstance(value, int) or value < 0: @@ -33,33 +57,172 @@ def _normalize_completion_usage(usage: Any) -> tuple[int, int]: context_tokens = _nonnegative_usage_int(context_tokens, "context_tokens") return input_tokens if context_tokens is None else context_tokens, max(reported_total, input_tokens + output_tokens) + +def _same_token_counts(native: Any, usage: Any) -> bool: + return type(native) is type(usage) and all( + getattr(native, name) == getattr(usage, name) + for name in ("input_tokens", "output_tokens", "total_tokens") + ) + + +def _same_usage_measurements(native: Any, usage: Any) -> bool: + return _same_token_counts(native, usage) and all( + getattr(native, name) == getattr(usage, name) + for name in ("cache_read_tokens", "cache_creation_tokens", "reasoning_tokens") + ) + + +@dataclass +class _ProviderUsageObservation: + usage: Any + error: BaseException | None + transport_timing: object | None + reported_usage: Any + + +class _CompletionUsageAccounting: + def __init__(self, runner: Any, *, protected_call: bool) -> None: + self.runner = runner + self.protected_call = protected_call + self.tokens = 0 + self._attempt_tokens: dict[object | None, int] = {} + self._callback_usages: dict[object, Any] = {} + self._native_usages: list[_ProviderUsageObservation] = [] + self._settled: list[tuple[Any, int]] = [] + self._reported = False + + @contextmanager + def observe_provider_responses(self) -> Iterator[None]: + token = _native_usage_observer.set(self._observe_provider_response) + try: + yield + finally: + _native_usage_observer.reset(token) + + def record_attempt(self, usage: Any) -> None: + owner = _native_usage_owner.get() + tokens = _normalize_completion_usage(usage)[1] + self._charge(tokens) + self._attempt_tokens[owner] = self._attempt_tokens.get(owner, 0) + tokens + + def settle_provider(self, usage: Any) -> None: + owner = _native_usage_owner.get() + native = self._find_native_usage(usage) if owner is None else None + if native is not None and native.usage is not usage and not _same_token_counts(native.usage, usage): + native = None + if native is not None and self._already_settled(native.reported_usage): + return + self._settle(usage, self._attempt_tokens.pop(owner, 0)) + if native is not None: + native.reported_usage = usage + elif owner is not None: + self._callback_usages[owner] = usage + + def _observe_provider_response( + self, usage: Any, error: BaseException | None, transport_timing: object | None, + ) -> None: + owner = _native_usage_owner.get() + # A callback may settle a redacted copy of this request's usage. + settled = self._callback_usages.pop(owner, usage) + if settled is not usage and not _same_token_counts(usage, settled): + settled = usage + self._native_usages.append(_ProviderUsageObservation(usage, error, transport_timing, settled)) + # A router can forward the retry hook and keep the final callback for its chosen response. + if owner is not None and owner in self._attempt_tokens: + self._settle(usage, self._attempt_tokens.pop(owner)) + + def _settle(self, usage: Any, attempt_tokens: int) -> None: + tokens = _normalize_completion_usage(usage)[1] + self._charge(tokens - attempt_tokens) + self._settled.append((usage, tokens)) + + def _already_settled(self, usage: Any) -> bool: + return any(usage is settled for settled, _tokens in self._settled) + + def _find_native_usage(self, usage: Any) -> _ProviderUsageObservation | None: + native = next((item for item in self._native_usages if item.usage is usage), None) + origin = getattr(usage, "_native_usage_origin", None) + if native is None and origin is not None: + native = next( + (item for item in self._native_usages if getattr(item.usage, "_native_usage_origin", None) is origin), + None, + ) + if native is None and origin is not None: + raw_usage = getattr(usage, "raw_usage", None) + if raw_usage is not None: + native = next( + (item for item in self._native_usages + if getattr(item.usage, "raw_usage", None) is raw_usage and _same_token_counts(item.usage, usage)), + None, + ) + return native + + def finish( + self, usage: Any, *, purpose: str, late: bool, error: BaseException | None = None, + transport_timing: object | None = None, + ) -> bool: + if self._reported: + return False + # A router can retain timing from a different call, so prefer the usage payload. + native = self._find_native_usage(usage) + if native is None and transport_timing is not None: + native = next( + (item for item in self._native_usages + if item.transport_timing is transport_timing and _same_token_counts(item.usage, usage)), + None, + ) + if usage is not None and native is None: + native = next((item for item in reversed(self._native_usages) if item.usage == usage), None) + if native is None and getattr(usage, "_native_usage_origin", None) is not None: + native = next( + (item for item in reversed(self._native_usages) if _same_usage_measurements(item.usage, usage)), + None, + ) + reported_usage = usage if native is None else native.reported_usage + if usage is not None and not self._already_settled(reported_usage): + attempt_tokens = self._attempt_tokens.pop(None, 0) if native is None else 0 + self._settle(reported_usage, attempt_tokens) + self._reported = True + record_usage = getattr(self.runner, "_record_usage_event", None) + if callable(record_usage): + for settled, tokens in self._settled: + provider_error = next( + (item.error for item in self._native_usages if item.reported_usage is settled), None, + ) + record_usage(settled, tokens, purpose=purpose, late=late, + error=error if settled is reported_usage and error is not None else provider_error) + if late: + self.record_late() + return bool(self._settled) + + def record_late(self) -> None: + if self._settled: + self.runner._late_provider_usage += (self.tokens,) + if self.runner.late_provider_usage_checkpoint is not None: + self.runner.late_provider_usage_checkpoint() + + def _charge(self, tokens: int) -> None: + self.runner._charge_completion_tokens(tokens) + self.tokens += tokens + self.runner._mark_budget_reserve_consumed(protected_call=self.protected_call) + + async def _complete_with_error_usage( runner: Any, complete: Callable[..., Awaitable[Any]], *args: Any, - protected_call: bool, + usage_accounting: _CompletionUsageAccounting, usage_purpose: str = "completion", **kwargs: Any, ) -> Any: """Charge reported error usage inside the provider owner exactly once.""" start = time.monotonic() try: - return await complete(*args, **kwargs) + with usage_accounting.observe_provider_responses(): + return await complete(*args, **kwargs) except BaseException as exc: - usage = getattr(exc, "usage", None) - if usage is not None: - _input_tokens, total_tokens = _normalize_completion_usage(usage) - runner.state.add_used_tokens(total_tokens) - late = asyncio.current_task() in runner._draining_provider_tasks - record_usage = getattr(runner, "_record_usage_event", None) - if callable(record_usage): - record_usage(usage, total_tokens, purpose=usage_purpose, - late=late, error=exc) - runner._mark_budget_reserve_consumed(protected_call=protected_call) - if late: - runner._late_provider_usage += (total_tokens,) - if runner.late_provider_usage_checkpoint is not None: - runner.late_provider_usage_checkpoint() + late = asyncio.current_task() in runner._draining_provider_tasks + if usage_accounting.finish(getattr(exc, "usage", None), purpose=usage_purpose, late=late, error=exc): try: runner.record_llm_trace( None, time.monotonic() - start, purpose=usage_purpose, diff --git a/opencollab/application/ports.py b/opencollab/application/ports.py index 13e8234c..6c3b43db 100644 --- a/opencollab/application/ports.py +++ b/opencollab/application/ports.py @@ -1,6 +1,9 @@ from __future__ import annotations from collections.abc import Mapping, Sequence +from contextvars import ContextVar +from copy import deepcopy +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Awaitable, Callable, Protocol, runtime_checkable from opencollab.domain.hooks import HookOutcome @@ -470,6 +473,61 @@ def transport_timing(self) -> dict[str, Any] | None: """ ... +class _NativeCompletionCapability: + def __get__(self, instance: object, owner: type) -> bool: + return True + + +class _NativeRequestAdmission: + def __get__(self, instance: Any, owner: type) -> bool: + return instance is not None and instance.request_pool is not None + + +# Native clients support these callbacks through their completion signature. +_NATIVE_COMPLETION_CAPABILITY = _NativeCompletionCapability() +_NATIVE_REQUEST_ADMISSION = _NativeRequestAdmission() + + +@dataclass +class _NativeCompletionControls: + admission_callback: Callable[[], None] | None + callbacks: dict[str, Callable[..., Any]] + legacy_request_key: object = field(default_factory=object, init=False) + + def prepare_messages(self, messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + if self.admission_callback is not None: + return messages + return [_LegacyCompletionMessage(message, self.legacy_request_key) for message in messages] + + def matches(self, messages: list[dict[str, Any]], on_admitted: Callable[[], None] | None) -> bool: + if self.admission_callback is not None: + return on_admitted is self.admission_callback + return any( + isinstance(message, _LegacyCompletionMessage) and message.request_key is self.legacy_request_key + for message in messages + ) + + +class _LegacyCompletionMessage(dict[str, Any]): + __slots__ = ("request_key",) + + def __init__(self, message: dict[str, Any], request_key: object) -> None: + super().__init__(message) + self.request_key = request_key + + def __deepcopy__(self, memo: dict[int, Any]) -> _LegacyCompletionMessage: + # Keep request identity when legacy routes copy messages before editing them. + result = _LegacyCompletionMessage({}, self.request_key) + memo[id(self)] = result + result.update(deepcopy(dict(self), memo)) + return result + + +_native_completion_controls: ContextVar[_NativeCompletionControls | None] = ContextVar( + "native_completion_controls", default=None, +) + + class LLMPort(Protocol): """LLM client surface used by the session run loop and compaction.""" @@ -605,6 +663,28 @@ def checkpoint_snapshot( ) -> None: ... +class SnapshotCapacityError(ValueError): + """An encoded snapshot exceeds the store's write limit.""" + + +class SnapshotSequenceConflict(ValueError): + """The requested cursor conflicts with a durable snapshot or delta.""" + + +@runtime_checkable +class SnapshotReplacementStorePort(Protocol): + def append_snapshot_replacement( + self, + path: str, + messages: list[dict[str, Any]], + *, + meta: dict[str, Any], + sequence: int, + reject_conflicts: bool = False, + ) -> bool: + """Return whether the replacement was appended at a new durable cursor.""" + ... + class TracePort(Protocol): """Trajectory recorder surface.""" diff --git a/opencollab/application/session.py b/opencollab/application/session.py index 74e2a51c..db1aff4f 100644 --- a/opencollab/application/session.py +++ b/opencollab/application/session.py @@ -20,6 +20,9 @@ PermissionPort, SafetyPolicyPort, SessionStorePort, + SnapshotCapacityError, + SnapshotReplacementStorePort, + SnapshotSequenceConflict, SnapshotStorePort, TracePort, ) @@ -30,6 +33,7 @@ _restore_pending_row, _restore_phase, _restore_queued_external_user_turn, + _serialize_anthropic_history, _serialize_pending_row, _snapshot_int, _snapshot_nonnegative_float, @@ -39,7 +43,7 @@ from opencollab.domain.agent import Agent from opencollab.domain.events import SessionRuntimeEvent as SessionEvent from opencollab.domain.pending import PendingRow, RowKind, RowStatus -from opencollab.domain.session import SessionPhase, SessionState +from opencollab.domain.session import SessionPhase, SessionState, same_snapshot_value from opencollab.domain.tool_loops import LoopState if TYPE_CHECKING: @@ -147,7 +151,9 @@ def __init__( self._terminal_checkpoint_error: Exception | None = None self._auto_save_sequence = 0 self._auto_save_reserved_sequence = 0 + self._auto_save_conflict: str | None = None self._auto_save_message_count = 0 + self._auto_save_messages: list[dict] | None = None self._auto_save_rewrite_from: int | None = 0 self._auto_save_rewrite_revision = 0 self._auto_save_lock = threading.Lock() @@ -317,9 +323,7 @@ def messages(self) -> list[dict]: def messages(self, value: list[dict]) -> None: self.state.replace_messages(value) if hasattr(self, "_auto_save_rewrite_from"): - with self._auto_save_lock: - self._auto_save_rewrite_from = 0 - self._auto_save_rewrite_revision += 1 + self._mark_auto_save_rewrite(0) @property def used_tokens(self) -> int: @@ -437,6 +441,10 @@ async def _checkpoint_terminal_snapshot(self) -> None: # The pending rows and their elapsed time are now complete. Persist # this suspension without emitting the step_end reserved for resume. return + with self._auto_save_lock: + if self._auto_save_conflict is not None: + # The autosave worker already reported the ownership conflict. + return # Journal-backed autosave normally appends a delta. A terminal turn # must also publish a self-contained base snapshot: callers and crash # recovery may read the base file directly, without replaying its @@ -533,7 +541,7 @@ def restore(self, path: str) -> None: messages=[], aid=_snapshot_int(snapshot.get("aid"), default=self.state.aid), ) - restored.replace_messages(messages) + restored.replace_messages(messages, restoring=True) pending_messages = snapshot.get("pending_messages", []) restored.pending_user_messages = ( [ @@ -749,11 +757,18 @@ def _restore_auto_save_tracking(self, snapshot: dict[str, Any]) -> None: sequence = _snapshot_nonnegative_int(snapshot.get("_autosave_sequence")) self._auto_save_sequence = sequence self._auto_save_reserved_sequence = sequence + self._auto_save_conflict = None stored_messages = snapshot.get("messages") self._auto_save_message_count = min( len(stored_messages) if isinstance(stored_messages, list) else 0, len(self.state.messages), ) + self._auto_save_messages = ( + copy.deepcopy(stored_messages) + if isinstance(stored_messages, list) and self._auto_save_path + and isinstance(self.store, JournalSnapshotStorePort) and self.state.anthropic_history is not None + else None + ) self._auto_save_rewrite_from = None self._auto_save_seen_result_hashes = set( self.state.turn.seen_result_hashes @@ -773,7 +788,25 @@ def _checkpoint_restored_auto_save_target( sequence = _snapshot_nonnegative_int(snapshot.get("_autosave_sequence")) if isinstance(self.store, JournalSnapshotStorePort): sequence += 1 - self.store.checkpoint_snapshot(target_path, messages, meta=meta, sequence=sequence) + try: + self.store.checkpoint_snapshot(target_path, messages, meta=meta, sequence=sequence) + except SnapshotSequenceConflict: + saved_message_count = len(snapshot.get("messages", ())) + if saved_message_count >= len(messages) or not isinstance(self.store, SnapshotStorePort): + raise + previous = self.store.load_snapshot(target_path, self.agent.system_prompt) + previous_messages = previous["messages"] + if previous.get("_autosave_sequence") != sequence or len(previous_messages) != len(messages): + raise + # A retry regenerates recovery rows with a new wall-clock time. + # The checkpoint still compares their content and every source row. + timestamps = [message.get("timestamp") for message in previous_messages[saved_message_count:]] + if not all(isinstance(timestamp, str) for timestamp in timestamps): + raise + for message, timestamp in zip(messages[saved_message_count:], timestamps): + message["timestamp"] = timestamp + self.store.checkpoint_snapshot(target_path, messages, meta=meta, sequence=sequence) + restored.message_timestamps[saved_message_count:] = timestamps else: meta["_autosave_sequence"] = sequence self.store.save(target_path, messages, meta=meta) @@ -936,6 +969,8 @@ def save(self, path: str) -> None: # skip its sequence and its completion cannot rewind our cursor. with self._auto_save_lock: messages, meta = self._snapshot_for_save() + track_messages = self._auto_save_messages is not None or self.state.anthropic_history is not None + tracked_messages = messages if track_messages else None sequence = self._reserve_auto_save_sequence() rewrite_revision = self._auto_save_rewrite_revision self._auto_save_pending_checkpoints.add(sequence) @@ -948,6 +983,8 @@ def save(self, path: str) -> None: with self._auto_save_lock: self._auto_save_sequence = sequence self._auto_save_message_count = len(messages) + self._auto_save_messages = tracked_messages + self._auto_save_conflict = None if self._auto_save_rewrite_revision == rewrite_revision: self._auto_save_rewrite_from = None self._auto_save_seen_result_hashes = set(meta["session_state"]["seen_result_hashes"]) @@ -1004,7 +1041,7 @@ def _snapshot_for_save( "completed_turn_start_message_index": state.completed_turn_start_message_index, "pending_step_latency": state.pending_step_latency, "submitted_summary": state.submitted_summary, - "anthropic_history": state.anthropic_history, + "anthropic_history": _serialize_anthropic_history(state), }, } if self.run_id is not None: @@ -1037,6 +1074,12 @@ def _prepare_auto_save(self) -> Callable[[], None] | None: with self._auto_save_lock: return self._prepare_journal_auto_save(path) + def _mark_auto_save_rewrite(self, start: int) -> None: + with self._auto_save_lock: + previous = self._auto_save_rewrite_from + self._auto_save_rewrite_from = start if previous is None else min(previous, start) + self._auto_save_rewrite_revision += 1 + def _reserve_auto_save_sequence(self) -> int: self._auto_save_reserved_sequence = max( self._auto_save_reserved_sequence, self._auto_save_sequence, @@ -1044,6 +1087,8 @@ def _reserve_auto_save_sequence(self) -> int: return self._auto_save_reserved_sequence def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: + if self._auto_save_conflict is not None: + raise SnapshotSequenceConflict(self._auto_save_conflict) preceding_write_pending = self._auto_save_reserved_sequence > self._auto_save_sequence sequence = self._reserve_auto_save_sequence() message_count = len(self.state.messages) @@ -1058,7 +1103,28 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: rewrite_revision = self._auto_save_rewrite_revision if self._auto_save_rewrite_from is not None: replace_from = min(replace_from, self._auto_save_rewrite_from) + has_native_history = self.state.anthropic_history is not None + track_messages = self._auto_save_messages is not None or has_native_history + stored_messages = self._auto_save_messages or [] + if track_messages: + # Native history references resolve against the last committed transcript. + replace_from = min(replace_from, len(stored_messages)) + if replace_from: + current_messages = self.state.enriched_messages() + for index in range(replace_from): + if not same_snapshot_value(current_messages[index], stored_messages[index]): + replace_from = index + break + # A full native rewrite can exceed the journal's remaining capacity. + replacement_store = None + if ( + has_native_history and replace_from == 0 + and (preceding_write_pending or self._auto_save_message_count > 0) + and isinstance(self.store, SnapshotReplacementStorePort) + ): + replacement_store = self.store messages, meta = self._snapshot_for_save(replace_from) + tracked_messages = [*stored_messages[:replace_from], *messages] if track_messages else None current_seen_hashes = set(self.state.turn.seen_result_hashes) seen_hashes_reset = preceding_write_pending or not self._auto_save_seen_result_hashes.issubset( current_seen_hashes @@ -1068,47 +1134,77 @@ def _prepare_journal_auto_save(self, path: str) -> Callable[[], None]: if seen_hashes_reset else current_seen_hashes - self._auto_save_seen_result_hashes ) - checkpoint_due = sequence >= self._next_auto_save_checkpoint - checkpoint_messages = ( - copy.deepcopy(self.state.enriched_messages()) - if checkpoint_due - else None - ) + checkpoint_messages = None + if replacement_store is not None or sequence >= self._next_auto_save_checkpoint: + checkpoint_messages = tracked_messages + if checkpoint_messages is None: + checkpoint_messages = copy.deepcopy(self.state.enriched_messages()) + + def advance_cursor() -> None: + self._auto_save_sequence = sequence + self._auto_save_message_count = message_count + self._auto_save_messages = tracked_messages + if self._auto_save_rewrite_revision == rewrite_revision: + self._auto_save_rewrite_from = None + self._auto_save_seen_result_hashes = current_seen_hashes + + def write_checkpoint(full_messages: list[dict]) -> bool: + try: + self.store.checkpoint_snapshot(path, full_messages, meta=meta, sequence=sequence) + except (OSError, SnapshotCapacityError) as checkpoint_error: + if replacement_store is None: + raise + try: + saved = replacement_store.append_snapshot_replacement( + path, + full_messages, + meta=meta, + sequence=sequence, + reject_conflicts=True, + ) + except SnapshotSequenceConflict: + raise + except Exception as journal_error: + raise checkpoint_error from journal_error + if not saved: + raise + return False + return True def persist() -> None: - self.store.append_snapshot_delta( - path, - sequence=sequence, - replace_from=replace_from, - messages=messages, - meta=meta, - seen_result_hashes_reset=seen_hashes_reset, - seen_result_hashes_added=seen_hashes_added, - ) - with self._auto_save_lock: - if sequence <= self._auto_save_sequence: - return - # The journal append is already fsync'd. Advance the absolute - # cursor before optional compaction so a failed base rewrite - # cannot cause the durable delta to be skipped on the next save. - self._auto_save_sequence = sequence - self._auto_save_message_count = message_count - if self._auto_save_rewrite_revision == rewrite_revision: - self._auto_save_rewrite_from = None - self._auto_save_seen_result_hashes = current_seen_hashes + if replacement_store is None: + self.store.append_snapshot_delta( + path, + sequence=sequence, + replace_from=replace_from, + messages=messages, + meta=meta, + seen_result_hashes_reset=seen_hashes_reset, + seen_result_hashes_added=seen_hashes_added, + ) + with self._auto_save_lock: + if sequence <= self._auto_save_sequence: + return + # The fsync'd append survives a failed base rewrite. + advance_cursor() if checkpoint_messages is not None: with self._auto_save_checkpoint_lock: with self._auto_save_lock: if sequence < self._auto_save_sequence: return - self.store.checkpoint_snapshot( - path, - checkpoint_messages, - meta=meta, - sequence=sequence, - ) + try: + compacted = write_checkpoint(checkpoint_messages) + except SnapshotSequenceConflict as conflict: + if replacement_store is not None: + # A later cursor from this stale session still refers to the same old base. + with self._auto_save_lock: + self._auto_save_conflict = str(conflict) + raise with self._auto_save_lock: - self._next_auto_save_checkpoint = 1 << sequence.bit_length() + if replacement_store is not None: + advance_cursor() + if compacted: + self._next_auto_save_checkpoint = 1 << sequence.bit_length() def persist_incrementally() -> None: with self._auto_save_lock: diff --git a/opencollab/application/session_run.py b/opencollab/application/session_run.py index f9184545..b9d54014 100644 --- a/opencollab/application/session_run.py +++ b/opencollab/application/session_run.py @@ -30,7 +30,7 @@ _TeamBudgetStop, _TokenBudgetStop, ) -from opencollab.application._session_run_usage import _normalize_completion_usage +from opencollab.application._session_run_usage import _CompletionUsageAccounting, _normalize_completion_usage from opencollab.application.context_pressure import ContextPressure from opencollab.application.events import SessionEventFactory, default_session_event_factory from opencollab.application.ports import ( @@ -185,6 +185,7 @@ def __init__( self._late_provider_usage: tuple[int, ...] = () # The session supplies its owned persistence queue after adopting this runner. self.late_provider_usage_checkpoint: Callable[[], object] | None = None + self._on_completion_tokens: Callable[[int], None] | None = None def reset_runtime_for_user_turn(self) -> None: """Restore agent capabilities narrowed by the previous turn.""" @@ -245,22 +246,22 @@ def _mark_budget_reserve_consumed(self, *, protected_call: bool | None = None) - ): self.state.budget_reserve_consumed = True + def _charge_completion_tokens(self, tokens: int) -> None: + self.state.add_used_tokens(tokens) + if tokens and self._on_completion_tokens is not None: + self._on_completion_tokens(tokens) + def _record_late_provider_result( - self, - task: asyncio.Future[Any], - *, - protected_call: bool = False, + self, task: asyncio.Future[Any], *, protected_call: bool = False, + accounting: _CompletionUsageAccounting | None = None, ) -> None: - """Charge a provider response that survived caller cancellation or timeout.""" try: response = task.result() - _input_tokens, total_tokens = _normalize_completion_usage(response.usage) - self._late_provider_usage += (total_tokens,) - self.state.add_used_tokens(total_tokens) - self._record_usage_event(response.usage, total_tokens, purpose="completion", late=True) - self._mark_budget_reserve_consumed(protected_call=protected_call) - if self.late_provider_usage_checkpoint is not None: - self.late_provider_usage_checkpoint() + if accounting is None: + accounting = _CompletionUsageAccounting(self, protected_call=protected_call) + accounting.finish(response.usage, purpose="completion", late=True) + else: + accounting.record_late() except BaseException: pass finally: @@ -699,11 +700,8 @@ async def run_llm_call(self) -> None: await self._stop_precheck("team budget exceeded: aggregate spend reached the global cap") return latency = time.monotonic() - start - input_tokens, total_tokens = _normalize_completion_usage(response.usage) - self.state.add_used_tokens(total_tokens) - self._record_usage_event(response.usage, total_tokens, purpose="completion") + input_tokens, _total_tokens = _normalize_completion_usage(response.usage) self._run_final_prompt = None - self._mark_budget_reserve_consumed() self.state.add_markup_recovered(getattr(response.usage, "markup_recovered", 0)) self.state.set_context_tokens(input_tokens) diff --git a/opencollab/application/session_snapshot.py b/opencollab/application/session_snapshot.py index 5b12ed94..81757610 100644 --- a/opencollab/application/session_snapshot.py +++ b/opencollab/application/session_snapshot.py @@ -6,8 +6,9 @@ import math from typing import Any +from opencollab.application._anthropic_history import _anthropic_source_changed, _rebase_anthropic_history from opencollab.domain.pending import PendingRow, RowKind, RowStatus -from opencollab.domain.session import SessionPhase +from opencollab.domain.session import SessionPhase, SessionState, same_snapshot_value # Translate legacy terminals and the removed enqueue phase so old snapshots # retain their disposition instead of falling back to IDLE. @@ -15,19 +16,69 @@ {"cancelled", "budget_exceeded", "step_limit_exceeded", "context_overflow"} ) -def _restore_anthropic_history(value: object, messages: list[dict]) -> dict | None: +def _serialize_anthropic_history(state: SessionState) -> dict | None: + history = state.anthropic_history + if history is None: + return None + source = state.messages + count = history["source_count"] + view = history["messages"] + if _anthropic_source_changed(state): + # Retire the stale projection before restore binds it to the new transcript. + view = _rebase_anthropic_history(history, source) + count = len(source) + + # Aligned and suffix-aligned entries can reuse the persisted transcript. + stored: list[dict | int] = [] + suffix_offset = count - len(view) + for index, message in enumerate(view): + for source_index in (index, index + suffix_offset): + if 0 <= source_index < count and same_snapshot_value(message, source[source_index]): + stored.append(source_index) + break + else: + stored.append(message) + return {"messages": stored, "source_count": count, "tools": history["tools"]} + +def decode_anthropic_history(value: object, messages: list[dict]) -> dict | None: + """Resolve native history references without copying nested messages.""" if not isinstance(value, dict): return None count = value.get("source_count") view = value.get("messages") tools = value.get("tools") + source = value.get("source_messages") if ( - isinstance(count, bool) or not isinstance(count, int) or not 0 <= count <= len(messages) - or not isinstance(view, list) or not all(isinstance(message, dict) for message in view) + isinstance(count, bool) or not isinstance(count, int) or count < 0 + or not isinstance(view, list) or (tools is not None and (not isinstance(tools, list) or not all(isinstance(tool, dict) for tool in tools))) ): return None - return copy.deepcopy({"messages": view, "source_count": count, "tools": tools}) + if source is None: + if count > len(messages): + return None + source = messages[:count] + # References use runtime messages. Snapshot timestamps are separate metadata. + for index, message in enumerate(source): + if "timestamp" in message: + source[index] = {key: item for key, item in message.items() if key != "timestamp"} + elif ( + not isinstance(source, list) or len(source) != count + or not all(isinstance(message, dict) for message in source) + ): + return None + restored_view = [] + for message in view: + if isinstance(message, dict): + restored_view.append(message) + elif type(message) is int and 0 <= message < len(source): + restored_view.append(source[message]) + else: + return None + return {"messages": restored_view, "source_count": count, "tools": tools, "source_messages": source} + +def _restore_anthropic_history(value: object, messages: list[dict]) -> dict | None: + return copy.deepcopy(decode_anthropic_history(value, messages)) def _restore_phase(raw: object) -> SessionPhase: value = str(raw) diff --git a/opencollab/application/workflow.py b/opencollab/application/workflow.py index 491f8a27..49ab29c0 100644 --- a/opencollab/application/workflow.py +++ b/opencollab/application/workflow.py @@ -31,6 +31,7 @@ import math import time from collections.abc import Mapping, Sequence +from contextlib import ExitStack from copy import deepcopy from typing import Any @@ -210,6 +211,7 @@ def __init__( _TaskConcurrencyPermit | None ] = contextvars.ContextVar("workflow_task_concurrency_permit", default=None) self._pending_cleanup_tasks: set[asyncio.Task[Any]] = set() + self._workflow_imports: ExitStack | None = None self._active_call_tasks: set[asyncio.Task[Any]] = set() self._active_session_tasks: set[asyncio.Task[Any]] = set() self._agent_failures: list[dict[str, Any]] = [] @@ -712,6 +714,10 @@ def _track_session(self, session: Any) -> None: lease = self._active_budget_lease.get() if lease is not None: lease.sessions.append(session) + runner = getattr(session, "runner", None) + if lease.reserved and runner is not None: + # Other leases keep their allocation when a sibling overspends. + runner.budget_reservations.append(lambda: max(0, -int(self.budget.remaining()))) def _active_call_has_pending_cleanup(self) -> bool: lease = self._active_budget_lease.get() diff --git a/opencollab/application/workflow_candidates.py b/opencollab/application/workflow_candidates.py index d54b5f3e..03c728c4 100644 --- a/opencollab/application/workflow_candidates.py +++ b/opencollab/application/workflow_candidates.py @@ -6,6 +6,7 @@ import inspect import os from collections.abc import Awaitable, Callable, Mapping, Sequence +from contextlib import ExitStack from dataclasses import dataclass, replace from typing import Any @@ -74,6 +75,9 @@ class CandidateCaptureError(RuntimeError): class CandidateWorkspaceTrackingError(RuntimeError): """The source worktree changed while a candidate was executing.""" +class CandidateAdoptionRecoveryError(RuntimeError): + """Candidate publication failed and the source still needs recovery.""" + def _candidate_worktree_location(lease: Any) -> str: """Include an optional location in candidate retention diagnostics.""" workspace = getattr(lease, "candidate_workspace", None) @@ -492,6 +496,7 @@ async def run() -> CandidateRun: lease = await self._candidate_workspace.acquire(label) budget_lease = None child = None + imports = ExitStack() candidate: CandidateRun | None = None failure: BaseException | None = None preserve_lease = False @@ -528,6 +533,7 @@ async def run() -> CandidateRun: child._active_task_concurrency_permit = self._active_task_concurrency_permit child._budget_escape_state = self._budget_escape_state child.budget._parent_allocation = (self.budget, budget_lease) + child._workflow_imports = imports try: output = await workflow_fn(child, dict(args)) except Exception as exc: # noqa: BLE001 - preserve candidate edits @@ -624,6 +630,9 @@ async def run() -> CandidateRun: getattr(child._factory, "has_pending_isolated_cleanup", False) ) finally: + if child is not None: + child._workflow_imports = None + imports.close() if budget_lease is not None: self.budget.release(budget_lease) return await self._finish_candidate_run( diff --git a/opencollab/bootstrap/_workflow_runtime_cleanup.py b/opencollab/bootstrap/_workflow_runtime_cleanup.py index aafdff67..aa4b4513 100644 --- a/opencollab/bootstrap/_workflow_runtime_cleanup.py +++ b/opencollab/bootstrap/_workflow_runtime_cleanup.py @@ -80,11 +80,24 @@ def _positive_cleanup_timeout(value: float) -> float: return timeout async def _wait_for_context_cleanup(ctx: WorkflowContext, *, timeout: float) -> bool: - """Wait once for the cleanup tasks currently owned by the context.""" - pending = set(ctx.pending_cleanup_tasks) - if pending: - _done, pending = await asyncio.wait(pending, timeout=timeout) - return not pending and not ctx.pending_cleanup_tasks + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + saw_empty = False + while True: + pending = set(ctx.pending_cleanup_tasks) + if pending: + saw_empty = False + remaining = deadline - loop.time() + if remaining <= 0: + return False + _done, unfinished = await asyncio.wait(pending, timeout=remaining) + if unfinished: + return False + elif saw_empty: + return True + else: + saw_empty = True + await asyncio.sleep(0) def _session_environments(ctx: WorkflowContext) -> tuple[Any, ...]: environments: list[Any] = [] diff --git a/opencollab/bootstrap/_workflow_runtime_discovery.py b/opencollab/bootstrap/_workflow_runtime_discovery.py index 2a4903bd..bf12bf29 100644 --- a/opencollab/bootstrap/_workflow_runtime_discovery.py +++ b/opencollab/bootstrap/_workflow_runtime_discovery.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import importlib.abc import importlib.machinery import importlib.util @@ -14,6 +15,7 @@ from types import ModuleType from opencollab.adapters.safe_files import read_regular_bytes +from opencollab.application.workflow import WorkflowContext from opencollab.application.workflow_registry import Registry, WorkflowSpec from opencollab.bootstrap._workflow_runtime_state import ( MAX_WORKFLOW_DIRECTORY_ENTRIES, @@ -158,16 +160,67 @@ def release(self) -> None: self.capture_modules() _remove_workflow_package(self.package_name) + def release_after_cleanup(self, ctx: object, *, finalized: bool = False) -> None: + if not isinstance(ctx, WorkflowContext): + self.release() + return + caller = asyncio.current_task() + + def pending_tasks(): + return {task for task in getattr(ctx, "pending_cleanup_tasks", ()) if task is not caller} + + if finalized and not pending_tasks(): + self.release() + return + + loop = asyncio.get_running_loop() + watched = set() + scheduled = False + saw_empty = False + released = False + + def schedule(_task=None) -> None: + nonlocal scheduled + if not scheduled and not released: + scheduled = True + loop.call_soon(observe) + + def observe() -> None: + nonlocal scheduled, saw_empty, released + if released: + return + scheduled = False + pending = pending_tasks() + watched.intersection_update(pending) + if pending: + saw_empty = False + for task in pending - watched: + watched.add(task) + task.add_done_callback(schedule) + elif saw_empty: + released = True + self.release() + else: + # Finishing task callbacks may register another cleanup task. + saw_empty = True + schedule() + + observe() + def _bind_runtime_package(spec: WorkflowSpec, owner: _WorkflowModuleOwner) -> WorkflowSpec: original = spec.fn @wraps(original) async def run(ctx, args): owner.acquire() + imports = ctx._workflow_imports if isinstance(ctx, WorkflowContext) else None + if imports is not None: + imports.callback(owner.release_after_cleanup, ctx, finalized=True) + return await original(ctx, args) try: return await original(ctx, args) finally: - owner.release() + owner.release_after_cleanup(ctx) bound = WorkflowSpec( name=spec.name, diff --git a/opencollab/bootstrap/_workflow_runtime_execution.py b/opencollab/bootstrap/_workflow_runtime_execution.py index 0d82f927..7e91c5e1 100644 --- a/opencollab/bootstrap/_workflow_runtime_execution.py +++ b/opencollab/bootstrap/_workflow_runtime_execution.py @@ -5,6 +5,7 @@ import asyncio import uuid from collections.abc import Callable +from contextlib import ExitStack from typing import Any, Literal from opencollab.adapters.trace import Tracer @@ -155,6 +156,8 @@ async def run_workflow( stop_reason: Literal["budget_exceeded"] | None = None workflow_failure: BaseException | None = None cancellation: asyncio.CancelledError | None = None + imports = ExitStack() + ctx._workflow_imports = imports try: result = await fn(ctx, args) except WorkflowBudgetExceeded as exc: @@ -181,6 +184,9 @@ async def run_workflow( cancellation = cancellation or cleanup_cancellation except BaseException as exc: cleanup_failure = exc + finally: + ctx._workflow_imports = None + imports.close() tracer_failure = _close_tracer_capture(tracer) if owns_tracer else None tracer_write_error, tracer_dropped_steps, inspection_failure = _inspect_tracer(tracer) diff --git a/opencollab/bootstrap/authored_workflow.py b/opencollab/bootstrap/authored_workflow.py index b6cb00da..730c6a07 100644 --- a/opencollab/bootstrap/authored_workflow.py +++ b/opencollab/bootstrap/authored_workflow.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, get_args from opencollab.adapters._docker_snapshot import _copy_artifact_to_host +from opencollab.adapters._env_base import ExecResult, _track_shell_command from opencollab.adapters._env_process import ProcessCleanupError from opencollab.adapters.session_environments import SessionEnvironmentPool from opencollab.adapters.tools._paths import checked_path @@ -55,6 +56,7 @@ class _Child: stop_requested: bool = False status: str = "running" tokens: int = 0 + reported_tokens: int = 0 result: dict[str, Any] | None = None cleanup_error: Exception | None = None bridge: AuthoredWorkflowBridge | None = None @@ -212,9 +214,18 @@ def _snapshot(self, child: _Child, *, include_result: bool) -> dict[str, Any]: return record def _reserved_tokens(self) -> int: - return sum(max(0, (item.budget or 0) - item.tokens) + # Retain native spend while the child's usage file catches up. + return sum(max(0, (item.budget or 0) - item.tokens, + (item.bridge.tokens if item.bridge is not None else item.tokens) - item.tokens) for item in self._children.values() if item.status == "running") + def _remaining_child_budget(self, child: _Child) -> int | None: + remaining = self._get_session().runner.remaining_budget_tokens + if remaining is None: + return None + spent = max(child.tokens, child.bridge.tokens if child.bridge is not None else 0) + return remaining + max(0, (child.budget or 0) - spent) + async def _launch(self, params: dict[str, Any], runtime: ToolRuntime) -> _Child: parent = self._get_session() environment = runtime.environment @@ -231,7 +242,7 @@ async def _launch(self, params: dict[str, Any], runtime: ToolRuntime) -> _Child: source = params.get("source") if not isinstance(source, str) or not source or "\x00" in source: raise ValueError("source must name a Python file in the workspace") - source = checked_path(runtime, source) + source = checked_path(runtime, source, native_file=True) await environment.read_file(source) budget = params.get("budget") if budget is not None and (isinstance(budget, bool) or not isinstance(budget, int) or budget < 1): @@ -291,6 +302,11 @@ async def _launch(self, params: dict[str, Any], runtime: ToolRuntime) -> _Child: await runtime.safety_policy.check_cmd_interactive(command, runtime.confirm_fn()) child.bridge = AuthoredWorkflowBridge( environment, artifacts, pool, config, runtime, + budget=budget, + remaining_budget=lambda: self._remaining_child_budget(child), + on_usage=lambda tokens: self._account_usage(child, tokens, reported=False), + team_budget_exhausted=parent.runner._team_budget_exhausted, + cancel_requested=lambda: child.stop_requested or parent.runner._run_cancellation_requested(), allow_unisolated_shell=self._allow_unisolated_shell and not getattr(shell, "require_process_isolation", False), ) @@ -316,24 +332,29 @@ async def _refresh_usage(self, child: _Child) -> None: return self._account_usage(child, usage["tokens"]) - def _account_usage(self, child: _Child, tokens: Any) -> None: - if isinstance(tokens, bool) or not isinstance(tokens, int) or tokens < child.tokens: + def _account_usage(self, child: _Child, tokens: Any, *, reported: bool = True) -> None: + minimum = child.reported_tokens if reported else child.tokens + if isinstance(tokens, bool) or not isinstance(tokens, int) or tokens < minimum: raise ValueError("invalid child usage total") - delta = tokens - child.tokens + if reported: + child.reported_tokens = tokens + delta = max(child.tokens, tokens) - child.tokens if delta: parent = self._get_session() parent.state.add_used_tokens(delta) parent.enqueue_auto_save() - child.tokens = tokens + child.tokens += delta async def _execute(self, child: _Child, command: str, request_bytes: bytes) -> None: + async def run() -> ExecResult: + with _track_shell_command(child.environment): + return await child.environment.exec_cmd(command, timeout=math.inf, input_bytes=request_bytes) + execution = None logs: dict[str, Any] = {} status = "running" try: - execution = asyncio.create_task(child.environment.exec_cmd( - command, timeout=math.inf, input_bytes=request_bytes, - )) + execution = asyncio.create_task(run()) child.execution = execution if child.stop_requested: execution.cancel() @@ -377,7 +398,7 @@ async def _execute(self, child: _Child, command: str, request_bytes: bytes) -> N assert child.bridge is not None await await_owned_operation(child.bridge.aclose(), propagate_cancellation=False) await self._refresh_usage(child) - self._account_usage(child, max(child.tokens, child.bridge.tokens)) + self._account_usage(child, max(child.tokens, child.bridge.tokens), reported=False) except Exception as exc: status = "failed" child.cleanup_error = exc diff --git a/opencollab/bootstrap/authored_workflow_bridge.py b/opencollab/bootstrap/authored_workflow_bridge.py index 5fc2f6e7..412ebbe4 100644 --- a/opencollab/bootstrap/authored_workflow_bridge.py +++ b/opencollab/bootstrap/authored_workflow_bridge.py @@ -7,26 +7,30 @@ import math import posixpath import time -from collections.abc import AsyncIterator -from contextlib import asynccontextmanager +from collections.abc import AsyncIterator, Sequence +from contextlib import asynccontextmanager, nullcontext from contextvars import ContextVar from dataclasses import asdict from pathlib import Path -from typing import Any +from typing import Any, Callable -from opencollab.adapters._env_base import Environment, ExecResult, TextFileRange +from opencollab.adapters._env_base import Environment, ExecResult, TextFileRange, _track_shell_command from opencollab.adapters.llm import errors as llm_errors from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.llm.types import LLMResponse, Usage, to_plain_data -from opencollab.adapters.tools.base import host_write_lock +from opencollab.adapters.tools.base import host_write_locks +from opencollab.application._session_run_shared import _TeamBudgetStop, _TokenBudgetStop from opencollab.application._session_run_usage import _normalize_completion_usage from opencollab.application.async_timeout import await_owned_operation, consume_task_result from opencollab.application.exception_notes import add_exception_note +from opencollab.application.run_control import _RunCancelled +from opencollab.domain.agent import DEFAULT_MAX_TOKENS_PER_STEP _METADATA = {"context_window", "estimate_request_tokens", "estimate_context_tokens", "minimum_output_tokens"} _COMPLETION = {"messages", "tools", "temperature", "thinking", "thinking_params", "reasoning_effort", "tool_choice", "top_p", "max_output_tokens", "response_session_id"} -_ENVIRONMENT = {"exec_cmd", "read_file", "read_text_range", "write_file", "write_temp_file", "remove_file"} +_ENVIRONMENT = {"exec_cmd", "read_file", "read_text_range", "write_file", "write_temp_file", "remove_file", + "resolve_search_path"} def _write_json(path: Path, value: Any) -> None: @@ -36,29 +40,41 @@ def _write_json(path: Path, value: Any) -> None: def _restore_error(value: dict[str, Any]) -> Exception: - name = value.pop("type") - cls = getattr(llm_errors, name, getattr(builtins, name, RuntimeError)) - if not isinstance(cls, type) or not issubclass(cls, Exception): - cls = RuntimeError - error = cls(value.pop("message")) + error_type = value["type"] + error: Exception + if error_type == "_RunCancelled": + error = _RunCancelled() + elif error_type == "_TeamBudgetStop": + error = _TeamBudgetStop() + elif error_type == "_TokenBudgetStop": + error = _TokenBudgetStop( + reserved_input_tokens=value["reserved_input_tokens"], remaining_budget=value["remaining_budget"], + minimum_output_tokens=value["minimum_output_tokens"], + ) + else: + cls = getattr(llm_errors, error_type, getattr(builtins, error_type, RuntimeError)) + if not isinstance(cls, type) or not issubclass(cls, Exception): + cls = RuntimeError + error = cls(value["message"]) for name, item in value.items(): - setattr(error, name, Usage(**item) if name == "usage" else item) + if name not in {"type", "message"}: + setattr(error, name, Usage(**item) if name == "usage" else item) return error class _FileEdit: """Run native operations in the task holding their environment's edit lock.""" - def __init__(self, environment: Any, path: str) -> None: + def __init__(self, environment: Any, paths: Sequence[str]) -> None: self.environment = environment self.ready = asyncio.get_running_loop().create_future() self.ready.add_done_callback(consume_task_result) self.calls: asyncio.Queue[tuple[str, dict[str, Any], asyncio.Future[Any]] | None] = asyncio.Queue() - self.task = asyncio.create_task(self._run(path)) + self.task = asyncio.create_task(self._run(paths)) - async def _run(self, path: str) -> None: + async def _run(self, paths: Sequence[str]) -> None: try: - async with host_write_lock(path, self.environment): + async with host_write_locks(paths, self.environment): self.ready.set_result(None) while (call := await self.calls.get()) is not None: action, arguments, reply = call @@ -102,7 +118,11 @@ class AuthoredWorkflowBridge: """Own only this authored run's strategy pools and admitted operations.""" def __init__(self, environment: Any, artifacts: str, pool: Any, llm: Any, runtime: Any, - *, allow_unisolated_shell: bool) -> None: + *, allow_unisolated_shell: bool, budget: int | None = None, + remaining_budget: Callable[[], int | None] | None = None, + on_usage: Callable[[int], None] | None = None, + team_budget_exhausted: Callable[[], bool] | None = None, + cancel_requested: Callable[[], bool] | None = None) -> None: relative = posixpath.relpath(artifacts, environment.workspace) if relative == ".." or relative.startswith("../") or posixpath.isabs(relative): raise ValueError("child mailbox must be inside the member workspace") @@ -114,6 +134,11 @@ def __init__(self, environment: Any, artifacts: str, pool: Any, llm: Any, runtim self._environments = {0: environment} self._next_environment = 0 self.tokens = 0 + self._budget = budget + self._remaining_budget = remaining_budget + self._on_usage = on_usage + self._team_budget_exhausted = team_budget_exhausted + self._cancel_requested = cancel_requested self._tasks: dict[str, asyncio.Task] = {} self._handles: dict[str, int] = {} self._edits: dict[str, _FileEdit] = {} @@ -142,6 +167,41 @@ def options(self) -> dict[str, Any]: "supports_request_admission": self._config.get("request_pool") is not None, "supports_response_session_identity": self._config.get("wire_protocol") == "responses"} + def _request_output_limit(self, arguments: dict[str, Any]) -> int: + if self._cancel_requested is not None and self._cancel_requested(): + raise _RunCancelled + if self._team_budget_exhausted is not None and self._team_budget_exhausted(): + raise _TeamBudgetStop + configured = arguments.get("max_output_tokens") + if configured is None: + configured = self._config.get("max_output_tokens", DEFAULT_MAX_TOKENS_PER_STEP) + remaining = None if self._budget is None else self._budget - self.tokens + if self._remaining_budget is not None: + parent_remaining = self._remaining_budget() + if parent_remaining is not None: + remaining = parent_remaining if remaining is None else min(remaining, parent_remaining) + if remaining is None: + return configured + model = self._model() + thinking = arguments.get("thinking", False) + thinking_params = arguments.get("thinking_params") + reserved = model.estimate_request_tokens( + arguments["messages"], arguments.get("tools"), thinking=thinking, thinking_params=thinking_params, + ) + minimum = model.minimum_output_tokens( + max_output_tokens=configured, thinking=thinking, thinking_params=thinking_params, + ) + if remaining - reserved < minimum: + raise _TokenBudgetStop( + reserved_input_tokens=reserved, remaining_budget=remaining, minimum_output_tokens=minimum, + ) + return min(configured, remaining - reserved) + + def _account_native_usage(self, usage: Any, *, accounted_tokens: int = 0) -> None: + self.tokens += _normalize_completion_usage(usage)[1] - accounted_tokens + if self._on_usage is not None: + self._on_usage(self.tokens) + async def poll(self) -> None: try: requests = json.loads(await self._environment.read_file(posixpath.join(self._directory, "requests.json"))) @@ -183,10 +243,13 @@ async def _respond(self, identity: str, request: dict[str, Any]) -> None: if not isinstance(exc, (Exception, asyncio.CancelledError)): raise error = {"type": type(exc).__name__, "message": str(exc)} - for name in ("usage", "status_code", "code", "param", "body"): + for name in ("usage", "status_code", "code", "param", "body", "reserved_input_tokens", + "remaining_budget", "minimum_output_tokens"): value = getattr(exc, name, None) if value is not None: - error[name] = to_plain_data(value) + error[name] = ( + asdict(value) if name == "usage" and isinstance(value, Usage) else to_plain_data(value) + ) reply = {"error": error} await await_owned_operation(self._environment.write_file( posixpath.join(self._directory, f"{identity}.json"), json.dumps(reply, ensure_ascii=False), @@ -200,10 +263,13 @@ async def _dispatch(self, identity: str, request: dict[str, Any]) -> Any: if isinstance(handle, bool) or not isinstance(handle, int) or handle not in self._environments: raise ValueError("unknown environment for this authored run") environment = self._environments[handle] - if action == "file_write_lock": - if set(arguments) != {"path"}: - raise ValueError("file edit lock needs a path") - edit = _FileEdit(environment, arguments["path"]) + if action in {"file_write_lock", "file_write_locks"}: + if set(arguments) != ({"path"} if action == "file_write_lock" else {"paths"}): + raise ValueError("file edit lock needs paths") + paths = [arguments["path"]] if action == "file_write_lock" else arguments["paths"] + if not isinstance(paths, list) or not paths or any(not isinstance(path, str) for path in paths): + raise ValueError("file edit lock needs non-empty text paths") + edit = _FileEdit(environment, paths) self._edits[identity] = edit try: await asyncio.shield(edit.ready) @@ -233,6 +299,22 @@ async def _dispatch(self, identity: str, request: dict[str, Any]) -> Any: if set(arguments) - _COMPLETION: raise ValueError("unknown completion controls") notifications = [] + native_accounting = getattr(self._model(), "supports_request_revalidation", False) + if native_accounting: + accounted_tokens = 0 + + def account_attempt(usage: Any) -> None: + nonlocal accounted_tokens + total = _normalize_completion_usage(usage)[1] + accounted_tokens += total + self._account_native_usage(usage) + + arguments["before_request"] = lambda: self._request_output_limit(arguments) + arguments["on_usage"] = lambda usage: self._account_native_usage( + usage, accounted_tokens=accounted_tokens, + ) + if getattr(self._model(), "supports_attempt_usage", False): + arguments["on_attempt_usage"] = account_attempt if getattr(self._model(), "supports_request_admission", False): def admitted() -> None: notifications.append(asyncio.create_task(self._environment.write_file( @@ -243,10 +325,11 @@ def admitted() -> None: try: response = await self._model().complete(**arguments) except BaseException as exc: - if getattr(exc, "usage", None) is not None: - self.tokens += _normalize_completion_usage(exc.usage)[1] + if not native_accounting and getattr(exc, "usage", None) is not None: + self._account_native_usage(exc.usage) raise - self.tokens += _normalize_completion_usage(response.usage)[1] + if not native_accounting: + self._account_native_usage(response.usage) return to_plain_data(response) finally: if notifications: @@ -259,6 +342,7 @@ def admitted() -> None: self._next_environment += 1 self._environments[self._next_environment] = acquired return {"handle": self._next_environment, "workspace": acquired.workspace, + "remote_filesystem": getattr(acquired, "remote_filesystem", False) is True, "process_isolated": bool(getattr(acquired, "process_isolated", False))} if action == "release": if arguments or handle == 0: @@ -287,13 +371,14 @@ def admitted() -> None: if "input_bytes" in arguments: arguments["input_bytes"] = base64.b64decode(arguments["input_bytes"], validate=True) edit_id = request.get("edit") - if edit_id is None: - result = await getattr(environment, action)(**arguments) - else: - edit = self._edits.get(edit_id) - if edit is None or edit.environment is not environment: - raise ValueError("unknown file edit lock for this environment") - result = await edit.call(action, arguments) + with _track_shell_command(environment) if action == "exec_cmd" else nullcontext(): + if edit_id is None: + result = await getattr(environment, action)(**arguments) + else: + edit = self._edits.get(edit_id) + if edit is None or edit.environment is not environment: + raise ValueError("unknown file edit lock for this environment") + result = await edit.call(action, arguments) if isinstance(result, (ExecResult, TextFileRange)): result = asdict(result) if result.get("stdout_bytes") is not None: @@ -377,7 +462,7 @@ async def acknowledged() -> Any: usage = getattr(exc, "usage", None) else: usage = Usage(**value["usage"]) if action == "complete" and isinstance(value, dict) else None - if action == "file_write_lock": + if action in {"file_write_lock", "file_write_locks"}: try: await await_owned_operation(self.call("file_write_unlock", {"id": value}, environment)) except Exception as exc: @@ -409,6 +494,7 @@ def __init__(self, client: BridgeClient, options: dict[str, Any]) -> None: super().__init__() self.client, self.handle = client, options["handle"] self.workspace, self.process_isolated = options["workspace"], options["process_isolated"] + self.remote_filesystem = options.get("remote_filesystem") is True self._released = False self._edit: ContextVar[str | None] = ContextVar("authored_file_edit", default=None) @@ -417,7 +503,12 @@ async def _call(self, action: str, arguments: dict[str, Any]) -> Any: @asynccontextmanager async def file_write_lock(self, path: str) -> AsyncIterator[None]: - identity = await self.client.call("file_write_lock", {"path": path}, self.handle) + async with self.file_write_locks((path,)): + yield None + + @asynccontextmanager + async def file_write_locks(self, paths: Sequence[str]) -> AsyncIterator[None]: + identity = await self.client.call("file_write_locks", {"paths": list(paths)}, self.handle) token = self._edit.set(identity) try: yield None @@ -440,6 +531,9 @@ async def exec_cmd(self, cmd: str, timeout: float = 120.0, *, input_bytes: bytes async def read_file(self, path: str) -> str: return await self._call("read_file", {"path": path}) + async def resolve_search_path(self, path: str) -> str: + return await self._call("resolve_search_path", {"path": path}) + async def read_text_range(self, path: str, **kwargs: Any) -> TextFileRange: return TextFileRange(**await self._call("read_text_range", {"path": path, **kwargs})) diff --git a/opencollab/bootstrap/container.py b/opencollab/bootstrap/container.py index 86f15ee8..1e5ea104 100644 --- a/opencollab/bootstrap/container.py +++ b/opencollab/bootstrap/container.py @@ -492,6 +492,8 @@ def build_session_runtime( resolved_llm = _resolve_llm(agent, llm, llm_timeout, provider_retry_budget) if agent_profile is not None: + if safety_policy is None: + safety_policy = build_workspace_safety_policy(resolved_env) safety_policy = agent_profile.wrap_safety( safety_policy, resolved_env.workspace, diff --git a/opencollab/bootstrap/runtime_context.py b/opencollab/bootstrap/runtime_context.py index 79e579b2..3ed1ff9a 100644 --- a/opencollab/bootstrap/runtime_context.py +++ b/opencollab/bootstrap/runtime_context.py @@ -61,4 +61,4 @@ def build_workspace_safety_policy(env: Environment) -> SafetyPolicyPort | None: """A sandbox interceptor scoped to ``env``'s workspace, or None without one.""" if env is None or not getattr(env, "workspace", None): return None - return SandboxInterceptor(env.workspace) + return SandboxInterceptor(env.workspace, remote_filesystem=getattr(env, "remote_filesystem", False) is True) diff --git a/opencollab/bootstrap/team_workflow.py b/opencollab/bootstrap/team_workflow.py index 1792b440..4988f620 100644 --- a/opencollab/bootstrap/team_workflow.py +++ b/opencollab/bootstrap/team_workflow.py @@ -85,7 +85,6 @@ def build_child(**kwargs: Any) -> Session: def on_event(event: RunEvent) -> None: if event.type == "usage": - parent.state.add_used_tokens(event.data["total_tokens"]) parent.enqueue_auto_save() if control.on_event is not None: control.on_event(event) @@ -97,7 +96,9 @@ def on_event(event: RunEvent) -> None: runtime.safety_policy if kwargs["env"] is runtime.environment else build_workspace_safety_policy(kwargs["env"]) ) - return build_session(**kwargs) + session = build_session(**kwargs) + session.runner._on_completion_tokens = parent.runner._charge_completion_tokens + return session save_dir = ( tempfile.mkdtemp(prefix=f"workflow-agent-{runtime.aid}-", dir=self._save_dir) diff --git a/opencollab/builtin_workflows/duo/_candidate_records.py b/opencollab/builtin_workflows/duo/_candidate_records.py index b08a83a5..bf7d0903 100644 --- a/opencollab/builtin_workflows/duo/_candidate_records.py +++ b/opencollab/builtin_workflows/duo/_candidate_records.py @@ -5,6 +5,7 @@ import copy from typing import Any +from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError from opencollab.workflows import CandidateRun MAX_PUBLIC_COMMAND_BYTES = 4_000 @@ -98,6 +99,8 @@ async def _adopt( attempts.append(label) try: await ctx.adopt_candidate(candidate, preserve_paths=preserve_paths) + except CandidateAdoptionRecoveryError: + raise except Exception as exc: # noqa: BLE001 await ctx.log(f"Duo adoption failed ({label}) after {type(exc).__name__}") continue diff --git a/opencollab/domain/session.py b/opencollab/domain/session.py index a833cb67..fc79aaa1 100644 --- a/opencollab/domain/session.py +++ b/opencollab/domain/session.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import math import uuid from dataclasses import dataclass, field from datetime import datetime, timezone @@ -13,10 +14,36 @@ MAX_SCOUT_LEDGER_CARDS = 256 +def same_snapshot_value(left: Any, right: Any) -> bool: + if left is right: + return True + # The JSON encoder writes builtin subclasses as their underlying values. + if isinstance(left, str) and isinstance(right, str): + return str.__eq__(left, right) + if isinstance(left, dict) and isinstance(right, dict): + return left.keys() == right.keys() and all( + same_snapshot_value(value, right[key]) for key, value in left.items() + ) + if isinstance(left, list) and isinstance(right, list): + return len(left) == len(right) and all(same_snapshot_value(a, b) for a, b in zip(left, right)) + if isinstance(left, int) and isinstance(right, int): + return (type(left) is bool) == (type(right) is bool) and int.__eq__(left, right) + if isinstance(left, float) and isinstance(right, float): + left, right = float.__float__(left), float.__float__(right) + if left == right == 0: + return math.copysign(1, left) == math.copysign(1, right) + return left == right + return type(left) is type(right) and left == right + + def _now_iso() -> str: return datetime.now(timezone.utc).isoformat() +class _MessageTimestamp(Enum): + UNKNOWN = 0 + + class SessionPhase(Enum): IDLE = "idle" PRECHECK = "precheck" @@ -180,7 +207,7 @@ class SessionState: # Per-message creation timestamps (UTC ISO-8601), index-aligned with # ``messages``. Kept as a sidecar so ``messages`` stays a clean, # API-shaped list; merged into each message only when persisting. - message_timestamps: list[str] = field(default_factory=list) + message_timestamps: list[str | _MessageTimestamp] = field(default_factory=list) # User messages queued by the scheduler but not yet appended to # ``messages`` because the session is mid-turn or awaiting tool results. # These are persisted for observability/recovery without changing the @@ -222,7 +249,7 @@ def enriched_messages(self, start: int = 0) -> list[dict[str, Any]]: raise ValueError("message start must be a non-negative integer") self._align_timestamps() return [ - {**msg, "timestamp": ts} + dict(msg) if ts is _MessageTimestamp.UNKNOWN else {**msg, "timestamp": ts} for msg, ts in zip( self.messages[start:], self.message_timestamps[start:], @@ -288,24 +315,23 @@ def append_message(self, message: dict[str, Any]) -> None: self.messages.append(message) self.message_timestamps.append(_now_iso()) - def replace_messages(self, messages: list[dict[str, Any]]) -> None: - """Swap the conversation, re-deriving aligned timestamps. - - Precedence per message: an embedded ``timestamp`` (e.g. a resumed - transcript) wins; else the prior timestamp of the same dict object - (preserved across a slice-and-rebuild); else now. - """ + def replace_messages(self, messages: list[dict[str, Any]], *, restoring: bool = False) -> None: prior = {id(m): ts for m, ts in zip(self.messages, self.message_timestamps)} - rebuilt = [ - ( - ({k: v for k, v in m.items() if k != "timestamp"}, m["timestamp"]) - if isinstance(m, dict) and "timestamp" in m - else (m, prior.get(id(m)) or _now_iso()) - ) - for m in messages - ] - self.messages = [m for m, _ in rebuilt] - self.message_timestamps = [ts for _, ts in rebuilt] + rebuilt_messages: list[dict[str, Any]] = [] + timestamps: list[str | _MessageTimestamp] = [] + for message in messages: + if isinstance(message, dict) and "timestamp" in message: + timestamp = message["timestamp"] + message = {key: value for key, value in message.items() if key != "timestamp"} + elif id(message) in prior: + timestamp = prior[id(message)] + else: + # A legacy transcript leaves the message creation time unknown. + timestamp = _MessageTimestamp.UNKNOWN if restoring else _now_iso() + rebuilt_messages.append(message) + timestamps.append(timestamp) + self.messages = rebuilt_messages + self.message_timestamps = timestamps self.anthropic_history = None def add_used_tokens(self, tokens: int) -> None: diff --git a/tests/agents/test_session_construction.py b/tests/agents/test_session_construction.py index 87732561..85899e11 100644 --- a/tests/agents/test_session_construction.py +++ b/tests/agents/test_session_construction.py @@ -6,11 +6,14 @@ import json import os import threading +from collections import OrderedDict +from enum import IntEnum import pytest from opencollab.adapters.storage import SessionStore from opencollab.application.autosave import AutoSaveSubscriber +from opencollab.application.ports import SnapshotSequenceConflict from opencollab.application.scheduler_types import LaunchSpec from opencollab.application.session import SessionRuntime from opencollab.bootstrap import build_session as Session @@ -295,7 +298,56 @@ def test_checkpoint_rejects_sequence_older_than_persisted_journal(tmp_path): assert restored["_autosave_sequence"] == 2 assert restored["messages"][-1]["content"] == "newer-delta" -def test_checkpoint_rejects_conflicting_same_sequence_journal(tmp_path): + +class _SavedAgentId(IntEnum): + READER = 12 + + +class _SavedScore(float): + pass + + +class _SavedScores(list): + pass + + +@pytest.mark.parametrize(("aid", "metadata"), [ + pytest.param(_SavedAgentId.READER, {}, id="integer-enum"), + pytest.param(12, {"scores": [_SavedScore(-0.0), _SavedScore(0.75)]}, id="float-subclass"), + pytest.param(12, OrderedDict(scores=[-0.0, 0.75]), id="dict-subclass"), + pytest.param(12, {"scores": _SavedScores([-0.0, 0.75])}, id="list-subclass"), +]) +async def test_autosave_checkpoints_json_builtin_subclasses(local_environment_factory, tmp_path, aid, metadata): + path = tmp_path / "subclasses.json" + session = _new_session( + local_environment_factory, + env=local_environment_factory(str(tmp_path)), + aid=aid, + auto_save_path=str(path), + seed_user_messages=[{"role": "user", "content": "seed", "metadata": metadata}], + ) + try: + await session.add_user_message("record this turn") + await asyncio.gather(*session.pending_cleanup_tasks) + + assert session.persistence_errors == () + saved = json.loads(path.read_bytes()) + assert saved["_autosave_sequence"] == 1 + assert saved["aid"] == 12 + assert json.dumps(saved["messages"][1]["metadata"]) == json.dumps(metadata) + assert saved["messages"][-1]["content"] == "record this turn" + assert path.with_suffix(".json.journal").read_bytes() == b"" + finally: + await session.aclose() + + +@pytest.mark.parametrize("publisher", ["journal", "snapshot"]) +@pytest.mark.parametrize( + ("published_value", "conflicting_value"), + [("published", "conflict"), (True, 1), (1, 1.0), (0.0, -0.0)], + ids=["content", "bool-number", "int-float", "signed-zero"], +) +def test_checkpoint_rejects_conflicting_same_sequence(tmp_path, publisher, published_value, conflicting_value): path = tmp_path / "same-sequence-checkpoint.json" store = SessionStore() base = [ @@ -304,20 +356,70 @@ def test_checkpoint_rejects_conflicting_same_sequence_journal(tmp_path): ] meta = {"snapshot_version": 1, "session_state": {}} store.checkpoint_snapshot(str(path), base, meta=meta, sequence=1) - store.append_snapshot_delta( - str(path), - sequence=2, - replace_from=2, - messages=[{"role": "assistant", "content": "newer-delta"}], - meta=meta, - ) + published = {"role": "assistant", "content": "newer", "metadata": {"value": published_value}} + conflict = {**published, "metadata": {"value": conflicting_value}} + if publisher == "journal": + store.append_snapshot_delta(str(path), sequence=2, replace_from=2, messages=[published], meta=meta) + else: + store.checkpoint_snapshot(str(path), [*base, published], meta=meta, sequence=2) - with pytest.raises(ValueError, match="conflicts with persisted journal state"): - store.checkpoint_snapshot(str(path), base, meta=meta, sequence=2) + with pytest.raises(ValueError, match=f"conflicts with persisted {publisher} state"): + store.checkpoint_snapshot(str(path), [*base, conflict], meta=meta, sequence=2) restored = store.load_snapshot(str(path), "system") assert restored["_autosave_sequence"] == 2 - assert restored["messages"][-1]["content"] == "newer-delta" + assert json.dumps(restored["messages"][-1]) == json.dumps(published) + + store.checkpoint_snapshot(str(path), [*base, published], meta=meta, sequence=2) + assert path.with_suffix(".json.journal").read_bytes() == b"" + store.checkpoint_snapshot(str(path), [*base, conflict], meta=meta, sequence=3) + assert store.load_snapshot(str(path), "system")["_autosave_sequence"] == 3 + +@pytest.mark.parametrize("publisher", ["snapshot", "journal"]) +@pytest.mark.parametrize( + "change", ["encoding", "transcript", "timestamp", "projection", "source", "tools", "malformed"], +) +def test_same_sequence_compares_native_history(tmp_path, publisher, change): + path = str(tmp_path / "native-checkpoint.json") + store = SessionStore() + system = {"role": "system", "content": "system"} + user = {"role": "user", "content": "request"} + published = [system, {**user, "timestamp": "2026-01-01T00:00:00+00:00"}] + legacy = {"messages": [system, user], "source_count": 2, "tools": None} + indexed = {**legacy, "messages": [0, 1]} + candidate = published.copy() + if change == "transcript": + candidate[1] = {**published[1], "content": "new request"} + elif change == "timestamp": + candidate[1] = {**published[1], "timestamp": "2026-01-01T00:00:01+00:00"} + elif change == "projection": + indexed["messages"] = [0, {**user, "content": "edited projection"}] + elif change == "source": + indexed.update(messages=[system, user], source_messages=[system, {**user, "content": "old source"}]) + elif change == "tools": + indexed["tools"] = [{"name": "lookup"}] + elif change == "malformed": + legacy["messages"] = indexed["messages"] = [0, 2] + + meta = {"session_state": {"anthropic_history": legacy}} + if publisher == "snapshot": + store.checkpoint_snapshot(path, published, meta=meta, sequence=1) + else: + store.append_snapshot_delta(path, messages=published, meta=meta, sequence=1, replace_from=0) + previous = store.load_snapshot(path, "system") + updated_meta = {"session_state": {"anthropic_history": indexed}} + if change == "encoding": + assert not store.append_snapshot_replacement( + path, candidate, meta=updated_meta, sequence=1, reject_conflicts=True, + ) + store.checkpoint_snapshot(path, candidate, meta=updated_meta, sequence=1) + assert store.load_snapshot(path, "system")["session_state"]["anthropic_history"] == indexed + else: + with pytest.raises(SnapshotSequenceConflict): + store.append_snapshot_replacement(path, candidate, meta=updated_meta, sequence=1, reject_conflicts=True) + with pytest.raises(SnapshotSequenceConflict, match=f"conflicts with persisted {publisher} state"): + store.checkpoint_snapshot(path, candidate, meta=updated_meta, sequence=1) + assert store.load_snapshot(path, "system") == previous def test_checkpoint_rejects_ambiguous_duplicate_journal_sequence(tmp_path): path = str(tmp_path / "duplicate-sequence-checkpoint.json") diff --git a/tests/agents/test_session_mixed_tool_snapshot.py b/tests/agents/test_session_mixed_tool_snapshot.py index 3723c5eb..62bd108a 100644 --- a/tests/agents/test_session_mixed_tool_snapshot.py +++ b/tests/agents/test_session_mixed_tool_snapshot.py @@ -120,7 +120,14 @@ async def pause(event): source.save(str(saved)) restored_workspace = tmp_path / "restored" shutil.copytree(workspace, restored_workspace) - recovered = load_session(str(saved), agent=agent, env=LocalEnvironment(str(restored_workspace)), llm=model) + recovered = load_session( + str(saved), agent=agent, env=LocalEnvironment(str(restored_workspace)), + llm=model, auto_save_path=str(tmp_path / "restored-autosave.json") if deferred else None, + ) + if deferred: + recovered_messages = recovered.state.enriched_messages() + recovered.restore(str(saved)) + assert recovered.state.enriched_messages() == recovered_messages tool_messages = [row for row in recovered.messages if row["role"] == "tool"] assert [row["tool_call_id"] for row in tool_messages] == [ diff --git a/tests/agents/test_session_persistence.py b/tests/agents/test_session_persistence.py index 4fda8ddb..413bb7c3 100644 --- a/tests/agents/test_session_persistence.py +++ b/tests/agents/test_session_persistence.py @@ -157,6 +157,62 @@ def test_store_replay_ignores_journal_records_covered_by_atomic_base(tmp_path) - "step 3", ] +def test_invalid_checkpoint_messages_preserve_the_durable_journal(tmp_path) -> None: + path = str(tmp_path / "session.json") + store = SessionStore() + messages = [{"role": "user", "content": "last durable prompt"}] + store.append_snapshot_delta(path, sequence=1, replace_from=0, messages=messages, meta={}) + + with pytest.raises(ValueError, match="missing 'content'"): + store.checkpoint_snapshot(path, [{"role": "user"}], meta={}, sequence=2) + + restored = store.load_snapshot(path, "fallback") + assert restored["_autosave_sequence"] == 1 + assert restored["messages"] == messages + +def test_journal_replacement_uses_the_durable_prefix_and_requires_a_new_sequence(tmp_path, monkeypatch) -> None: + path = tmp_path / "replacement.json" + journal = path.with_suffix(".json.journal") + store = SessionStore() + base = [{"role": "system", "content": "prompt"}] + archive = {"role": "user", "content": "\u20ac" * 1024} + replacement = [*base, archive, {"role": "assistant", "content": "latest"}] + store.checkpoint_snapshot(str(path), base, meta={}, sequence=1) + store.append_snapshot_delta( + str(path), sequence=2, replace_from=1, messages=[archive], meta={}, + seen_result_hashes_added=["prior"], + ) + before = journal.read_bytes() + monkeypatch.setattr(storage_module, "MAX_SESSION_SNAPSHOT_BYTES", len(before) + 512) + + meta = {"session_state": {"seen_result_hashes": ["latest"]}} + for sequence in (1, 2): + assert not store.append_snapshot_replacement(str(path), replacement, meta=meta, sequence=sequence) + assert journal.read_bytes() == before + + edited_archive = {"role": "user", "content": "\u96ea" * 1024} + with pytest.raises(ValueError, match="autosave journal exceeds"): + store.append_snapshot_replacement(str(path), [*base, edited_archive], meta=meta, sequence=3) + assert journal.read_bytes() == before + + assert store.append_snapshot_replacement(str(path), replacement, meta=meta, sequence=3) + saved = SessionStore().load_snapshot(str(path), "prompt") + assert saved == { + "_autosave_sequence": 3, + "messages": replacement, + "session_state": {"seen_result_hashes": ["latest"]}, + } + assert json.loads(journal.read_bytes().splitlines()[-1])["replace_from"] == 2 + +def test_journal_replacement_uses_real_messages_from_blank_legacy_base(tmp_path) -> None: + path = tmp_path / "empty.json" + path.touch() + messages = [{"role": "system", "content": ""}, {"role": "user", "content": "persist me"}] + store = SessionStore() + + assert store.append_snapshot_replacement(str(path), messages, meta={}, sequence=1) + assert store.load_messages(str(path), "") == messages + def test_store_rejects_complete_invalid_journal_record(tmp_path) -> None: path = tmp_path / "session.json" store = SessionStore() diff --git a/tests/agents/test_session_snapshot_context.py b/tests/agents/test_session_snapshot_context.py index decadc0b..7fe2f3f8 100644 --- a/tests/agents/test_session_snapshot_context.py +++ b/tests/agents/test_session_snapshot_context.py @@ -299,7 +299,12 @@ async def hold_submission(event): restored = load_session( str(saved), agent=agent, env=LocalEnvironment(str(restored_workspace)), llm=model, max_steps=max_steps, + auto_save_path=str(tmp_path / "restored-autosave.json") if event_type == "tool_end" else None, ) + if event_type == "tool_end": + recovered_messages = restored.state.enriched_messages() + restored.restore(str(saved)) + assert restored.state.enriched_messages() == recovered_messages for session in (branch, restored): assert await session.run_loop() == summary assert session.phase is SessionPhase.DONE diff --git a/tests/interfaces/test_member_authored_workflow.py b/tests/interfaces/test_member_authored_workflow.py index c74f0b57..1f0109fe 100644 --- a/tests/interfaces/test_member_authored_workflow.py +++ b/tests/interfaces/test_member_authored_workflow.py @@ -3,14 +3,18 @@ import asyncio import json import os +import shutil import sys import threading from contextlib import contextmanager +from dataclasses import replace from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import pytest +from opencollab.adapters._env_docker import DockerEnvironment +from opencollab.adapters._env_process import ProcessResult from opencollab.adapters.llm.types import LLMResponse, Usage from opencollab.adapters.tools.bash import BashTool from opencollab.application.event_bus import EventBus @@ -19,11 +23,13 @@ from opencollab.bootstrap import container from opencollab.bootstrap.authored_workflow import AuthoredWorkflowTool from opencollab.bootstrap.context_builder import SpawnConfig +from opencollab.bootstrap.runtime_context import build_workspace_safety_policy from opencollab.bootstrap.session_factory import build_session from opencollab.bootstrap.team_workflow import TeamWorkflowTool from opencollab.bootstrap.workflow_runtime import run_workflow from opencollab.domain.agent import Agent from opencollab.sdk import OpenCollab +from tests.support.docker_native_edits_support import RemoteDockerTransport from tests.support.environment_test_support import local_environment_factory as local_environment_factory from tests.support.paths import REPO_ROOT @@ -203,6 +209,36 @@ async def wait(): await asyncio.wait_for(wait(), 10) +async def test_docker_authored_source_resolves_parent_in_the_runner(tmp_path): + class RunnerTransport(RemoteDockerTransport): + async def __call__(self, *argv, input_bytes=None, **kwargs): + if argv[0] == "cp": + source = argv[-2].split(":", 1)[1].replace(self.remote, str(self.workspace)) + shutil.copyfile(source, argv[-1]) + return ProcessResult(0, b"", b"") + if input_bytes is not None and "opencollab.bootstrap.authored_workflow_runner" in argv[-1]: + input_bytes = input_bytes.replace(self.remote.encode(), str(self.workspace).encode()) + return await super().__call__(*argv, input_bytes=input_bytes, **kwargs) + + (tmp_path / "nested").mkdir() + (tmp_path / "authored.py").write_text('async def main(ctx, args):\n return "WORKFLOW_MARKER"\n') + transport = RunnerTransport(tmp_path) + environment = DockerEnvironment( + workspace=transport.remote, exec_workdir=transport.remote, container_id="b" * 64, + ) + transport.attach(environment) + parent, tool, runtime = _manager(environment) + runtime = replace(runtime, safety_policy=build_workspace_safety_policy(environment)) + try: + child = await _launch(tool, runtime, source="nested/../authored.py") + result = json.loads(await tool.execute_with_runtime({"action": "wait", "id": child["id"]}, runtime)) + assert result["status"] == "completed", result + assert result["result"]["output"] == "WORKFLOW_MARKER" + finally: + await parent.aclose() + await environment.cleanup() + + @pytest.mark.parametrize("stop", ["explicit", "parent-close", "cancel-wait"]) async def test_stop_cleans_child_before_returning(local_environment_factory, tmp_path, stop): (tmp_path / "authored.py").write_text('''import asyncio diff --git a/tests/interfaces/test_member_authored_workflow_queued_budget.py b/tests/interfaces/test_member_authored_workflow_queued_budget.py new file mode 100644 index 00000000..df936bc1 --- /dev/null +++ b/tests/interfaces/test_member_authored_workflow_queued_budget.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +import asyncio +import json +import sys +import threading +from contextlib import asynccontextmanager + +import pytest + +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm import client as transport +from opencollab.application.scheduler import Scheduler +from opencollab.bootstrap.authored_workflow import _Child +from opencollab.bootstrap.session_factory import _team_budget_guard +from opencollab.domain.scheduler import SessionControlBlock +from tests.interfaces.test_member_authored_workflow import _manager, _serve_model +from tests.support.paths import REPO_ROOT + + +@pytest.mark.parametrize("first_tokens, parent_stop, child_escape", [ + (5_000, None, False), (20_210, None, False), (5_000, "team", False), (5_000, "cancel", False), + (20_210, None, True), +]) +async def test_authored_child_stops_a_queued_request_after_shared_budget_exhaustion( + tmp_path, monkeypatch, first_tokens, parent_stop, child_escape, +): + monkeypatch.setenv("PYTHONPATH", str(REPO_ROOT)) + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + (tmp_path / "authored.py").write_text('''import asyncio +from pathlib import Path + +async def main(ctx, args): + first = asyncio.create_task(ctx.agent("first", budget=18000, tools=[])) + if args.get("escape"): + await first + while not Path("first-network").exists(): + await asyncio.sleep(0.01) + second = asyncio.create_task(ctx.agent("second", budget=2000, tools=[], over_budget_ok=args.get("escape", False))) + answers = await asyncio.gather(first, second) + return {"answers": answers, "reason": ctx.sessions[1].state.terminal_reason} +''', encoding="utf-8") + pool = tmp_path / "pool" + pool.mkdir() + release, first_entered, queued = threading.Event(), threading.Event(), asyncio.Event() + native_slot = transport.request_slot + entries = 0 + + @asynccontextmanager + async def observed_slot(*args): + nonlocal entries + entries += 1 + if entries == 2: + queued.set() + async with native_slot(*args): + yield + + monkeypatch.setattr(transport, "request_slot", observed_slot) + + def reply(body): + first = body["messages"][-1]["content"].startswith("first") + if first: + (tmp_path / "first-network").write_text("ready", encoding="utf-8") + first_entered.set() + assert release.wait(15) + total = first_tokens if first else 4 + return ({"role": "assistant", "content": "first answer" if first else "second answer"}, + {"prompt_tokens": total - 2, "completion_tokens": 2, "total_tokens": total}) + + environment = LocalEnvironment(str(tmp_path)) + with _serve_model(reply) as (url, requests): + parent, tool, runtime = _manager(environment, budget=20_000) + aggregate_exhausted, cancel = asyncio.Event(), asyncio.Event() + parent.runner._team_budget_exhausted = aggregate_exhausted.is_set + parent.runner.default_cancel_event = cancel + tool._config.base_url = parent.agent.base_url = url + tool._config.api_key = parent.agent.api_key = "controlled-test" # pragma: allowlist secret + tool._config.request_pool = parent.agent.request_pool = str(pool) + tool._config.request_capacity = parent.agent.request_capacity = 1 + parent.agent.max_tokens_per_step = 128 + try: + launched = json.loads(await tool.execute_with_runtime( + {"action": "launch", "source": "authored.py", "python": sys.executable, + "budget": 20_000, "concurrency": 2, "args": {"escape": child_escape}}, runtime, + )) + child = tool._children[launched["id"]] + if child_escape: + assert await asyncio.to_thread(first_entered.wait, 15) + else: + await asyncio.wait_for(queued.wait(), 15) + assert len(requests) == 1 + if parent_stop == "team": + aggregate_exhausted.set() + elif parent_stop == "cancel": + cancel.set() + release.set() + await asyncio.wait_for(child.task, 15) + cancelled = parent_stop == "cancel" and child.result["status"] == "stopped" + if cancelled: + assert child.result["reason"] == "cancelled" + assert child.result["output"] is None + assert len(requests) == 1 + else: + assert child.result["status"] == "completed", child.result + assert child.result["agent_failures"] == [] + if first_tokens >= 20_000 or parent_stop is not None: + assert len(requests) == 1 + assert child.result["output"]["answers"] == ["first answer", ""] + reason = {"team": "team budget exceeded", "cancel": "interrupted by user"}.get( + parent_stop, "budget exhausted before model call", + ) + assert reason in child.result["output"]["reason"] + else: + assert len(requests) == 2 + assert child.result["output"]["answers"] == ["first answer", "second answer"] + assert child.bridge.tokens == child.tokens == parent.used_tokens == first_tokens + 4 * (len(requests) - 1) + finally: + release.set() + await parent.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize("first_tokens, parent_budget", [(5_000, 20_000), (20_210, 20_000), (20_210, None)]) +@pytest.mark.parametrize("separate_parents", [False, True], ids=["one-role", "two-team-roles"]) +async def test_parallel_authored_children_share_fresh_parent_budget( + tmp_path, monkeypatch, first_tokens, parent_budget, separate_parents, +): + monkeypatch.setenv("PYTHONPATH", str(REPO_ROOT)) + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + (tmp_path / "authored.py").write_text( + 'async def main(ctx, args):\n return await ctx.agent(args["prompt"], tools=[], budget=10000)\n', + encoding="utf-8", + ) + pool = tmp_path / "pool" + pool.mkdir() + first_entered, release = threading.Event(), threading.Event() + queued, released_slot = asyncio.Event(), asyncio.Event() + native_slot = transport.request_slot + entries = 0 + + @asynccontextmanager + async def ordered_slot(*args): + nonlocal entries + entries += 1 + index = entries + if index > 1: + queued.set() + await released_slot.wait() + async with native_slot(*args): + yield + if index == 1: + released_slot.set() + + monkeypatch.setattr(transport, "request_slot", ordered_slot) + + observed_spend = [] + + def reply(body): + first = body["messages"][-1]["content"].startswith("first") + observed_spend.append(sum(parent.used_tokens for parent, _tool, _runtime in managers)) + if first: + first_entered.set() + assert release.wait(15) + total = first_tokens if first else 4 + return ({"role": "assistant", "content": "answer"}, + {"prompt_tokens": total - 2, "completion_tokens": 2, "total_tokens": total}) + + environment = LocalEnvironment(str(tmp_path)) + managers = [] + scheduler = Scheduler(session_factory=None, worktree_pool=None, event_sink=None, max_budget_tokens=20_000) + with _serve_model(reply) as (url, requests): + allowance = 10_000 if separate_parents and parent_budget is not None else parent_budget + for _ in range(2 if separate_parents else 1): + parent, tool, runtime = _manager(environment, budget=allowance) + if separate_parents: + parent.runner._team_budget_exhausted = _team_budget_guard(scheduler) + aid = scheduler.table.allocate_aid() + scheduler.table.add(SessionControlBlock( + aid=aid, parent_aid=None, agent=parent.agent, state=parent.state, + )) + scheduler._sessions[aid] = parent + tool._config.base_url = parent.agent.base_url = url + tool._config.api_key = parent.agent.api_key = "controlled-test" # pragma: allowlist secret + tool._config.request_pool = parent.agent.request_pool = str(pool) + tool._config.request_capacity = parent.agent.request_capacity = 1 + parent.agent.max_tokens_per_step = 128 + managers.append((parent, tool, runtime)) + children = [] + try: + for index, prompt in enumerate(("first", "second")): + parent, tool, runtime = managers[index if separate_parents else 0] + launched = json.loads(await tool.execute_with_runtime( + {"action": "launch", "source": "authored.py", "python": sys.executable, + "budget": 10_000, "args": {"prompt": prompt}}, runtime, + )) + children.append(tool._children[launched["id"]]) + if prompt == "first": + assert await asyncio.to_thread(first_entered.wait, 15) + await asyncio.wait_for(queued.wait(), 15) + assert len(requests) == 1 + release.set() + await asyncio.wait_for(asyncio.gather(*(child.task for child in children)), 20) + admitted = (parent_budget is None and not separate_parents) or first_tokens < 20_000 + assert len(requests) == (2 if admitted else 1) + assert all(child.result["status"] == "completed" for child in children) + assert children[1].result["output"] == ("answer" if admitted else "") + assert children[0].tokens == children[0].bridge.tokens == first_tokens + assert children[1].tokens == children[1].bridge.tokens == (4 if admitted else 0) + total = first_tokens + (4 if admitted else 0) + assert sum(parent.used_tokens for parent, _tool, _runtime in managers) == total + if admitted: + assert observed_spend[1] == first_tokens + if separate_parents: + assert scheduler.used_tokens == first_tokens + (4 if admitted else 0) + finally: + release.set() + for parent, _tool, _runtime in managers: + await parent.aclose() + await environment.cleanup() + + +async def test_host_usage_and_stale_worker_reports_are_deduplicated(tmp_path): + environment = LocalEnvironment(str(tmp_path)) + artifacts = tmp_path / "child" + artifacts.mkdir() + parent, tool, _runtime = _manager(environment, budget=1_000) + child = _Child("child", environment, str(artifacts), 400) + try: + tool._account_usage(child, 90, reported=False) + assert parent.used_tokens == child.tokens == 90 + assert child.reported_tokens == 0 + for reported in (0, 30, 30, 60, 90): + (artifacts / "usage.json").write_text(json.dumps({"tokens": reported}), encoding="utf-8") + await tool._refresh_usage(child) + assert parent.used_tokens == child.tokens == 90 + assert child.reported_tokens == reported + tool._account_usage(child, 100, reported=False) + tool._account_usage(child, 90) + assert child.reported_tokens == 90 + tool._account_usage(child, 100) + assert parent.used_tokens == child.tokens == child.reported_tokens == 100 + for invalid in (99, -1, True, 100.0, "100"): + with pytest.raises(ValueError, match="invalid child usage total"): + tool._account_usage(child, invalid) + assert parent.used_tokens == child.tokens == child.reported_tokens == 100 + finally: + await parent.aclose() + await environment.cleanup() diff --git a/tests/interfaces/test_sdk_budget_stop_answer.py b/tests/interfaces/test_sdk_budget_stop_answer.py index 42ba5113..dac26d17 100644 --- a/tests/interfaces/test_sdk_budget_stop_answer.py +++ b/tests/interfaces/test_sdk_budget_stop_answer.py @@ -8,12 +8,15 @@ import pytest from opencollab import BudgetDecision, OpenCollab, RunControl +from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.tools.base import Tool from opencollab.bootstrap import agent_runtime from tests.support.provider_sdk_http import http_response, install_sdk_transport from tests.support.responses_provider_test_support import function_item, message_item ANSWER = "The measured total is 42. The calculation is complete." +REQUEST_INPUT_ESTIMATE = 500 +INITIAL_BUDGET = 1075 PROMPT = "Read these notes and return the measured total.\n" + ( "The measured count was recorded as 42 and checked by the operator.\n" * 30 ) @@ -41,7 +44,7 @@ async def test_budget_stop_preserves_answer_after_native_retry(tmp_path, monkeyp retry = case not in {"single", "truncated"} with_tool = case.endswith("_tool") has_text = case not in {"soft_tool", "hard_tool"} - hard_budget = 1000 if case.startswith("hard_") else 3000 + hard_budget = INITIAL_BUDGET if case.startswith("hard_") else 3000 expected_tokens = 1115 if retry else 565 stopped_for_budget = case.startswith(("soft_", "hard_")) original = agent_runtime.build_session @@ -93,6 +96,7 @@ def handler(request): monkeypatch.delenv("OPENCOLLAB_UNBOUNDED_LIMITS", raising=False) monkeypatch.delenv("OPENCOLLAB_EXTERNAL_PROVIDER_ISOLATION", raising=False) monkeypatch.setattr(agent_runtime, "build_session", observed_session) + monkeypatch.setattr(LLMClient, "estimate_request_tokens", lambda *_args, **_kwargs: REQUEST_INPUT_ESTIMATE) install_sdk_transport(monkeypatch, "openai", handler) artifacts = tmp_path / "artifacts" result = await OpenCollab( @@ -102,7 +106,7 @@ def handler(request): ).agent2( PROMPT, tools=(tool,) if with_tool else (), system_prompt="Answer the user using the supplied notes.", budget=hard_budget, max_steps=4, trace=trace, artifacts=artifacts, - run_control=RunControl(initial_soft_budget_tokens=1000, decide_budget=decide, on_event=events.append), + run_control=RunControl(initial_soft_budget_tokens=INITIAL_BUDGET, decide_budget=decide, on_event=events.append), ) saved = OpenCollab.read_session_snapshot(artifacts / "agent.json") messages = saved["messages"] @@ -127,7 +131,9 @@ def handler(request): assert all((0 if retry and index == 0 else 15) <= body["max_output_tokens"] for index, body in enumerate(requests)) request_snapshot = next(snapshot for snapshot in snapshots if snapshot.reason == "request") - assert request_snapshot.reserved_input_tokens >= 550 + assert request_snapshot.reserved_input_tokens == REQUEST_INPUT_ESTIMATE + if retry: + assert requests[1]["max_output_tokens"] == INITIAL_BUDGET - 550 - REQUEST_INPUT_ESTIMATE == 25 assert [event.data["total_tokens"] for event in events if event.type == "usage"] == [expected_tokens] assert len(ledger) == 1 and ledger[0]["usage"]["total_tokens"] == expected_tokens assert ledger[0]["usage"]["input_tokens"] == (1100 if retry else 550) @@ -141,7 +147,7 @@ def handler(request): assert all(not message.get("response_items") and not message.get("provider_state") for message in messages) if case.startswith("soft_"): assert snapshots[-1].reason == "after_response" - assert snapshots[-1].used_tokens == expected_tokens and snapshots[-1].soft_budget_tokens == 1000 + assert snapshots[-1].used_tokens == expected_tokens and snapshots[-1].soft_budget_tokens == INITIAL_BUDGET else: assert all(snapshot.used_tokens < hard_budget for snapshot in snapshots) elif case == "growth": diff --git a/tests/interfaces/test_sdk_file_path_identity.py b/tests/interfaces/test_sdk_file_path_identity.py new file mode 100644 index 00000000..12ffae44 --- /dev/null +++ b/tests/interfaces/test_sdk_file_path_identity.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +import asyncio +import json +import os +from types import SimpleNamespace + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters._env_docker import DockerEnvironment +from opencollab.adapters._team_candidate_environment import _TeamCandidateEnvironment +from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.adapters.safety import SandboxInterceptor +from opencollab.adapters.tools.base import host_write_lock +from opencollab.adapters.tools.fs import FileWriteTool +from opencollab.application.tool_execution import ToolRuntime +from opencollab.bootstrap.runtime_context import build_workspace_safety_policy +from opencollab.environments import local_environment +from opencollab.tools import builtin_tools +from tests.support.docker_native_edits_support import RemoteDockerTransport + + +class _PathModel: + def __init__(self, tool, arguments): + self.tool = tool + self.arguments = arguments + self.called = False + + async def complete(self, messages, **kwargs): + if self.called: + return LLMResponse(content="finished", usage=Usage(10, 5), finish_reason="stop") + self.called = True + return LLMResponse(tool_calls=[{ + "id": "selected-file", "type": "function", "function": { + "name": self.tool, "arguments": json.dumps(self.arguments), + }, + }], usage=Usage(10, 5), finish_reason="tool_calls") + + +@pytest.mark.parametrize("absolute", [False, True]) +@pytest.mark.parametrize("tool", ["file_read", "file_write", "apply_patch", "grep"]) +@pytest.mark.parametrize(("backend", "parent"), [("local", "alias"), ("docker", "alias"), ("docker", "sub")]) +async def test_sdk_file_tools_match_shell_through_parent(tmp_path, absolute, tool, backend, parent): + (tmp_path / "actual/child").mkdir(parents=True) + (tmp_path / "alias").symlink_to("actual/child", target_is_directory=True) + (tmp_path / "sub").mkdir() + target = tmp_path / ("actual/module.py" if parent == "alias" else "module.py") + unrelated = tmp_path / ("module.py" if parent == "alias" else "actual/module.py") + target.write_text("VALUE = 1\n") + reading = tool in {"file_read", "grep"} + unrelated.write_text("UNRELATED = 7\n" if reading else "VALUE = 1\n") + if backend == "docker": + transport = RemoteDockerTransport(tmp_path) + environment = DockerEnvironment( + workspace=transport.remote, container_id="a" * 64, exec_workdir=transport.remote, + ) + transport.attach(environment) + assert not os.path.exists(transport.remote) + else: + environment = local_environment(tmp_path) + raw = f"{parent}/../module.py" + path = os.path.join(environment.workspace, raw) if absolute else raw + arguments = {"path": path} + if tool == "file_write": + arguments.update(mode="str_replace", old_str="VALUE = 1", new_str="VALUE = 42") + elif tool == "apply_patch": + arguments.update( + mode="line_replace", start_line=1, end_line=1, + expected_str="VALUE = 1", new_str="VALUE = 42", + ) + elif tool == "grep": + arguments["pattern"] = "VALUE" + model = _PathModel(tool, arguments) + outputs = [] + + async def flow(ctx, _): + result = await ctx.agent_run( + "Use the requested file tool", tools=list(builtin_tools(tool, headless=False)), + isolation="shared", budget=2000, system_prompt="Use the requested file tool", + ) + outputs.extend( + message["content"] for session in ctx.sessions for message in session.state.messages + if message.get("role") == "tool" + ) + return result.status + + try: + result = await OpenCollab(tmp_path, environment=environment, model="local-scripted").workflow( + flow, llm=model, trace=False, timeout=15, budget=5000, concurrency=1, + limit_mode="explicit", agent_profile="single2", + ) + assert result.ok, result.reason + assert result.output == "completed" + assert len(outputs) == 1 and not outputs[0].startswith("Error") + if reading: + assert ("1\tVALUE = 1" if tool == "file_read" else "1:VALUE = 1") in outputs[0] + assert "UNRELATED" not in outputs[0] + else: + assert target.read_text() == "VALUE = 42\n" + assert unrelated.read_text() == ("UNRELATED = 7\n" if reading else "VALUE = 1\n") + shell = await environment.exec_cmd(f"cat {raw}") + assert shell.stdout == target.read_text() + finally: + await environment.cleanup() + + +async def test_wrapped_docker_workspace_resolves_parent_components_in_the_container(tmp_path): + (tmp_path / "sub").mkdir() + target = tmp_path / "module.py" + target.write_text("VALUE = 1\n") + transport = RemoteDockerTransport(tmp_path) + environment = DockerEnvironment( + workspace=transport.remote, container_id="a" * 64, exec_workdir=transport.remote, + ) + transport.attach(environment) + wrapped = _TeamCandidateEnvironment(SimpleNamespace(environment=environment), transport.remote) + runtime = ToolRuntime(wrapped, build_workspace_safety_policy(wrapped), None) + try: + result = await FileWriteTool().execute_with_runtime({ + "path": "sub/../module.py", "mode": "str_replace", + "old_str": "VALUE = 1", "new_str": "VALUE = 42", + }, runtime) + assert result.startswith("Replaced"), result + assert target.read_text() == "VALUE = 42\n" + finally: + await environment.cleanup() + + +async def test_native_edit_uses_the_selected_physical_file_lock(tmp_path): + (tmp_path / "actual/child").mkdir(parents=True) + (tmp_path / "alias").symlink_to("actual/child", target_is_directory=True) + target = tmp_path / "actual/module.py" + target.write_text("VALUE = 1\n") + unrelated = tmp_path / "module.py" + unrelated.write_text("VALUE = 1\n") + environment = local_environment(tmp_path) + runtime = ToolRuntime(environment, SandboxInterceptor(str(tmp_path)), None) + editor = None + try: + async with host_write_lock(str(target), environment): + editor = asyncio.create_task(FileWriteTool().execute_with_runtime({ + "path": "alias/../module.py", "mode": "str_replace", + "old_str": "VALUE = 1", "new_str": "VALUE = 42", + }, runtime)) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(asyncio.shield(editor), 0.05) + assert target.read_text() == "VALUE = 1\n" + assert unrelated.read_text() == "VALUE = 1\n" + assert not (await asyncio.wait_for(editor, 2)).startswith("Error") + assert target.read_text() == "VALUE = 42\n" + assert unrelated.read_text() == "VALUE = 1\n" + finally: + if editor is not None and not editor.done(): + editor.cancel() + await asyncio.gather(editor, return_exceptions=True) + await environment.cleanup() + + +@pytest.mark.parametrize("component", ["missing", "regular-file"]) +def test_workspace_policy_checks_directory_before_parent_traversal(tmp_path, component): + target = tmp_path / "value.txt" + target.write_text("original\n") + if component == "regular-file": + (tmp_path / "component").write_text("regular\n") + policy = SandboxInterceptor(str(tmp_path)) + with pytest.raises(OSError): + policy.check_path("component/../value.txt") + assert target.read_text() == "original\n" + (tmp_path / "sub").mkdir() + assert policy.check_path("sub/../new/value.txt") == str(tmp_path / "new/value.txt") + + +def test_workspace_policy_root_resolves_symlink_before_parent(tmp_path): + (tmp_path / "actual/child").mkdir(parents=True) + (tmp_path / "alias").symlink_to("actual/child", target_is_directory=True) + policy = SandboxInterceptor(os.path.join(str(tmp_path), "alias/..")) + assert policy.check_path("module.py") == str(tmp_path / "actual/module.py") diff --git a/tests/interfaces/test_sdk_runtime.py b/tests/interfaces/test_sdk_runtime.py index b036af37..2b81bbba 100644 --- a/tests/interfaces/test_sdk_runtime.py +++ b/tests/interfaces/test_sdk_runtime.py @@ -8,6 +8,7 @@ import re import shlex import sys +from enum import Enum from pathlib import Path import pytest @@ -100,6 +101,27 @@ async def test_agent_uses_real_hardened_runtime_and_persists_evidence( transcript = json.loads((artifacts / "agent.json").read_text()) assert transcript["messages"][-1]["content"] == "finished" + +async def test_agent2_persists_string_enum_prompt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + class Prompt(str, Enum): + FINISH = "finish once" + + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + artifacts = tmp_path / "agent2-run" + result = await OpenCollab(tmp_path, model="model", provider="openai").agent2( + Prompt.FINISH, tools=(), llm=ReplyLLM(), artifacts=artifacts, trace=False, + ) + + assert result.status == "completed" + assert result.output == "finished" + path = artifacts / "agent.json" + snapshot = OpenCollab.read_session_snapshot(path) + assert any(row["content"] == Prompt.FINISH.value for row in snapshot["messages"] if row["role"] == "user") + assert snapshot["messages"][-1]["content"] == "finished" + assert path.with_suffix(".json.journal").read_bytes() == b"" + + async def test_client_resolves_config_once_and_delegates_plain_arguments( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/interfaces/test_sdk_workflow_import_cleanup.py b/tests/interfaces/test_sdk_workflow_import_cleanup.py new file mode 100644 index 00000000..7f57de67 --- /dev/null +++ b/tests/interfaces/test_sdk_workflow_import_cleanup.py @@ -0,0 +1,281 @@ +"""Loaded workflows retain relative imports through owned tool cleanup.""" + +from __future__ import annotations + +import asyncio +import json +import subprocess +import sys + +import pytest + +from opencollab import OpenCollab, RunError +from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.bootstrap import _workflow_runtime_discovery as discovery + + +def _import_state(): + return ( + {name for name in sys.modules if name.startswith("_opencollab_workflow_")}, + dict(discovery._WORKFLOW_IMPORT_FINDER._package_roots), + ) + + +class ResourceModel: + def context_window(self): + return 200_000 + + async def complete(self, messages, **kwargs): + return LLMResponse(tool_calls=[{ + "id": "resource", "type": "function", "function": { + "name": "resource", "arguments": json.dumps({}), + }, + }], usage=Usage(input_tokens=5, output_tokens=5), finish_reason="tool_calls") + + +def _loaded_workflow(tmp_path): + directory = tmp_path / "workflows" + directory.mkdir() + (directory / "_cleanup.py").write_text( + "def close_resource(events):\n events['closed'].set()\n", + encoding="utf-8", + ) + (directory / "resource.py").write_text('''import asyncio +from opencollab import workflow + +class Resource: + name = "resource" + description = "Hold a resource until cancellation." + parameters = {"type": "object", "properties": {}} + + def __init__(self, events): + self.events = events + + def to_openai_schema(self): + return {"type": "function", "function": { + "name": self.name, "description": self.description, "parameters": self.parameters, + }} + + async def execute_with_runtime(self, args, runtime): + self.events["entered"].set() + try: + await asyncio.Event().wait() + finally: + self.events["cleanup_started"].set() + while not self.events["release"].is_set(): + try: + await self.events["release"].wait() + except asyncio.CancelledError: + self.events["cancelled_again"].set() + from ._cleanup import close_resource + close_resource(self.events) + +@workflow +async def resource_cleanup(ctx, args): + events = args["events"] + events["context"] = ctx + timeout = 0.05 if args["mode"] == "timeout" else None + operation = asyncio.create_task(ctx.agent("Hold the resource", tools=[Resource(events)], timeout=timeout)) + await events["entered"].wait() + if args["mode"] == "recover": + operation.cancel() + try: + await operation + except asyncio.CancelledError: + if args["mode"] != "recover": + raise + events["returned"].set() + return "recovered" +''', encoding="utf-8") + return discovery.discover_workflows(str(directory)).get("resource_cleanup") + + +def _events(): + return {name: asyncio.Event() for name in ( + "entered", "cleanup_started", "release", "closed", "returned", "cancelled_again", + )} + + +@pytest.mark.parametrize("mode", ["recover", "timeout", "caller", "repeated"]) +async def test_loaded_workflow_keeps_imports_until_cancelled_tool_closes(tmp_path, mode): + before = _import_state() + spec = _loaded_workflow(tmp_path) + events = _events() + running = asyncio.create_task(OpenCollab(tmp_path, model="offline").workflow( + spec, {"events": events, "mode": mode}, llm=ResourceModel(), trace=False, + cleanup_timeout=2, + )) + try: + await asyncio.wait_for(events["entered"].wait(), 5) + if mode in {"caller", "repeated"}: + running.cancel() + else: + await asyncio.wait_for(events["returned"].wait(), 5) + await asyncio.wait_for(events["cleanup_started"].wait(), 5) + if mode == "repeated": + running.cancel() + await asyncio.sleep(0) + running.cancel() + await asyncio.sleep(0) + assert _import_state() != before + assert not events["closed"].is_set() + events["release"].set() + if mode in {"caller", "repeated"}: + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(running, 5) + else: + result = await asyncio.wait_for(running, 5) + assert result.status == "completed", result.reason + assert result.output == "recovered" + assert events["closed"].is_set() + assert _import_state() == before + finally: + events["release"].set() + if not running.done(): + running.cancel() + await asyncio.gather(running, return_exceptions=True) + + +async def test_cleanup_deadline_keeps_imports_for_lingering_owned_work(tmp_path): + before = _import_state() + spec = _loaded_workflow(tmp_path) + events = _events() + running = asyncio.create_task(OpenCollab(tmp_path, model="offline").workflow( + spec, {"events": events, "mode": "recover"}, llm=ResourceModel(), trace=False, + cleanup_timeout=0.01, + )) + try: + await asyncio.wait_for(events["returned"].wait(), 5) + with pytest.raises(RunError, match="without finalized execution evidence"): + await asyncio.wait_for(asyncio.shield(running), 5) + assert _import_state() != before + assert not events["closed"].is_set() + events["release"].set() + await asyncio.wait_for(events["closed"].wait(), 5) + await asyncio.wait_for(events["context"].wait_for_pending_cleanup(), 5) + assert _import_state() == before + finally: + events["release"].set() + if not running.done(): + running.cancel() + await asyncio.gather(running, return_exceptions=True) + + +async def test_concurrent_loaded_workflows_retain_each_others_cleanup_imports(tmp_path): + before = _import_state() + spec = _loaded_workflow(tmp_path) + client = OpenCollab(tmp_path, model="offline") + events = [_events(), _events()] + calls = [asyncio.create_task(client.workflow( + spec, {"events": signals, "mode": "recover"}, llm=ResourceModel(), trace=False, + )) for signals in events] + try: + await asyncio.wait_for(asyncio.gather(*(signals["returned"].wait() for signals in events)), 5) + events[0]["release"].set() + assert (await asyncio.wait_for(calls[0], 5)).status == "completed" + assert events[0]["closed"].is_set() + assert not events[1]["closed"].is_set() + assert _import_state() != before + events[1]["release"].set() + assert (await asyncio.wait_for(calls[1], 5)).status == "completed" + assert events[1]["closed"].is_set() + assert _import_state() == before + finally: + for signals in events: + signals["release"].set() + await asyncio.gather(*calls, return_exceptions=True) + + +async def test_candidate_child_context_retains_loaded_workflow_imports(tmp_path): + before = _import_state() + spec = _loaded_workflow(tmp_path) + workspace = tmp_path / "repo" + workspace.mkdir() + for arguments in ( + ("init", "-q"), + ("-c", "user.name=OpenCollab Tests", "-c", "user.email=tests@example.invalid", + "-c", "commit.gpgsign=false", "commit", "--allow-empty", "-qm", "test: create workspace"), + ): + subprocess.run(["git", "-C", str(workspace), *arguments], check=True, capture_output=True) + events = _events() + + async def parent(ctx, args): + return await ctx.candidate_workflow(spec.fn, args, label="resource-worker") + + running = asyncio.create_task(OpenCollab(workspace, model="offline").workflow( + parent, {"events": events, "mode": "recover"}, llm=ResourceModel(), trace=False, + cleanup_timeout=5, + )) + try: + await asyncio.wait_for(events["returned"].wait(), 15) + assert _import_state() != before + events["release"].set() + result = await asyncio.wait_for(running, 15) + assert result.status == "completed", result.reason + assert result.output.output == "recovered" + assert events["closed"].is_set() + assert _import_state() == before + finally: + events["release"].set() + if not running.done(): + running.cancel() + await asyncio.gather(running, return_exceptions=True) + + +async def test_named_workflow_keeps_imports_for_followup_dispatched_by_completion_callback(tmp_path): + directory = tmp_path / "workflows" + directory.mkdir() + (directory / "_cleanup.py").write_text("def close(events):\n events['closed'] = True\n", encoding="utf-8") + (directory / "callback.py").write_text('''import asyncio +from opencollab import workflow + +class CloseResource: + name = "resource" + description = "Close the resource managed by this workflow" + parameters = {"type": "object", "properties": {}} + + def __init__(self, events): + self.events = events + + def to_openai_schema(self): + return {"type": "function", "function": { + "name": self.name, "description": self.description, "parameters": self.parameters, + }} + + async def execute_with_runtime(self, args, runtime): + from ._cleanup import close + close(self.events) + return "closed" + +@workflow +async def callback_cleanup(ctx, args): + events = args["events"] + def start_followup(): + events["tasks"].append(asyncio.create_task(ctx.agent("finish", tools=[CloseResource(events)]))) + + first = asyncio.create_task(ctx.agent("start")) + first.add_done_callback(lambda _: asyncio.get_running_loop().call_soon(start_followup)) + await first + return "returned" +''', encoding="utf-8") + + class Model(ResourceModel): + async def complete(self, messages, **kwargs): + if any(m["role"] == "user" and "finish" in str(m["content"]) for m in messages) and not any( + m["role"] == "tool" for m in messages + ): + return await super().complete(messages, **kwargs) + return LLMResponse(content="done", usage=Usage(input_tokens=5, output_tokens=5), finish_reason="end_turn") + + events = {"closed": False, "tasks": []} + before = _import_state() + result = await OpenCollab(tmp_path, model="offline").workflow( + "callback_cleanup", {"events": events}, llm=Model(), trace=False, cleanup_timeout=2, + ) + assert result.status == "completed", result.reason + assert result.output == "returned" + assert len(events["tasks"]) == 1 + followup, = events["tasks"] + assert followup.result() == "done" + assert events["closed"] + assert _import_state() == before diff --git a/tests/runtime/test_anthropic_history_edits.py b/tests/runtime/test_anthropic_history_edits.py new file mode 100644 index 00000000..82be42af --- /dev/null +++ b/tests/runtime/test_anthropic_history_edits.py @@ -0,0 +1,150 @@ +import json +from copy import deepcopy + +import pytest + +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm.client import LLMClient +from opencollab.application._anthropic_history import _anthropic_history_input, _remember_anthropic_history +from opencollab.application.shaping import AutoCompactShaper +from opencollab.bootstrap import build_session, load_session +from opencollab.bootstrap.context_policy import ContextPolicy +from opencollab.domain.agent import Agent +from opencollab.domain.session import SessionState +from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport + + +@pytest.mark.parametrize("save_after_edit", [False, True], ids=["edit-after-restore", "edit-before-save"]) +async def test_edit_of_repeated_turn_keeps_saved_summary(monkeypatch, tmp_path, save_after_edit): + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + document = "EARLY DOCUMENT " + ("Prior grant application details. " * 650) + summary = "The grant application was acknowledged." + correction = "Continue using Friday as the revised deadline." + requests = [] + summaries = [] + + def summarize(messages): + summaries.append(messages) + return summary + + def peer(request): + requests.append(json.loads(request.content)) + if len(requests) > 3 and len(request.content) > 8_000: + return http_response(request, 400, json={ + "type": "error", + "error": {"type": "invalid_request_error", "message": "prompt is too long for the context window"}, + }) + return completion_http_response(request, text="Acknowledged") + + install_sdk_transport(monkeypatch, "anthropic", peer) + model = "claude-sonnet-5-5" + agent = Agent( + name="reviewer", system_prompt="Review the grant application.", model=model, provider="anthropic", + ) + env = LocalEnvironment(str(tmp_path)) + snapshot = str(tmp_path / "session.json") + async with LLMClient( + model=model, provider="anthropic", api_key="test-placeholder", # pragma: allowlist secret + base_url="https://provider.invalid/v1", max_retries=0, + ) as client: + settings = {"agent": agent, "llm": client, "env": env, "max_budget_tokens": 100_000} + session = build_session( + **settings, + shaper=AutoCompactShaper( + trigger_tokens=600, target_tokens=300, keep_recent_groups=3, summarizer=summarize, + estimate_tokens=lambda rows: sum(len(str(row["content"])) for row in rows) // 4, + ), + ) + try: + for prompt in (document, "Continue", "Continue"): + await session.add_user_message(prompt) + assert await session.run_loop() == "Acknowledged" + assert summary in json.dumps(requests[-1]["messages"]) + assert document in json.dumps(session.messages) + if save_after_edit: + session.messages[5]["content"] = correction + session.save(snapshot) + await session.aclose() + session = load_session(snapshot, **settings, context_policy=ContextPolicy(name="no_history_compaction")) + if not save_after_edit: + session.messages[5]["content"] = correction + await session.add_user_message("What should we do next?") + assert await session.run_loop() == "Acknowledged" + assert len(requests) == 4 + assert len(summaries) == 1 + assert session.messages[5]["content"] == correction + wire = json.dumps(requests[-1]["messages"]) + assert summary in wire + assert correction in wire + assert "EARLY DOCUMENT" not in wire + finally: + await session.aclose() + await env.cleanup() + + +@pytest.mark.parametrize("edit", ["insert-outside-summary", "insert-inside-summary", "delete-inside-summary"]) +def test_disjoint_edits_keep_independent_summaries_and_compacted_tools(edit): + system = {"role": "system", "content": "Review the two grant applications."} + first_document = {"role": "user", "content": "First application"} + first_reply = {"role": "assistant", "content": "First review"} + first_summary = {"role": "system", "compacted": True, "content": "Summary of the first application"} + bridge = {"role": "user", "content": "Read the archive for context"} + call = {"role": "assistant", "content": None, "tool_calls": [ + {"id": "read-archive", "type": "function", "function": {"name": "file_read", "arguments": "{}"}}, + ]} + result = {"role": "tool", "tool_call_id": "read-archive", "content": "Archive data"} + cleared = {**result, "content": "[Old tool result content cleared]"} + second_document = {"role": "user", "content": "Second application"} + second_reply = {"role": "assistant", "content": "Second review"} + second_summary = {"role": "system", "compacted": True, "content": "Summary of the second application"} + repeat = {"role": "user", "content": "Continue"} + acknowledgement = {"role": "assistant", "content": "Acknowledged"} + state = SessionState(messages=[ + system, first_document, first_reply, bridge, call, result, + second_document, second_reply, repeat, acknowledgement, repeat, + ]) + _remember_anthropic_history(state, [ + system, first_summary, bridge, call, cleared, second_summary, repeat, acknowledgement, repeat, + ], None) + + revised_bridge = {"role": "user", "content": "Check the archive's dates"} + revised_tail = {"role": "user", "content": "Continue with the new dates"} + insertion = {"role": "user", "content": "Check the updated budget"} + state.messages[3] = revised_bridge + state.messages[-1] = revised_tail + if edit == "insert-outside-summary": + state.messages.insert(8, insertion) + remaining = [second_summary, insertion] + elif edit == "insert-inside-summary": + state.messages.insert(7, insertion) + remaining = [second_document, insertion, second_reply] + else: + del state.messages[7] + remaining = [second_document] + assert _anthropic_history_input(state) == [ + system, first_summary, revised_bridge, call, cleared, *remaining, repeat, acknowledgement, revised_tail, + ] + + +@pytest.mark.parametrize("edited_index", [1, -1], ids=["earlier-edit", "later-edit"]) +def test_reused_tool_ids_keep_a_compacted_exchange_in_an_unchanged_end(edited_index): + call = {"role": "assistant", "content": None, "tool_calls": [ + {"id": "read-archive", "type": "function", "function": {"name": "file_read", "arguments": "{}"}}, + ]} + result = {"role": "tool", "tool_call_id": "read-archive", "content": "First archive version"} + state = SessionState(messages=[ + {"role": "system", "content": "Review the grant application."}, + {"role": "user", "content": "Read the original archive"}, + call, result, + {"role": "user", "content": "Read the updated archive"}, + deepcopy(call), {**result, "content": "New archive version"}, + {"role": "user", "content": "Compare the dates"}, + ]) + view = deepcopy(state.messages) + view[3]["content"] = "[Old tool result content cleared]" + _remember_anthropic_history(state, view, None) + state.messages[edited_index]["content"] = "Look up the deadline" + view[edited_index]["content"] = "Look up the deadline" + assert _anthropic_history_input(state) == view diff --git a/tests/runtime/test_anthropic_preserved_history.py b/tests/runtime/test_anthropic_preserved_history.py index c39d4771..fa7822b6 100644 --- a/tests/runtime/test_anthropic_preserved_history.py +++ b/tests/runtime/test_anthropic_preserved_history.py @@ -2,19 +2,27 @@ from __future__ import annotations +import asyncio import copy import json +import shutil +import threading import pytest from opencollab import OpenCollab +from opencollab.adapters import storage as storage_module from opencollab.adapters.env import LocalEnvironment from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.tools.fs import FileReadTool, FileWriteTool +from opencollab.application._anthropic_history import _anthropic_history_input, _remember_anthropic_history +from opencollab.application.session_snapshot import _restore_anthropic_history from opencollab.application.shaping import AutoCompactShaper, PerToolResultBudgetShaper, ToolOutputClearShaper from opencollab.application.steering import READS_NUDGE_HARD from opencollab.bootstrap import build_session, load_session from opencollab.domain.agent import Agent +from opencollab.domain.session import SessionState +from tests.runtime.test_manual_autosave_ordering import PausedAppendStore from tests.support.provider_sdk_http import http_response, install_sdk_transport _THINKING = {"thinking", "redacted_thinking"} @@ -45,10 +53,11 @@ def _signatures(body): class _BoundThinkingProvider: """Compare the documented prefixes through the real Anthropic SDK wire.""" - def __init__(self, reads=2, *, thinking=True, correction=False): + def __init__(self, reads=2, *, thinking=True, correction=False, read_input=None): self.reads = reads self.thinking = thinking self.correction = correction + self.read_input = read_input or {} self.requests = [] self.origins = {} self.retired = set() @@ -89,7 +98,7 @@ def __call__(self, request): ] if number <= self.reads: content.append({"type": "tool_use", "id": f"read-{number}", "name": "file_read", - "input": {"path": "source.txt"}}) + "input": {"path": "source.txt", **self.read_input}}) elif self.correction and [tool["name"] for tool in body.get("tools", [])] == ["structured_output"]: content.append({"type": "tool_use", "id": "capture", "name": "structured_output", "input": {"answer": 42}}) else: @@ -158,7 +167,10 @@ async def test_native_steering_prefix_survives_tool_rounds_and_new_user_turns(mo assert _signatures(peer.requests[2]) == ["signed-1", "redacted-1", "signed-2", "redacted-2"] @pytest.mark.parametrize("mode", ["clear", "summary", "first-trim"]) -async def test_compacted_native_view_survives_restore_and_disabling_the_shaper(monkeypatch, tmp_path, mode): +@pytest.mark.parametrize("snapshot_format", ["current", "legacy", "source-copy"]) +async def test_compacted_native_view_survives_restore_and_disabling_the_shaper( + monkeypatch, tmp_path, mode, snapshot_format, +): monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") peer = _BoundThinkingProvider(reads=4) install_sdk_transport(monkeypatch, "anthropic", peer) @@ -179,10 +191,21 @@ async def test_compacted_native_view_survives_restore_and_disabling_the_shaper(m assert len([message for message in raw if message["role"] == "tool"]) == 4 assert all("file evidence" in message["content"] for message in raw if message["role"] == "tool") before = copy.deepcopy(session.state.anthropic_history) - session.save(str(tmp_path / "session.json")) + path = tmp_path / "session.json" + session.save(str(path)) + saved = json.loads(path.read_text()) + native_view = saved["session_state"]["anthropic_history"] + assert "source_messages" not in native_view + assert native_view["messages"][0] == 0 + if snapshot_format != "current": + legacy = dict(before) + if snapshot_format == "legacy": + del legacy["source_messages"] + saved["session_state"]["anthropic_history"] = legacy + path.write_text(json.dumps(saved)) await session.aclose() await session.env.cleanup() - session = load_session(str(tmp_path / "session.json"), agent=_agent(), llm=client, + session = load_session(str(path), agent=_agent(), llm=client, env=LocalEnvironment(str(tmp_path)), max_budget_tokens=100_000, max_steps=20) assert session.state.anthropic_history == before assert session.messages == raw @@ -200,6 +223,162 @@ async def test_compacted_native_view_survives_restore_and_disabling_the_shaper(m assert peer.retired assert _signatures(peer.requests[-1])[-2:] == ["signed-5", "redacted-5"] + +@pytest.mark.parametrize("mode", ["projected", "journal"]) +async def test_native_snapshot_preserves_typed_tool_input(monkeypatch, tmp_path, mode): + peer = _BoundThinkingProvider(reads=1, read_input={"marker": 1}) + install_sdk_transport(monkeypatch, "anthropic", peer) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + + def read_input(blocks): + return next(block["input"] for block in blocks if block["type"] == "tool_use") + + def native_input(message): + return read_input(message["provider_state"]["anthropic_content"]) + + def sent_input(request): + return read_input([block for message in request["messages"] for block in _blocks(message["content"])]) + + class TypedProjection: + def shape(self, messages): + projected = copy.deepcopy(messages) + if len(projected) > 2: + native_input(projected[2])["marker"] = True + return projected + + path = tmp_path / "typed.json" + environment = LocalEnvironment(str(tmp_path)) + async with _client() as client: + session = build_session( + agent=_agent(), llm=client, env=environment, + shaper=TypedProjection() if mode == "projected" else None, + auto_save_path=str(path) if mode == "journal" else None, + ) + restored = None + try: + await session.add_user_message("Read the file") + assert await session.run_loop() == "done" + session.save(str(path)) + if mode == "projected": + assert sent_input(peer.requests[-1])["marker"] is True + assert type(native_input(session.messages[2])["marker"]) is int + else: + if session._auto_save_sequence + 1 == session._next_auto_save_checkpoint: + await session.enqueue_auto_save() + native_input(session.messages[2])["marker"] = True + await session.enqueue_auto_save() + journal = path.with_suffix(".json.journal") + delta = json.loads(journal.read_text().splitlines()[-1]) + assert delta["replace_from"] == 2 + assert native_input(delta["messages"][0])["marker"] is True + + restored = load_session(str(path), agent=_agent(), llm=client, env=environment) + await restored.add_user_message("Continue with the recorded tool use") + assert await restored.run_loop() == "done" + assert sent_input(peer.requests[-1])["marker"] is True + assert "signed-1" not in _signatures(peer.requests[-1]) + finally: + if restored is not None: + await restored.aclose() + await session.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize("moment", ["live", "before-export", "after-restore"]) +@pytest.mark.parametrize("target", ["recent-user", "first-user", "both-users", "insert", "delete", "cleared-result"]) +async def test_source_edits_keep_unrelated_native_tool_compaction(monkeypatch, tmp_path, moment, target): + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + marker = "Catalogue entry from the old archive." + clear_marker = "[Old tool result content cleared]" + archive = tmp_path / "archive.txt" + archive.write_text(f"{marker}\n" * 400) + peer = _BoundThinkingProvider(reads=2, thinking=False) + context_limit = 8_000 + resumed_requests = [] + resuming = False + + def transport(request): + if resuming: + resumed_requests.append(json.loads(request.content)) + if len(request.content) > context_limit: + return http_response(request, 400, json={"type": "error", "error": { + "type": "invalid_request_error", "message": "prompt is too long for the context window", + }}) + peer.read_input = {"path": archive.name, "limit": 400} if not peer.requests else {} + return peer(request) + + install_sdk_transport(monkeypatch, "anthropic", transport) + first_user = "Inspect the archive" + recent_user = "Use a formal tone" + first_edit = (first_user, "Review the archive entries") + recent_edit = (recent_user, "Use an informal tone") + inserted = "Additional context from the editor" + edits = { + "recent-user": [recent_edit], + "first-user": [first_edit], + "both-users": [first_edit, recent_edit], + "insert": [(None, inserted)], + "delete": [(first_user, None)], + "cleared-result": [(marker, "The entries in the old archive have been revised.")], + }[target] + + def edit(session): + if target == "cleared-result": + result = next(message for message in session.messages if message["role"] == "tool") + assert marker in result["content"] + result["content"] = edits[0][1] + elif target == "insert": + session.messages.insert(1, {"role": "user", "content": inserted}) + elif target == "delete": + del session.messages[1] + else: + for original, revised in edits: + message = next(message for message in session.messages if message.get("content") == original) + message["content"] = revised + + model = "claude-sonnet-5-5" + env = LocalEnvironment(str(tmp_path)) + path = str(tmp_path / "compacted.json") + async with _client(model) as client: + options = {"agent": _agent(thinking=False, model=model), "llm": client, "env": env} + session = build_session( + **options, shaper=ToolOutputClearShaper(trigger_tokens=2, target_tokens=1, keep_recent=1), + ) + try: + await session.add_user_message(first_user) + assert await session.run_loop() == "done" + await session.add_user_message(recent_user) + assert await session.run_loop() == "done" + assert marker in json.dumps(peer.requests[1]["messages"]) + assert clear_marker in json.dumps(peer.requests[-1]["messages"]) + assert marker in json.dumps(session.messages) + if moment != "after-restore": + edit(session) + if moment != "live": + session.save(path) + await session.aclose() + session = load_session(path, **options) + if moment == "after-restore": + edit(session) + session.runner.shaper = None + + await session.add_user_message("Summarize the evidence") + resuming = True + assert await session.run_loop() == "done" + assert len(resumed_requests) == 1 + continued = json.dumps(resumed_requests[0]["messages"]) + for original, revised in edits: + if original is not None: + assert original not in continued + if revised is not None: + assert revised in continued + assert marker not in continued + assert (clear_marker in continued) is (target != "cleared-result") + finally: + await session.aclose() + await env.cleanup() + + async def test_appended_system_and_changed_tools_retire_old_native_thinking(monkeypatch, tmp_path): monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") peer = _BoundThinkingProvider(reads=0) @@ -333,3 +512,802 @@ async def test_prefix_constraint_uses_native_transport_and_documented_models(mod async with LLMClient(model=model, provider=provider, api_key="controlled-test", # pragma: allowlist secret base_url="https://controlled.invalid/v1") as client: assert client.requires_thinking_prefix is expected + + +@pytest.mark.parametrize("moment", ["live", "before-save", "before-autosave", "after-load"]) +@pytest.mark.parametrize("edit", ["text", "nested", "replacement", "deletion", "tail-deletion", "system"]) +async def test_source_edits_reach_native_requests_and_retire_only_invalid_thinking( + monkeypatch, tmp_path, moment, edit, +): + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + peer = _BoundThinkingProvider(reads=0) + install_sdk_transport(monkeypatch, "anthropic", peer) + path = tmp_path / "session.json" + + def apply_edit(session): + if edit == "text": + session.messages[3]["content"] = "Choose green" + elif edit == "nested": + session.messages[3]["content"][0]["text"] = "Choose green" + elif edit == "replacement": + session.messages[3] = {"role": "user", "content": "Choose green"} + elif edit == "deletion": + del session.messages[3] + elif edit == "tail-deletion": + del session.messages[3:] + else: + session.messages[0]["content"] = "Follow the revised instructions" + + async with _client() as client: + session = build_session(agent=_agent(), llm=client, env=LocalEnvironment(str(tmp_path)), + max_budget_tokens=100_000, max_steps=20, + auto_save_path=str(path) if moment == "before-autosave" else None) + try: + await session.add_user_message("Answer the first question") + assert await session.run_loop() == "done" + await session.add_user_message("Choose red") + session.messages[-1]["content"] = [{"type": "text", "text": "Choose red"}] + assert await session.run_loop() == "done" + if moment == "before-autosave": + session.save(str(path)) + if moment != "after-load": + apply_edit(session) + if moment != "live": + if moment == "before-autosave": + edited = copy.deepcopy(session.messages) + await session.enqueue_auto_save() + assert not session.persistence_errors + else: + session.save(str(path)) + await session.aclose() + await session.env.cleanup() + session = load_session(str(path), agent=_agent(), llm=client, + env=LocalEnvironment(str(tmp_path)), max_budget_tokens=100_000, max_steps=20) + if moment == "before-autosave": + assert session.messages == edited + if moment == "after-load": + apply_edit(session) + edited = copy.deepcopy(session.messages) + await session.add_user_message("Continue using the current instructions") + assert await session.run_loop() == "done" + assert session.messages[:len(edited)] == edited + await session.add_user_message("Continue once more") + assert await session.run_loop() == "done" + finally: + await session.aclose() + await session.env.cleanup() + + assert len(peer.requests) == 4 + body = peer.requests[2] + if edit == "system": + assert body["system"] == "Follow the revised instructions" + assert _signatures(body) == [] + else: + assert _signatures(body) == ["signed-1", "redacted-1"] + assert "Choose red" not in json.dumps(body) + if edit not in {"deletion", "tail-deletion"}: + assert "Choose green" in json.dumps(body) + assert _signatures(peer.requests[-1]) == [*_signatures(body), "signed-3", "redacted-3"] + + +@pytest.mark.parametrize("legacy", [False, True]) +def test_restored_unchanged_prefix_preserves_projection_and_tracks_future_nested_edits(legacy): + state = SessionState(messages=[ + {"role": "system", "content": "Identity"}, + {"role": "user", "content": [{"type": "text", "text": "Old requirement"}]}, + {"role": "assistant", "content": "Large old result"}, + ]) + view = [state.messages[0], {"role": "system", "compacted": True, "content": "Earlier context"}] + assert _remember_anthropic_history(state, view, None) == view + stored = copy.deepcopy(state.anthropic_history) + if legacy: + del stored["source_messages"] + state.anthropic_history = _restore_anthropic_history(stored, state.messages) + assert _anthropic_history_input(state) == view + state.append_message({"role": "user", "content": "New request"}) + assert _anthropic_history_input(state) == [*view, state.messages[-1]] + assert "Large old result" not in json.dumps(_anthropic_history_input(state)) + state.messages[1]["content"][0]["text"] = "Revised requirement" + assert _anthropic_history_input(state) == state.messages + assert state.anthropic_history["source_messages"][1]["content"][0]["text"] == "Old requirement" + + +def test_edit_in_summary_invalidates_it_even_when_a_later_message_is_identical(): + state = SessionState(messages=[ + {"role": "system", "content": "System instructions"}, + {"role": "user", "content": "First request"}, + {"role": "assistant", "content": "Acknowledged"}, + {"role": "user", "content": "Second request"}, + {"role": "assistant", "content": "Acknowledged"}, + ]) + summary = {"role": "system", "compacted": True, "content": "The first request was acknowledged"} + _remember_anthropic_history(state, [state.messages[0], summary, state.messages[4]], None) + state.messages[2]["content"] = "The first request still needs a decision" + assert _anthropic_history_input(state) == state.messages + + +def test_disjoint_edits_do_not_reuse_a_projection_for_reused_tool_call_ids(): + call = { + "role": "assistant", "content": None, "tool_calls": [{ + "id": "read-again", "type": "function", + "function": {"name": "file_read", "arguments": '{"path":"archive.txt"}'}, + }], + } + state = SessionState(messages=[ + {"role": "user", "content": "Read the old archive"}, + call, {"role": "tool", "tool_call_id": "read-again", "content": "Old file data"}, + {"role": "user", "content": "Check the archive again"}, + copy.deepcopy(call), {"role": "tool", "tool_call_id": "read-again", "content": "Revised file data"}, + {"role": "user", "content": "Summarize what changed"}, + ]) + view = copy.deepcopy(state.messages) + view[2]["content"] = "[Old tool result content cleared]" + _remember_anthropic_history(state, view, None) + state.messages[0]["content"] = "Read the archive entries" + state.messages[-1]["content"] = "Summarize the latest entries" + assert _anthropic_history_input(state) == state.messages + + +@pytest.mark.parametrize("checkpoint", [False, True], ids=["journal", "checkpoint"]) +async def test_first_tool_step_autosaves_an_edit_already_used_in_the_request(monkeypatch, tmp_path, checkpoint): + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + peer = _BoundThinkingProvider(reads=0) + held_request, resume = asyncio.Event(), asyncio.Event() + hold_after_read = False + + async def provider(request): + if hold_after_read and len(peer.requests) == peer.reads: + held_request.set() + await resume.wait() + return peer(request) + + install_sdk_transport(monkeypatch, "anthropic", provider) + path = tmp_path / "session.json" + + async with _client() as client: + environment = LocalEnvironment(str(tmp_path)) + session = build_session(agent=_agent(), llm=client, env=environment, + auto_save_path=str(path), max_budget_tokens=100_000, max_steps=20) + running = restored = None + try: + for index in range(3): + await session.add_user_message(f"Choose red for turn {index}") + assert await session.run_loop() == "done" + await session.add_user_message("Read the current evidence") + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + until_checkpoint = session._next_auto_save_checkpoint - session._auto_save_sequence + extra_saves = until_checkpoint - 1 if checkpoint else int(until_checkpoint == 1) + for _ in range(extra_saves): + await session.enqueue_auto_save() + saved = session.store.load_snapshot(str(path), session.agent.system_prompt) + assert saved["messages"][1]["content"] == "Choose red for turn 0" + + session.messages[1]["content"] = "Choose green for turn 0" + peer.reads = len(peer.requests) + 1 + hold_after_read = True + running = asyncio.create_task(session.run_loop()) + await asyncio.wait_for(held_request.wait(), 5) + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + assert not running.done() + assert "Choose green for turn 0" in json.dumps(peer.requests[-1]) + assert not session.persistence_errors + + saved = session.store.load_snapshot(str(path), session.agent.system_prompt) + base = json.loads(path.read_text()) + assert (base["_autosave_sequence"] == saved["_autosave_sequence"]) is checkpoint + restored = load_session(str(path), agent=_agent(), llm=client, env=environment) + assert restored.messages[1]["content"] == "Choose green for turn 0" + assert restored.state.anthropic_history["messages"][1]["content"] == "Choose green for turn 0" + assert sum(message["role"] == "tool" for message in restored.messages) == 1 + + resume.set() + assert await running == "done" + finally: + resume.set() + if running is not None: + await asyncio.gather(running, return_exceptions=True) + if restored is not None: + await restored.aclose() + await session.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize(("first_save", "revert"), [ + ("journal", True), ("manual", True), ("periodic", True), ("manual-after-freeze", True), + ("journal", False), +]) +async def test_native_history_delta_uses_the_last_saved_transcript(monkeypatch, tmp_path, first_save, revert): + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + peer = _BoundThinkingProvider(reads=0) + install_sdk_transport(monkeypatch, "anthropic", peer) + path = tmp_path / "session.json" + journal = path.with_suffix(".json.journal") + resumed_path = tmp_path / "resumed.json" + red, green = "Use the red option.", "Use the green option." + store = PausedAppendStore() + paused_append = first_save == "manual-after-freeze" + + async with _client() as client: + environment = LocalEnvironment(str(tmp_path)) + session = build_session( + agent=_agent(), llm=client, env=environment, store=store, auto_save_path=str(path), + seed_user_messages=[{"role": "user", "content": _blocks(red)}], + ) + restored = None + try: + for index in range(3): + await session.add_user_message(f"Warmup {index}") + assert await session.run_loop() == "done" + until_checkpoint = session._next_auto_save_checkpoint - session._auto_save_sequence + if first_save == "periodic": + for _ in range(until_checkpoint - 1): + await session.enqueue_auto_save() + elif until_checkpoint <= (3 if paused_append else 2): + for _ in range(until_checkpoint): + await session.enqueue_auto_save() + if paused_append: + store.pause_next = True + session.enqueue_auto_save() + assert await asyncio.to_thread(store.started.wait, 1) + + source = session.messages[1]["content"][0] + source["text"] = green + if first_save.startswith("manual"): + session.save(str(path)) + else: + await session.enqueue_auto_save() + saved_edit = session.store.load_snapshot(str(path), session.agent.system_prompt) + assert saved_edit["messages"][1]["content"] == _blocks(green) + base_sequence = json.loads(path.read_text())["_autosave_sequence"] + assert (base_sequence == saved_edit["_autosave_sequence"]) is (first_save != "journal") + + expected = red if revert else green + source["text"] = expected + await session.add_user_message("Continue with the current instructions.") + store.release.set() + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + delta = json.loads(journal.read_text().splitlines()[-1]) + assert delta["replace_from"] == (1 if revert else len(session.messages) - 2) + native_view = delta["meta"]["session_state"]["anthropic_history"] + assert native_view["messages"][1] == 1 + assert "source_messages" not in native_view + + shutil.copyfile(path, resumed_path) + shutil.copyfile(journal, resumed_path.with_suffix(".json.journal")) + restored = load_session(str(resumed_path), agent=_agent(), llm=client, env=environment, + auto_save_path=str(resumed_path)) + assert restored.messages == session.messages + assert restored.state.anthropic_history["messages"][1]["content"] == _blocks(expected) + assert await restored.run_loop() == "done" + assert await session.run_loop() == "done" + assert _plain_messages(peer.requests[-2]["messages"]) == _plain_messages(peer.requests[-1]["messages"]) + assert expected in json.dumps(peer.requests[-1]["messages"]) + assert not session.persistence_errors and not restored.persistence_errors + finally: + store.release.set() + if restored is not None: + await restored.aclose() + await session.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize("previous_save", ["journal", "reopen-journal", "manual-covers-append"]) +async def test_autosave_native_view_after_large_transcript_replacement(monkeypatch, tmp_path, previous_save): + monkeypatch.setattr(storage_module, "MAX_SESSION_SNAPSHOT_BYTES", 256 * 1024) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + archived = "A" * (140 * 1024) + path = tmp_path / "session.json" + journal = path.with_suffix(".json.journal") + held_request, resume = asyncio.Event(), asyncio.Event() + peer = _BoundThinkingProvider(reads=0, thinking=False) + store = PausedAppendStore() + + class ArchiveShaper: + def shape(self, messages): + return [ + {**message, "content": "[Archived context]"} if message.get("content") == archived else message + for message in messages + ] + + async def provider(request): + response = peer(request) + if len(peer.requests) == 3: + held_request.set() + await resume.wait() + return response + + install_sdk_transport(monkeypatch, "anthropic", provider) + model = "claude-sonnet-5-5" + async with _client(model) as client: + environment = LocalEnvironment(str(tmp_path)) + options = { + "agent": _agent(thinking=False, model=model), "llm": client, "env": environment, + "auto_save_path": str(path), "shaper": ArchiveShaper(), "store": store, + } + session = build_session(**options) + running = None + try: + await session.add_user_message("Warm up the conversation") + assert await session.run_loop() == "done" + assert session.state.anthropic_history is not None + while session._next_auto_save_checkpoint - session._auto_save_sequence <= 3: + await session.enqueue_auto_save() + session.messages = [*session.messages, {"role": "user", "content": archived}] + store.pause_next = previous_save == "manual-covers-append" + await session.add_user_message("Read source.txt before answering") + if previous_save == "manual-covers-append": + assert await asyncio.to_thread(store.started.wait, 1) + session.save(str(path)) + store.release.set() + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + before_bytes = journal.stat().st_size + before = session.store.load_snapshot(str(path), session.agent.system_prompt) + assert before_bytes > len(archived) + assert before["session_state"]["anthropic_history"] is None + assert before["messages"][-2]["content"] == archived + + if previous_save == "reopen-journal": + await session.aclose() + session = load_session(str(path), **options) + peer.reads = 2 + running = asyncio.create_task(session.run_loop()) + await asyncio.wait_for(held_request.wait(), 5) + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + assert not running.done() + assert "file evidence" in json.dumps(peer.requests[-1]["messages"]) + assert not session.persistence_errors + + saved = session.store.load_snapshot(str(path), session.agent.system_prompt) + assert saved["_autosave_sequence"] > before["_autosave_sequence"] + assert any("file evidence" in message["content"] for message in saved["messages"] + if message["role"] == "tool") + history = saved["session_state"]["anthropic_history"] + assert "source_messages" not in history + assert any(type(message) is int for message in history["messages"]) + assert journal.stat().st_size < before_bytes + 8 * 1024 + + resume.set() + assert await running == "done" + finally: + resume.set() + store.release.set() + if running is not None: + await asyncio.gather(running, return_exceptions=True) + await session.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize("failure", ["capacity", "format", "before-publish", "after-publish"]) +async def test_native_rewrite_recovers_tool_result_when_checkpoint_fails(monkeypatch, tmp_path, failure): + limit = 128 * 1024 + monkeypatch.setattr(storage_module, "MAX_SESSION_SNAPSHOT_BYTES", limit) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + size = (limit - 14_000) // 2 if failure in {"capacity", "format"} else 1_000 + archive = _blocks("\u00e9" * size) + _blocks("x") * 256 + path = tmp_path / "fallback.json" + journal = path.with_suffix(".json.journal") + held_request, resume = asyncio.Event(), asyncio.Event() + peer = _BoundThinkingProvider(reads=0, thinking=False) + + class ArchiveShaper: + def shape(self, messages): + return [ + {**message, "content": "[Archived context]"} if message.get("content") == archive else message + for message in messages + ] + + async def provider(request): + response = peer(request) + if len(peer.requests) == 3: + held_request.set() + await resume.wait() + return response + + install_sdk_transport(monkeypatch, "anthropic", provider) + model = "claude-sonnet-5-5" + environment = LocalEnvironment(str(tmp_path)) + async with _client(model) as client: + agent = _agent(thinking=False, model=model) + session = build_session(agent=agent, llm=client, env=environment, + auto_save_path=str(path), shaper=ArchiveShaper()) + running = restored = None + try: + if failure == "capacity": + session.messages = [*session.messages, { + "role": "user", "content": _blocks("[Archived prefix] " + "p" * 20_000), + }] + await session.add_user_message("Warm up the conversation") + assert await session.run_loop() == "done" + await session.add_user_message("Read source.txt before answering") + await asyncio.gather(*session.pending_cleanup_tasks) + session.save(str(path)) + base = path.read_bytes() + sequence = json.loads(base)["_autosave_sequence"] + 1 + session.messages = [*session.messages, {"role": "user", "content": archive}] + assert session.state.anthropic_history is None + + checkpoint_error = OSError(f"controlled checkpoint failure {failure}") + original_write = session.store._atomic_json_write + original_compact = storage_module.write_regular_bytes_atomic + + def write_snapshot(target, document, **options): + if failure == "before-publish" and document["_autosave_sequence"] == sequence: + raise checkpoint_error + original_write(target, document, **options) + + def compact_journal(target, payload, **options): + if failure == "after-publish": + raise checkpoint_error + original_compact(target, payload, **options) + + peer.reads = 2 + with monkeypatch.context() as fault: + fault.setattr(session.store, "_atomic_json_write", write_snapshot) + fault.setattr(storage_module, "write_regular_bytes_atomic", compact_journal) + running = asyncio.create_task(session.run_loop()) + await asyncio.wait_for(held_request.wait(), 5) + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + + assert not running.done() + saved = session.store.load_snapshot(str(path), agent.system_prompt) + assert saved["_autosave_sequence"] == sequence + assert any(message["content"] == archive for message in saved["messages"]) + restored = load_session(str(path), agent=agent, llm=client, env=environment) + assert restored.state.anthropic_history is not None + assert [row["content"] for row in restored.messages if row["role"] == "tool"] == [ + row["content"] for row in session.messages if row["role"] == "tool" + ] + assert sum(row["role"] == "tool" for row in restored.messages) == 1 + if failure == "after-publish": + assert session.persistence_errors == (checkpoint_error,) + assert json.loads(path.read_bytes())["_autosave_sequence"] == sequence + assert journal.read_bytes() == b"" + elif failure == "format": + assert not session.persistence_errors + assert len(json.dumps(saved, ensure_ascii=False, indent=2).encode("utf-8")) > limit + assert path.stat().st_size < limit + assert json.loads(path.read_bytes())["_autosave_sequence"] == sequence + assert journal.read_bytes() == b"" + else: + assert not session.persistence_errors + record = json.loads(journal.read_bytes()) + assert record["sequence"] == sequence + assert record["replace_from"] > 0 + assert path.read_bytes() == base + if failure == "capacity": + assert len(journal.read_bytes()) < limit + assert len(json.dumps(saved, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) > limit + + resume.set() + if failure == "capacity": + with pytest.raises(ValueError, match="session snapshot exceeds"): + await running + else: + assert await running == "done" + finally: + resume.set() + if running is not None: + await asyncio.gather(running, return_exceptions=True) + if restored is not None: + await restored.aclose() + await session.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize("peer_publishes_during_fallback", [False, True], ids=["checkpoint", "fallback"]) +async def test_first_native_autosave_preserves_a_compacted_peer( + monkeypatch, tmp_path, peer_publishes_during_fallback, +): + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + (tmp_path / "source.txt").write_text("file evidence\n", encoding="utf-8") + path = tmp_path / "peer-checkpoint.json" + first_request, reply_to_first = asyncio.Event(), asyncio.Event() + second_request, reply_to_second = asyncio.Event(), asyncio.Event() + fallback_started, release_fallback = threading.Event(), threading.Event() + peer = _BoundThinkingProvider(reads=1, thinking=False) + + async def provider(request): + response = peer(request) + if len(peer.requests) == 1: + first_request.set() + await reply_to_first.wait() + elif len(peer.requests) == 2: + second_request.set() + await reply_to_second.wait() + return response + + install_sdk_transport(monkeypatch, "anthropic", provider) + model = "claude-sonnet-5-5" + environment = LocalEnvironment(str(tmp_path)) + async with _client(model) as client: + options = {"agent": _agent(thinking=False, model=model), "llm": client, "env": environment, + "auto_save_path": str(path)} + writer = build_session(**options) + stale = running = None + try: + await writer.add_user_message("Read source.txt and then answer") + await asyncio.gather(*writer.pending_cleanup_tasks) + writer.save(str(path)) + stale = load_session(str(path), **options) + running = asyncio.create_task(stale.run_loop()) + await asyncio.wait_for(first_request.wait(), 10) + + if peer_publishes_during_fallback: + append_replacement = stale.store.append_snapshot_replacement + + def fail_checkpoint(*args, **kwargs): + raise OSError("controlled failure before checkpoint publication") + + def pause_fallback(*args, **kwargs): + fallback_started.set() + if not release_fallback.wait(10): + raise TimeoutError("fallback was not released") + return append_replacement(*args, **kwargs) + + monkeypatch.setattr(stale.store, "_atomic_json_write", fail_checkpoint) + monkeypatch.setattr(stale.store, "append_snapshot_replacement", pause_fallback) + reply_to_first.set() + assert await asyncio.to_thread(fallback_started.wait, 10) + + writer.messages = [*writer.messages, {"role": "user", "content": "peer checkpoint evidence"}] + writer.save(str(path)) + published = OpenCollab.read_session_snapshot(path) + release_fallback.set() + reply_to_first.set() + await asyncio.wait_for(second_request.wait(), 10) + await asyncio.gather(*stale._auto_save_subscriber.pending_tasks) + assert OpenCollab.read_session_snapshot(path) == published + assert any("file evidence" in row["content"] for row in stale.messages if row["role"] == "tool") + + reply_to_second.set() + assert await running == "done" + assert stale.messages[-1]["content"] == "done" + assert OpenCollab.read_session_snapshot(path) == published + error, = stale.persistence_errors + assert "conflicts with persisted" in str(error) + assert path.with_suffix(".json.journal").read_bytes() == b"" + + writer.messages.append({"role": "assistant", "content": "peer continued"}) + writer.save(str(path)) + resumed = OpenCollab.read_session_snapshot(path) + assert resumed["_autosave_sequence"] == published["_autosave_sequence"] + 1 + assert resumed["messages"][-1]["content"] == "peer continued" + finally: + release_fallback.set() + reply_to_first.set() + reply_to_second.set() + if running is not None: + await asyncio.gather(running, return_exceptions=True) + if stale is not None: + await stale.aclose() + await writer.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize("legacy_timestamps", [False, True], ids=["missing-timestamps", "timestamps"]) +async def test_native_tool_step_preserves_a_near_capacity_legacy_prefix(monkeypatch, tmp_path, legacy_timestamps): + limit = 96 * 1024 + monkeypatch.setattr(storage_module, "MAX_SESSION_SNAPSHOT_BYTES", limit) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + (tmp_path / "source.txt").write_text("file evidence\n", encoding="utf-8") + model = "claude-sonnet-5-5" + agent = _agent(thinking=False, model=model) + path = tmp_path / "legacy.json" + archive = {"role": "user", "content": ""} + prompt = "Read source.txt and then answer" + messages = [ + {"role": "system", "content": agent.system_prompt, "timestamp": "2026-01-01T00:00:00+00:00"}, + archive, + {"role": "assistant", "content": "That context is archived"}, + {"role": "user", "content": prompt}, + ] + if legacy_timestamps: + for message in messages: + message["timestamp"] = "2026-01-01T00:00:00+00:00" + snapshot = { + "snapshot_version": 1, "_autosave_sequence": 2, "messages": messages, + "session_state": { + "phase": "idle", "pending_external_user_turn": { + "turn_id": "legacy-turn", "status": "queued", "content": prompt, "message_index": 3, + }, + }, + } + archive["content"] = "x" * (limit - len(json.dumps(snapshot, separators=(",", ":"))) - 100) + path.write_text(json.dumps(snapshot, separators=(",", ":")), encoding="utf-8") + assert path.stat().st_size == limit - 100 + + class ArchiveShaper: + def shape(self, messages): + return [ + {**message, "content": "[Archived context]"} + if message.get("content") == archive["content"] else message + for message in messages + ] + + second_request = asyncio.Event() + peer = _BoundThinkingProvider(reads=1, thinking=False) + + async def provider(request): + response = peer(request) + if len(peer.requests) == 2: + second_request.set() + await asyncio.Future() + return response + + install_sdk_transport(monkeypatch, "anthropic", provider) + environment = LocalEnvironment(str(tmp_path)) + async with _client(model) as client: + session = load_session( + str(path), agent=agent, llm=client, env=environment, auto_save_path=str(path), shaper=ArchiveShaper(), + ) + running = asyncio.create_task(session.run_loop()) + waiting = asyncio.create_task(second_request.wait()) + restored = None + try: + done, _ = await asyncio.wait({waiting, running}, timeout=10, return_when=asyncio.FIRST_COMPLETED) + if running in done: + pytest.fail(f"Turn ended before its second request with status {running.result()}") + assert waiting in done + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + assert session.persistence_errors == () + saved = OpenCollab.read_session_snapshot(path) + assert saved["_autosave_sequence"] == 3 + assert saved["messages"][:len(messages)] == messages + record = json.loads(path.with_suffix(".json.journal").read_text(encoding="utf-8")) + assert record["replace_from"] == len(messages) + restored = load_session(str(path), agent=agent, llm=client, env=environment) + assert restored.state.anthropic_history is not None + assert any("file evidence" in row["content"] for row in restored.messages if row["role"] == "tool") + finally: + running.cancel() + waiting.cancel() + await asyncio.gather(running, waiting, return_exceptions=True) + if restored is not None: + await restored.aclose() + await session.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize( + ("base_sequence", "failed_write", "expected_writes"), + [ + (4, "checkpoint", ["append"]), + (7, None, ["append", "checkpoint"]), + (7, "checkpoint", ["append", "checkpoint"]), + (7, "append", ["append"]), + ], + ids=["between-checkpoints", "periodic-checkpoint", "checkpoint-error", "append-error"], +) +async def test_legacy_journal_persists_native_rewrites_before_checkpoint( + monkeypatch, tmp_path, base_sequence, failed_write, expected_writes, +): + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + path = str(tmp_path / "session.json") + backend = storage_module.SessionStore() + writes = [] + sequence = base_sequence + 1 + write_error = OSError(f"controlled {failed_write} failure") + + class LegacyJournalStore: + save = backend.save + load_messages = backend.load_messages + load_snapshot = backend.load_snapshot + save_manifest = backend.save_manifest + has_snapshot = backend.has_snapshot + + def append_snapshot_delta(self, path, **delta): + if delta["sequence"] == sequence: + writes.append("append") + assert delta["replace_from"] == 0 + if failed_write == "append": + raise write_error + backend.append_snapshot_delta(path, **delta) + + def checkpoint_snapshot(self, path, messages, *, meta, sequence): + if sequence == base_sequence + 1: + writes.append("checkpoint") + if failed_write == "checkpoint": + raise write_error + backend.checkpoint_snapshot(path, messages, meta=meta, sequence=sequence) + + held_request, resume = asyncio.Event(), asyncio.Event() + peer = _BoundThinkingProvider(reads=1, thinking=False) + + async def provider(request): + response = peer(request) + if len(peer.requests) == 2: + held_request.set() + await resume.wait() + return response + + install_sdk_transport(monkeypatch, "anthropic", provider) + model = "claude-sonnet-5-5" + environment = LocalEnvironment(str(tmp_path)) + store = LegacyJournalStore() + async with _client(model) as client: + options = {"agent": _agent(thinking=False, model=model), "llm": client, "env": environment, "store": store} + session = build_session(auto_save_path=path, **options) + running = restored = None + try: + for _ in range(base_sequence): + session.save(path) + session.messages = [*session.messages, {"role": "user", "content": "Read source.txt before answering"}] + running = asyncio.create_task(session.run_loop()) + await asyncio.wait_for(held_request.wait(), 5) + await asyncio.gather(*session._auto_save_subscriber.pending_tasks) + + restored = load_session(path, **options) + saved = store.load_snapshot(path, session.agent.system_prompt) + if failed_write == "append": + assert saved["_autosave_sequence"] == base_sequence + assert restored.state.anthropic_history is None + assert len(restored.messages) == 1 + else: + assert saved["_autosave_sequence"] == sequence + assert restored.state.anthropic_history is not None + assert restored.messages == session.messages + assert sum(row["role"] == "tool" for row in restored.messages) == 1 + assert writes == expected_writes + assert session.persistence_errors == ((write_error,) if failed_write in expected_writes else ()) + + resume.set() + assert await running == "done" + finally: + resume.set() + if running is not None: + await asyncio.gather(running, return_exceptions=True) + if restored is not None: + await restored.aclose() + await session.aclose() + await environment.cleanup() + + +async def test_autosave_without_native_history_uses_only_session_state(tmp_path, monkeypatch): + path = tmp_path / "session.json" + environment = LocalEnvironment(str(tmp_path)) + async with _client() as client: + session = build_session(agent=_agent(), llm=client, env=environment, auto_save_path=str(path)) + + def unavailable(): + raise RuntimeError("model metadata unavailable during snapshot") + + monkeypatch.setattr(session.runner, "_requires_thinking_prefix", unavailable) + try: + await session.add_user_message("Save this prompt before the first request.") + await session.enqueue_auto_save() + assert not session.persistence_errors + session.save(str(path)) + snapshot = session.store.load_snapshot(str(path), session.agent.system_prompt) + assert snapshot["messages"][-1]["content"] == "Save this prompt before the first request." + assert snapshot["session_state"]["anthropic_history"] is None + finally: + await session.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize("source", ["invalid", [], [{}, {}], [False]]) +def test_restore_rejects_malformed_optional_source_prefix(source): + message = {"role": "user", "content": "Request"} + assert _restore_anthropic_history({ + "messages": [message], "source_count": 1, "tools": None, "source_messages": source, + }, [message]) is None + + +@pytest.mark.parametrize("reference", [True, -1, 1, "0"]) +def test_restore_rejects_invalid_native_message_references(reference): + assert _restore_anthropic_history({ + "messages": [reference], "source_count": 1, "tools": None, + }, [{"role": "user", "content": "Request"}]) is None + + +@pytest.mark.parametrize("count", [0, 2]) +def test_legacy_native_history_restores_without_an_optional_source_prefix(count): + messages = [{"role": "user", "content": "First"}, {"role": "assistant", "content": "Answer"}] + view = [{"role": "system", "compacted": True, "content": "Earlier context"}] + restored = _restore_anthropic_history({"messages": view, "source_count": count, "tools": None}, messages) + assert restored == {"messages": view, "source_count": count, "tools": None, "source_messages": messages[:count]} + messages[0]["content"] = "Changed" + if count: + assert restored["source_messages"][0]["content"] == "First" diff --git a/tests/runtime/test_anthropic_request_budget_http.py b/tests/runtime/test_anthropic_request_budget_http.py index fde65e30..1de21e69 100644 --- a/tests/runtime/test_anthropic_request_budget_http.py +++ b/tests/runtime/test_anthropic_request_budget_http.py @@ -32,6 +32,26 @@ def _wire_input_tokens(request): messages.insert(0, {"role": "system", "content": request["system"]}) return estimate_request_tokens(messages, request.get("tools")) + +@pytest.mark.parametrize(("requested", "expected"), [(None, 8192), (0, 8192), (32, 32)]) +async def test_anthropic_admission_applies_the_native_output_default(monkeypatch, requested, expected): + requests = [] + + async def handler(request): + requests.append(json.loads(request.content)) + return completion_http_response(request) + + install_sdk_transport(monkeypatch, "anthropic", handler) + async with LLMClient( + model=_MODEL, provider="anthropic", api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + ) as client: + result = await client.complete([{"role": "user", "content": "Answer ok"}], max_output_tokens=requested) + + assert result.content == "ok" + assert [body["max_tokens"] for body in requests] == [expected] + + @pytest.mark.parametrize("budget,thinking,expected_tokens,expected_calls", [ (11_000, True, 9702, 2), (None, True, 9702, 2), diff --git a/tests/runtime/test_authored_artifact_transfer.py b/tests/runtime/test_authored_artifact_transfer.py index 35846d06..b9b74fa4 100644 --- a/tests/runtime/test_authored_artifact_transfer.py +++ b/tests/runtime/test_authored_artifact_transfer.py @@ -19,6 +19,63 @@ from opencollab.application.tool_execution import ToolRuntime from opencollab.bootstrap.authored_workflow import AuthoredWorkflowTool, _Child from opencollab.bootstrap.authored_workflow_bridge import AuthoredWorkflowBridge, BridgeClient, BridgeEnvironment +from tests.support.docker_native_edits_support import LocalDockerTransport, docker_environment + + +@pytest.mark.parametrize("absolute", [False, True]) +@pytest.mark.parametrize("component", ["directory", "symlink", "missing"]) +async def test_artifact_transfer_keeps_parent_traversal_identity(tmp_path, absolute, component): + workspace = tmp_path / "workspace" + (workspace / "actual/child").mkdir(parents=True) + (workspace / "result.json").write_text('{"selected": 42}') + (workspace / "actual/result.json").write_text('{"other": 7}') + if component == "directory": + (workspace / "parent").mkdir() + elif component == "symlink": + (workspace / "parent").symlink_to("actual/child", target_is_directory=True) + relative = "parent/../result.json" + source = os.path.join(str(workspace), relative) if absolute else relative + target = tmp_path / "copied.json" + environment = LocalEnvironment(str(workspace)) + try: + if component == "directory": + await _copy_artifact_to_host(environment, source, target) + assert target.read_text() == '{"selected": 42}' + else: + with pytest.raises(OSError): + await _copy_artifact_to_host(environment, source, target) + assert not target.exists() + assert (workspace / "result.json").read_text() == '{"selected": 42}' + assert (workspace / "actual/result.json").read_text() == '{"other": 7}' + finally: + await environment.cleanup() + + +@pytest.mark.parametrize("absolute", [False, True]) +async def test_docker_artifact_transfer_resolves_parent_in_execution_environment(tmp_path, absolute): + (tmp_path / "actual/child").mkdir(parents=True) + (tmp_path / "alias").symlink_to("actual/child", target_is_directory=True) + (tmp_path / "actual/result.json").write_text('{"selected": 42}') + (tmp_path / "result.json").write_text('{"other": 7}') + + class ArtifactTransport(LocalDockerTransport): + async def __call__(self, *argv, **kwargs): + if argv[0] == "cp": + shutil.copyfile(argv[2].split(":", 1)[1], argv[3]) + return ProcessResult(0, b"", b"") + return await super().__call__(*argv, **kwargs) + + transport = ArtifactTransport(tmp_path, pair_reads=False) + environment = docker_environment(tmp_path, transport) + relative = "alias/../result.json" + source = os.path.join(str(tmp_path), relative) if absolute else relative + target = tmp_path / "copied.json" + try: + await _copy_artifact_to_host(environment, source, target) + assert target.read_text() == '{"selected": 42}' + assert (tmp_path / "result.json").read_text() == '{"other": 7}' + finally: + await environment.cleanup() @pytest.mark.parametrize("kind", ["local", "worktree"]) diff --git a/tests/runtime/test_authored_request_budget_http.py b/tests/runtime/test_authored_request_budget_http.py new file mode 100644 index 00000000..5abcfa1c --- /dev/null +++ b/tests/runtime/test_authored_request_budget_http.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager + +import pytest + +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm import client as transport +from opencollab.adapters.llm.client import LLMClient +from opencollab.adapters.llm.request_admission import request_slot +from opencollab.adapters.session_environments import SessionEnvironmentPool +from opencollab.application._session_run_shared import _TokenBudgetStop +from opencollab.bootstrap.authored_workflow_bridge import AuthoredWorkflowBridge +from tests.runtime.test_llm_chat_error_usage_http import ReportedUsageStream +from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport + + +@pytest.fixture(params=[ + ({"provider": "openai", "wire_protocol": "chat_completions", "model": "gpt-4o"}, "max_tokens"), + ({"provider": "openai", "wire_protocol": "responses", "model": "gpt-4o"}, "max_output_tokens"), + ({"provider": "anthropic", "model": "claude-sonnet-4-6"}, "max_tokens"), +], ids=["chat", "responses", "anthropic"]) +def configuration(request): + return request.param + + +@pytest.mark.parametrize("outcome", ["success", "ledger_error", "ledger_cancellation"]) +async def test_native_usage_is_reported_once_inside_the_slot( + tmp_path, monkeypatch, configuration, outcome, +): + config, _field = configuration + usages = [] + released = asyncio.Event() + install_sdk_transport(monkeypatch, config["provider"], completion_http_response) + native_slot = transport.request_slot + + @asynccontextmanager + async def observed_slot(*args): + async with native_slot(*args): + try: + yield + finally: + assert [usage.total_tokens for usage in usages] == [4] + released.set() + + async def record(**kwargs): + assert released.is_set() + if outcome == "ledger_error": + raise ValueError("controlled ledger failure") + if outcome == "ledger_cancellation": + raise asyncio.CancelledError + + monkeypatch.setattr(transport, "request_slot", observed_slot) + monkeypatch.setattr(transport, "_record_api_usage_async", record) + async with LLMClient( + **config, api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", request_pool=str(tmp_path), request_capacity=1, + ) as client: + operation = client.complete([{"role": "user", "content": "finish"}], on_usage=usages.append) + if outcome == "success": + assert (await operation).content == "ok" + else: + with pytest.raises(ValueError if outcome == "ledger_error" else asyncio.CancelledError) as caught: + await operation + assert caught.value.usage.total_tokens == 4 + async with request_slot(str(tmp_path), 1): + assert len(usages) == 1 + + +@pytest.mark.parametrize("change", ["remaining", "reduced", "exhausted", "unbounded", "slow_ledger"]) +async def test_host_bridge_rechecks_queued_budget_before_native_dispatch( + tmp_path, monkeypatch, configuration, change, +): + config, field = configuration + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + entered, release, queued = asyncio.Event(), asyncio.Event(), asyncio.Event() + requests = [] + first_tokens = 5_000 if change == "remaining" else 20_210 + + async def handler(request): + body = json.loads(request.content) + requests.append(body) + if len(requests) == 1: + entered.set() + await release.wait() + total = first_tokens if len(requests) == 1 else 4 + response = completion_http_response(request) + if request.url.path.endswith("/responses"): + return http_response( + request, 200, headers={"content-type": "text/event-stream"}, + text=response.text.replace('"input_tokens": 2', f'"input_tokens": {total - 2}').replace( + '"total_tokens": 4', f'"total_tokens": {total}', + ), + ) + payload = response.json() + if config["provider"] == "anthropic": + payload["usage"]["input_tokens"] = total - 2 + else: + payload["usage"].update(prompt_tokens=total - 2, total_tokens=total) + return http_response(request, 200, json=payload) + + install_sdk_transport(monkeypatch, config["provider"], handler) + record = transport._record_api_usage_async + + async def delayed_record(**kwargs): + if change == "slow_ledger": + await asyncio.sleep(0.12) + await record(**kwargs) + + monkeypatch.setattr(transport, "_record_api_usage_async", delayed_record) + pool = tmp_path / "pool" + pool.mkdir() + artifacts = tmp_path / "artifacts" + (artifacts / "bridge").mkdir(parents=True) + environment = LocalEnvironment(str(tmp_path)) + config = {**config, "api_key": "controlled-test", # pragma: allowlist secret + "base_url": "https://controlled.invalid/v1", + "request_pool": str(pool), "request_capacity": 1, "max_output_tokens": 128} + bridge = AuthoredWorkflowBridge( + environment, str(artifacts), SessionEnvironmentPool(str(tmp_path), isolation="shared", + base_environment=environment), + config, None, allow_unisolated_shell=True, budget=None if change == "unbounded" else 20_000, + ) + second_messages = [{"role": "user", "content": "second"}] + if change == "reduced": + first_tokens = 20_000 - bridge._model().estimate_request_tokens(second_messages) - 50 + native_slot = transport.request_slot + entries = 0 + + @asynccontextmanager + async def observed_slot(*args): + nonlocal entries + entries += 1 + if entries == 2: + queued.set() + async with native_slot(*args): + yield + + monkeypatch.setattr(transport, "request_slot", observed_slot) + + async def complete(identity, messages): + return await bridge._dispatch(identity, {"action": "complete", "arguments": { + "messages": messages, "max_output_tokens": 128, + }}) + + first = asyncio.create_task(complete("1", [{"role": "user", "content": "first"}])) + second = None + try: + await asyncio.wait_for(entered.wait(), 5) + second = asyncio.create_task(complete("2", second_messages)) + await asyncio.wait_for(queued.wait(), 5) + assert len(requests) == 1 + release.set() + await asyncio.wait_for(first, 5) + if change in {"exhausted", "slow_ledger"}: + with pytest.raises(_TokenBudgetStop): + await asyncio.wait_for(second, 5) + assert len(requests) == 1 + assert bridge.tokens == first_tokens + else: + assert (await asyncio.wait_for(second, 5))["content"] == "ok" + assert len(requests) == 2 + assert requests[1][field] == (50 if change == "reduced" else 128) + assert bridge.tokens == first_tokens + 4 + finally: + release.set() + for task in (first, second): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather(*(task for task in (first, second) if task is not None), return_exceptions=True) + await bridge.aclose() + await environment.cleanup() + + +async def test_host_bridge_charges_reported_error_usage_once_before_rechecking(tmp_path, monkeypatch): + requests = [] + + async def handler(request): + requests.append(json.loads(request.content)) + return http_response( + request, 200, headers={"content-type": "text/event-stream"}, + stream=ReportedUsageStream({"prompt_tokens": 20_200, "completion_tokens": 10, "total_tokens": 20_210}, + disconnect=True), + ) + + install_sdk_transport(monkeypatch, "openai", handler) + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + environment = LocalEnvironment(str(tmp_path)) + artifacts = tmp_path / "artifacts" + (artifacts / "bridge").mkdir(parents=True) + bridge = AuthoredWorkflowBridge( + environment, str(artifacts), SessionEnvironmentPool(str(tmp_path), isolation="shared", + base_environment=environment), + {"model": "gpt-4o", "provider": "openai", + "api_key": "controlled-test", # pragma: allowlist secret + "base_url": "https://controlled.invalid/v1", "request_pool": str(tmp_path), "request_capacity": 1, + "llm_stream_chat": True, "llm_max_retries": 0}, + None, allow_unisolated_shell=True, budget=20_000, + ) + try: + with pytest.raises(Exception) as caught: + await bridge._dispatch("1", {"action": "complete", "arguments": { + "messages": [{"role": "user", "content": "first"}], "max_output_tokens": 128, + }}) + assert caught.value.usage.total_tokens == bridge.tokens == 20_210 + with pytest.raises(_TokenBudgetStop): + await bridge._dispatch("2", {"action": "complete", "arguments": { + "messages": [{"role": "user", "content": "second"}], "max_output_tokens": 128, + }}) + assert len(requests) == 1 and bridge.tokens == 20_210 + finally: + await bridge.aclose() + await environment.cleanup() diff --git a/tests/runtime/test_authored_workflow_bridge.py b/tests/runtime/test_authored_workflow_bridge.py index f1ac07af..3782ec39 100644 --- a/tests/runtime/test_authored_workflow_bridge.py +++ b/tests/runtime/test_authored_workflow_bridge.py @@ -1,33 +1,179 @@ from __future__ import annotations import asyncio +import json import os +import shlex +import sys +from dataclasses import asdict from types import SimpleNamespace import pytest +from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace from opencollab.adapters.env import DockerEnvironment, LocalEnvironment +from opencollab.adapters.hooks import ShellHookRunner from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.llm.errors import TransientProviderError from opencollab.adapters.llm.request_admission import request_slot from opencollab.adapters.llm.types import LLMResponse, Usage from opencollab.adapters.session_environments import SessionEnvironmentPool +from opencollab.adapters.tools.bash import BashTool from opencollab.adapters.tools.fs import FileWriteTool -from opencollab.application._session_run_shared import _TokenBudgetStop +from opencollab.application._session_run_shared import _TeamBudgetStop, _TokenBudgetStop from opencollab.application.ports import ( ContextTokenEstimatorPort, RequestOutputRequirementPort, RequestTokenEstimatorPort, ) +from opencollab.application.run_control import _RunCancelled from opencollab.application.tool_execution import ToolRuntime +from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError from opencollab.bootstrap.authored_workflow_bridge import ( AuthoredWorkflowBridge, BridgeClient, BridgeEnvironment, BridgeLLM, + _restore_error, ) from opencollab.bootstrap.session_factory import build_session from opencollab.domain.agent import Agent +from opencollab.domain.hooks import HookSpec +from tests.interfaces.test_member_authored_workflow import _launch, _manager +from tests.workflows.test_workflow_candidate_workspace import _git + + +@pytest.mark.parametrize("error", [ + _TokenBudgetStop(reserved_input_tokens=1027, remaining_budget=500, minimum_output_tokens=1), + _TeamBudgetStop(), + _RunCancelled(), +]) +def test_bridge_restores_usage_on_control_errors(error): + usage = Usage(1_498, 2, cache_read_tokens=19, raw_usage={"total_tokens": 1_500}) + payload = {"type": type(error).__name__, "message": str(error), "usage": asdict(usage), **vars(error)} + restored = _restore_error(payload) + + assert type(restored) is type(error) + assert str(restored) == str(error) + assert vars(restored) == {**vars(error), "usage": usage} + + +@pytest.mark.parametrize("writer_kind", ["bridge", "authored-process", "tool-hook"]) +async def test_managed_commands_keep_parked_candidate_bytes(tmp_path, monkeypatch, writer_kind): + monkeypatch.chdir(tmp_path) + workspace = tmp_path / "workspace" + workspace.mkdir() + _git(workspace, "init", "-q") + (workspace / "record.txt").write_text("original\n") + _git(workspace, "add", "record.txt") + _git(workspace, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare bridged writer") + (workspace / "settings").mkdir() + (workspace / "settings/notes.txt").write_text("Existing settings\n") + patch = ( + "diff --git a/record.txt b/record.txt\n--- a/record.txt\n+++ b/record.txt\n" + "@@ -1 +1 @@\n-original\n+candidate\n" + "diff --git a/settings b/settings\nnew file mode 100644\n--- /dev/null\n+++ b/settings\n" + "@@ -0,0 +1 @@\n+replacement\n" + ) + (workspace / "authored.py").write_text( + "import asyncio, os\n" + "from pathlib import Path\n" + "async def main(ctx, args):\n" + " with open('record.txt', 'a') as stream:\n" + " Path('writer.ready').touch()\n" + " while not Path('writer.release').is_file(): await asyncio.sleep(0.01)\n" + " stream.write('authored append\\n')\n" + " stream.flush()\n" + " os.fsync(stream.fileno())\n" + " Path('writer.flushed').touch()\n" + " return 'saved'\n" + "if __name__ == '__main__': asyncio.run(main(None, {}))\n" + ) + index = (workspace / ".git/index").read_bytes() + host = LocalEnvironment(str(workspace)) + mailbox = LocalEnvironment(str(tmp_path)) + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + bridge = AuthoredWorkflowBridge( + mailbox, str(artifacts), SessionEnvironmentPool(str(tmp_path), isolation="shared", base_environment=mailbox), + {}, ToolRuntime(mailbox, None, None), allow_unisolated_shell=True, + ) + bridge._environments[1] = host + proxy = BridgeEnvironment(BridgeClient(artifacts, bridge.options()), { + "handle": 1, "workspace": str(workspace), "process_isolated": False, + }) + parent, manager, runtime = _manager(host) if writer_kind == "authored-process" else (None, None, None) + published, permit_recovery, stop = asyncio.Event(), asyncio.Event(), asyncio.Event() + native_exec = host.exec_cmd + + async def pause_failed_apply(cmd, *args, **kwargs): + result = await native_exec(cmd, *args, **kwargs) + if " apply --binary " in cmd and ".candidate-adopt-" in cmd: + assert result.returncode == 128 + published.set() + await permit_recovery.wait() + return result + + monkeypatch.setattr(host, "exec_cmd", pause_failed_apply) + + async def serve(): + while not stop.is_set(): + await bridge.poll() + await asyncio.sleep(0.005) + + async def writer_ready(): + while not (workspace / "writer.ready").is_file(): + if writer.done(): + writer.result() + raise AssertionError("writer finished before opening the file") + await asyncio.sleep(0.005) + + async def run_writer(): + command = shlex.join((sys.executable, "authored.py")) + if writer_kind == "bridge": + outcome = await BashTool(require_process_isolation=False).execute_with_runtime( + {"command": command}, ToolRuntime(proxy, None, None), + ) + assert "Exit code: 0" in outcome + elif writer_kind == "authored-process": + started = await _launch(manager, runtime, isolation="shared") + outcome = json.loads(await manager.execute_with_runtime({"action": "wait", "id": started["id"]}, runtime)) + assert outcome["status"] == "completed" + assert outcome["result"]["output"] == "saved" + else: + hook = HookSpec(event="PreToolUse", action_type="command", command=command) + runner = ShellHookRunner((hook,), scheduler=SimpleNamespace(agent_environment=lambda _: host)) + await runner.fire("PreToolUse", {"hook_event_name": "PreToolUse", "aid": 0, "tool": "bash"}) + assert (workspace / "writer.flushed").is_file() + + server = asyncio.create_task(serve()) + publication = asyncio.create_task(EnvCandidateWorkspace(host).adopt(patch)) + writer = None + try: + await asyncio.wait_for(published.wait(), 10) + writer = asyncio.create_task(run_writer()) + await asyncio.wait_for(writer_ready(), 10) + permit_recovery.set() + with pytest.raises(CandidateAdoptionRecoveryError, match="parked files retained"): + await asyncio.wait_for(publication, 10) + (workspace / "writer.release").touch() + await asyncio.wait_for(writer, 10) + saved, = (workspace / ".git").glob("candidate-recovery-*/observed-*/entry") + assert saved.read_text() == "candidate\nauthored append\n" + assert (workspace / "record.txt").read_text() == "original\n" + assert (workspace / ".git/index").read_bytes() == index + finally: + permit_recovery.set() + (workspace / "writer.release").touch() + await asyncio.gather(publication, *([writer] if writer else []), return_exceptions=True) + stop.set() + await server + await bridge.aclose() + if parent is not None: + await parent.aclose() + await mailbox.cleanup() + await host.cleanup() @pytest.mark.parametrize("docker", [False, True]) diff --git a/tests/runtime/test_docker_owned_temp_cancellation.py b/tests/runtime/test_docker_owned_temp_cancellation.py new file mode 100644 index 00000000..b74b40b9 --- /dev/null +++ b/tests/runtime/test_docker_owned_temp_cancellation.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from tests.support.docker_native_edits_support import LocalDockerTransport, docker_environment + + +class _PublishedReceipt(LocalDockerTransport): + def __init__(self, workspace, *, delay): + super().__init__(workspace, pair_reads=False) + self.delay = delay + self.published = asyncio.Event() + + async def __call__(self, *argv, **kwargs): + result = await super().__call__(*argv, **kwargs) + if kwargs.get("input_bytes") is not None and "opencollab-write " in argv[-1]: + self.published.set() + if self.delay: + await asyncio.Event().wait() + return result + + +@pytest.mark.parametrize("cancel", [False, True], ids=["completed", "lost-receipt"]) +async def test_attached_cleanup_removes_published_temporary_after_lost_receipt(tmp_path, cancel): + transport = _PublishedReceipt(tmp_path, delay=cancel) + env = docker_environment(tmp_path, transport) + targets = [] + write_atomic = env._write_file_atomic + + async def observe_target(target, payload, **kwargs): + targets.append(Path(target)) + return await write_atomic(target, payload, **kwargs) + + env._write_file_atomic = observe_target + writer = asyncio.create_task(env.write_temp_file("retained candidate\n", prefix="oc-owned-temp-")) + try: + await asyncio.wait_for(transport.published.wait(), 5) + if cancel: + writer.cancel() + with pytest.raises(asyncio.CancelledError): + await writer + else: + await writer + assert len(targets) == 1 + assert targets[0].read_text() == "retained candidate\n" + assert {Path(path).resolve() for path in env._temporary_files} == {targets[0].resolve()} + await env.cleanup() + assert not targets[0].exists() + assert not env._temporary_files + finally: + if not writer.done(): + writer.cancel() + await asyncio.gather(writer, return_exceptions=True) + await env.cleanup() + for target in targets: + target.unlink(missing_ok=True) diff --git a/tests/runtime/test_exported_session_resume.py b/tests/runtime/test_exported_session_resume.py index cb657c54..2d28e702 100644 --- a/tests/runtime/test_exported_session_resume.py +++ b/tests/runtime/test_exported_session_resume.py @@ -8,10 +8,14 @@ import pytest from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.storage import SessionStore +from opencollab.application.ports import SnapshotSequenceConflict from opencollab.bootstrap import build_session, load_session +from opencollab.domain import session as session_module from opencollab.domain.agent import Agent from opencollab.tools import builtin_tools +from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport from tests.support.session_run_test_support import FakeLLM, llm_response, tool_call @@ -172,6 +176,126 @@ async def test_export_without_original_autosave_can_start_autosaving(tmp_path, t await restored.aclose() await restored.env.cleanup() +async def test_old_anthropic_export_can_resume_an_existing_autosave_twice(tmp_path, monkeypatch): + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + install_sdk_transport(monkeypatch, "anthropic", completion_http_response) + exported = tmp_path / "exported.json" + target = tmp_path / "automatic.json" + agent = Agent(name="reviewer", system_prompt="Remember the prior conversation.", model="claude-sonnet-5-5", + provider="anthropic", thinking=False, max_tokens_per_step=1024) + environment = LocalEnvironment(str(tmp_path)) + try: + async with LLMClient(model=agent.model, provider="anthropic", max_retries=0, + api_key="fake-key", base_url="https://example.invalid/v1") as client: + options = {"agent": agent, "llm": client, "env": environment} + original = build_session(**options) + try: + await original.add_user_message("Remember the review deadline.") + assert await original.run_loop() == "ok" + original.save(str(exported)) + messages = original.messages.copy() + finally: + await original.aclose() + + old_export = json.loads(exported.read_text()) + history = old_export["session_state"]["anthropic_history"] + history["messages"] = [messages[row] if type(row) is int else row for row in history["messages"]] + exported.write_text(json.dumps(old_export)) + first = load_session(str(exported), **options, auto_save_path=str(target)) + await first.aclose() + + old_target = json.loads(target.read_text()) + assert old_target["_autosave_sequence"] == 1 + assert "timestamp" in old_target["messages"][1] + old_target["session_state"]["anthropic_history"] = history + target.write_text(json.dumps(old_target)) + + resumed = load_session(str(exported), **options, auto_save_path=str(target)) + try: + assert resumed.messages == messages + assert resumed.state.anthropic_history["messages"] == history["messages"] + assert json.loads(target.read_text())["_autosave_sequence"] == 1 + finally: + await resumed.aclose() + finally: + await environment.cleanup() + + +@pytest.mark.parametrize("conflict", [None, "source-timestamp", "recovered-content"]) +async def test_interrupted_tool_export_retries_the_same_autosave(tmp_path, monkeypatch, conflict): + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + exported, target = tmp_path / "interrupted.json", tmp_path / "automatic.json" + (tmp_path / "archive.txt").write_text("The archive opened on April 18.") + model = "claude-sonnet-5-5" + agent = Agent(name="reviewer", system_prompt="Read the archive.", model=model, + provider="anthropic", tools=builtin_tools("file_read", headless=False)) + env = LocalEnvironment(str(tmp_path)) + requests = [] + + def peer(request): + requests.append(json.loads(request.content)) + if len(requests) > 1: + return completion_http_response(request, text="Done") + return http_response(request, 200, json={ + "id": "msg_read", "type": "message", "role": "assistant", "model": model, + "content": [{"type": "tool_use", "id": "read-archive", "name": "file_read", + "input": {"path": "archive.txt"}}], + "stop_reason": "tool_use", "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 5}, + }) + + async def export_at_tool_start(event): + if event.type == "tool_start": + original.save(str(exported)) + + install_sdk_transport(monkeypatch, "anthropic", peer) + try: + async with LLMClient( + model=model, provider="anthropic", max_retries=0, + api_key="test-placeholder", base_url="https://provider.invalid/v1", # pragma: allowlist secret + ) as client: + options = {"agent": agent, "llm": client, "env": env} + original = build_session(**options, event_sink=export_at_tool_start) + try: + await original.add_user_message("Look up the date.") + assert await original.run_loop() == "Done" + finally: + await original.aclose() + + first_stamp = "2026-01-01T00:00:00+00:00" + monkeypatch.setattr(session_module, "_now_iso", lambda: first_stamp) + first = load_session(str(exported), **options, auto_save_path=str(target)) + await first.aclose() + stored = json.loads(target.read_text()) + assert stored["_autosave_sequence"] == 1 + assert stored["messages"][-1]["timestamp"] == first_stamp + assert stored["messages"][-1]["content"] == "Tool execution interrupted by session restore." + + if conflict == "source-timestamp": + stored["messages"][1]["timestamp"] = "2026-01-01T00:00:02+00:00" + elif conflict == "recovered-content": + stored["messages"][-1]["content"] = "A different result was confirmed." + target.write_text(json.dumps(stored)) + monkeypatch.setattr(session_module, "_now_iso", lambda: "2026-01-01T00:00:01+00:00") + + if conflict: + with pytest.raises(SnapshotSequenceConflict, match="conflicts with persisted snapshot state"): + load_session(str(exported), **options, auto_save_path=str(target)) + else: + resumed = load_session(str(exported), **options, auto_save_path=str(target)) + try: + assert resumed.state.enriched_messages() == stored["messages"] + assert resumed.state.anthropic_history is not None + finally: + await resumed.aclose() + assert json.loads(target.read_text()) == stored + assert len(requests) == 2 + finally: + await env.cleanup() + + class SnapshotOnlyStore: """Exercise full-snapshot persistence without journal support.""" @@ -207,7 +331,7 @@ async def complete(self, messages, **kwargs): def autosave_tracking(session): return copy.deepcopy({name: getattr(session, name) for name in ( - "_auto_save_sequence", "_auto_save_reserved_sequence", "_auto_save_message_count", + "_auto_save_sequence", "_auto_save_reserved_sequence", "_auto_save_message_count", "_auto_save_messages", "_auto_save_rewrite_from", "_auto_save_rewrite_revision", "_auto_save_pending_checkpoints", "_auto_save_seen_result_hashes", "_next_auto_save_checkpoint", "_loop_checkpoint_results", "_loop_checkpoint_prefix", "_loop_checkpoint_step", diff --git a/tests/runtime/test_llm_chat_error_usage_http.py b/tests/runtime/test_llm_chat_error_usage_http.py index 59b310ea..3035d372 100644 --- a/tests/runtime/test_llm_chat_error_usage_http.py +++ b/tests/runtime/test_llm_chat_error_usage_http.py @@ -174,6 +174,7 @@ async def wait_for_cancellation(_seconds): await session.add_user_message("Reply ok.") run = asyncio.create_task(session.run_loop()) await asyncio.wait_for(entered_backoff.wait(), 1) + assert session.used_tokens == 27 run.cancel() with pytest.raises(asyncio.CancelledError): await run diff --git a/tests/runtime/test_local_env_path_identity.py b/tests/runtime/test_local_env_path_identity.py new file mode 100644 index 00000000..6b4ca5c4 --- /dev/null +++ b/tests/runtime/test_local_env_path_identity.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import asyncio +import os +import subprocess +import sys +import tempfile +import threading +from pathlib import Path + +import pytest + +from opencollab.adapters import safe_anchored_files +from opencollab.environments import local_environment + + +async def _operate(environment, operation, path): + if operation == "read": + return await environment.read_file(path) + if operation == "range": + result = await environment.read_text_range(path, offset=1, limit=1, max_chars=100) + return "\n".join(result.lines) + "\n" + await environment.write_file(path, "changed\n") + + +@pytest.mark.parametrize("absolute", [False, True]) +@pytest.mark.parametrize("operation", ["read", "range", "write"]) +@pytest.mark.parametrize("component", ["directory", "symlink", "missing", "regular-file"]) +async def test_local_parent_traversal_checks_each_original_component( + tmp_path, absolute, operation, component, +): + (tmp_path / "actual/child").mkdir(parents=True) + target = tmp_path / "value.txt" + target.write_text("original\n") + other = tmp_path / "actual/value.txt" + other.write_text("other\n") + parent = tmp_path / "component" + if component == "directory": + parent.mkdir() + elif component == "symlink": + parent.symlink_to("actual/child", target_is_directory=True) + elif component == "regular-file": + parent.write_text("regular\n") + relative = "component/../value.txt" + path = os.path.join(str(tmp_path), relative) if absolute else relative + environment = local_environment(tmp_path) + try: + if component == "directory": + result = await _operate(environment, operation, path) + if operation != "write": + assert result == "original\n" + assert target.read_text() == ("changed\n" if operation == "write" else "original\n") + else: + with pytest.raises(OSError): + await _operate(environment, operation, path) + assert target.read_text() == "original\n" + assert other.read_text() == "other\n" + if component == "missing": + assert not parent.exists() + finally: + await environment.cleanup() + + +@pytest.mark.parametrize("operation", ["read", "range", "write"]) +async def test_local_final_symlink_remains_rejected(tmp_path, operation): + target = tmp_path / "value.txt" + target.write_text("original\n") + (tmp_path / "link").symlink_to(target.name) + environment = local_environment(tmp_path) + try: + with pytest.raises(OSError): + await _operate(environment, operation, "link") + assert target.read_text() == "original\n" + assert (tmp_path / "link").is_symlink() + finally: + await environment.cleanup() + + +async def test_local_parent_traversal_keeps_new_parent_creation_and_owned_removal(tmp_path): + (tmp_path / "sub").mkdir() + environment = local_environment(tmp_path) + try: + await environment.write_file("sub/../new/value.txt", "created\n") + assert (tmp_path / "new/value.txt").read_text() == "created\n" + temporary = await environment.write_temp_file("temporary", prefix="owned-") + await environment.remove_file("sub/../" + Path(temporary).name) + assert not Path(temporary).exists() + finally: + await environment.cleanup() + + +async def test_local_workspace_selection_resolves_symlink_before_parent(tmp_path): + (tmp_path / "actual/child").mkdir(parents=True) + (tmp_path / "alias").symlink_to("actual/child", target_is_directory=True) + (tmp_path / "actual/value.txt").write_text("selected\n") + (tmp_path / "value.txt").write_text("unrelated\n") + environment = local_environment(os.path.join(str(tmp_path), "alias/..")) + try: + assert await environment.read_file("value.txt") == "selected\n" + finally: + await environment.cleanup() + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS system-directory aliases") +@pytest.mark.parametrize("alias_root", ["/tmp", "/var/tmp"]) +async def test_local_absolute_system_alias_preserves_parent_checks(alias_root): + with tempfile.TemporaryDirectory(dir=os.path.realpath(alias_root)) as directory: + root = Path(directory) + (root / "sub").mkdir() + (root / "actual/child").mkdir(parents=True) + (root / "alias").symlink_to("actual/child", target_is_directory=True) + target = root / "value.txt" + target.write_text("original\n") + public_root = os.path.join(alias_root, root.name) + environment = local_environment(root) + try: + await environment.write_file(os.path.join(public_root, "sub/../value.txt"), "changed\n") + with pytest.raises(OSError): + await environment.write_file(os.path.join(public_root, "alias/../value.txt"), "wrong\n") + assert target.read_text() == "changed\n" + assert not (root / "actual/value.txt").exists() + finally: + await environment.cleanup() + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX descriptor limits") +@pytest.mark.parametrize("traversal", ["plain", "early-parent", "late-parent"]) +def test_deep_ordinary_local_paths_use_bounded_descriptors(tmp_path, traversal): + relative = "/".join(["sub"] * 64 + ["value.txt"]) + target = tmp_path / relative + target.parent.mkdir(parents=True) + target.write_text("original\n") + if traversal == "early-parent": + (tmp_path / "cancel").mkdir() + relative = "cancel/../" + relative + elif traversal == "late-parent": + (target.parent / "cancel").mkdir() + relative = "/".join(["sub"] * 64 + ["cancel", "..", "value.txt"]) + code = ( + "import asyncio, os, resource, sys\n" + "from opencollab.environments import local_environment\n" + "resource.setrlimit(resource.RLIMIT_NOFILE, (32, resource.getrlimit(resource.RLIMIT_NOFILE)[1]))\n" + "before = len(os.listdir('/dev/fd'))\n" + "async def run():\n" + " environment = local_environment(sys.argv[1])\n" + " try:\n" + " assert await environment.read_file(sys.argv[2]) == 'original\\n'\n" + " await environment.write_file(sys.argv[2], 'changed\\n')\n" + " finally:\n" + " await environment.cleanup()\n" + "asyncio.run(run())\n" + "assert len(os.listdir('/dev/fd')) == before\n" + ) + result = subprocess.run( + [sys.executable, "-c", code, str(tmp_path), relative], capture_output=True, text=True, timeout=10, + ) + assert result.returncode == 0, result.stderr + assert target.read_text() == "changed\n" + + +async def test_parent_step_keeps_the_opened_ancestor_when_its_path_is_replaced(tmp_path, monkeypatch): + (tmp_path / "parent/child").mkdir(parents=True) + (tmp_path / "parent/value.txt").write_text("original\n") + environment = local_environment(tmp_path) + started, release = threading.Event(), threading.Event() + original_open = safe_anchored_files.os.open + + def blocked_open(path, flags, mode=0o777, *, dir_fd=None): + if os.fspath(path) == "value.txt": + started.set() + assert release.wait(2) + return original_open(path, flags, mode, dir_fd=dir_fd) + + monkeypatch.setattr(safe_anchored_files.os, "open", blocked_open) + reading = asyncio.create_task(environment.read_file("parent/child/../value.txt")) + try: + assert await asyncio.to_thread(started.wait, 2) + (tmp_path / "parent").rename(tmp_path / "saved-parent") + (tmp_path / "parent").mkdir() + (tmp_path / "parent/value.txt").write_text("replacement\n") + release.set() + assert await reading == "original\n" + assert (tmp_path / "parent/value.txt").read_text() == "replacement\n" + finally: + release.set() + await asyncio.gather(reading, return_exceptions=True) + await environment.cleanup() diff --git a/tests/runtime/test_model_request_admission.py b/tests/runtime/test_model_request_admission.py index 108b3306..8cc116e1 100644 --- a/tests/runtime/test_model_request_admission.py +++ b/tests/runtime/test_model_request_admission.py @@ -4,6 +4,7 @@ import os import subprocess import sys +from contextlib import asynccontextmanager import pytest @@ -14,6 +15,60 @@ from tests.support.paths import PACKAGE_ROOT +@pytest.mark.parametrize("configuration, provider_function", [ + ({"model": "gpt-4o"}, "complete_openai"), + ({"model": "gpt-4o", "wire_protocol": "responses"}, "complete_responses"), + ({"model": "claude-sonnet-4-6", "provider": "anthropic"}, "complete_anthropic"), +], ids=["chat", "responses", "anthropic"]) +@pytest.mark.parametrize("consume_hooks", [False, True]) +@pytest.mark.parametrize("configured_limit", [None, 50, 128]) +async def test_slot_admission_prepares_first_request_before_provider_entry( + tmp_path, monkeypatch, configuration, provider_function, consume_hooks, configured_limit, +): + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + queued = asyncio.Event() + events = [] + limits = iter([80, 31]) + native_slot = transport.request_slot + first_limit = 80 if configured_limit is None else min(configured_limit, 80) + + @asynccontextmanager + async def observed_slot(*args): + queued.set() + async with native_slot(*args): + yield + + def revalidate(): + limit = next(limits) + events.append(f"limit_{limit}") + return limit + + async def complete(*_args, **kwargs): + events.append("provider") + assert events == ["limit_80", "admitted", "provider"] + assert kwargs["max_output_tokens"] == first_limit + if consume_hooks: + assert kwargs["before_attempt"]() == first_limit + assert events == ["limit_80", "admitted", "provider"] + assert kwargs["before_attempt"]() == 31 + return LLMResponse(content="answer") + + monkeypatch.setattr(transport, "request_slot", observed_slot) + monkeypatch.setattr(transport, provider_function, complete) + async with LLMClient( + **configuration, api_key="local-test", request_pool=str(tmp_path), request_capacity=1, + ) as client: + async with native_slot(str(tmp_path), 1): + owner = asyncio.create_task(client.complete( + [], max_output_tokens=configured_limit, before_request=revalidate, + on_admitted=lambda: events.append("admitted"), + )) + await asyncio.wait_for(queued.wait(), 1) + assert events == [] + assert (await asyncio.wait_for(owner, 1)).content == "answer" + assert events == ["limit_80", "admitted", "provider", *(["limit_31"] if consume_hooks else [])] + + async def test_cancellation_retains_slot_until_native_cleanup_finishes(tmp_path, monkeypatch): entered = asyncio.Event() cleaning = asyncio.Event() diff --git a/tests/runtime/test_pooled_generation_timeout.py b/tests/runtime/test_pooled_generation_timeout.py index ceefde22..ae78e922 100644 --- a/tests/runtime/test_pooled_generation_timeout.py +++ b/tests/runtime/test_pooled_generation_timeout.py @@ -1,6 +1,10 @@ from __future__ import annotations import asyncio +import json +import threading +from copy import deepcopy +from functools import wraps import pytest @@ -8,16 +12,98 @@ from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.llm.request_admission import request_slot from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.application.run_control import RunControl from opencollab.application.session_run import GenerationTimeoutError from opencollab.bootstrap import build_session from opencollab.domain.agent import Agent from tests.runtime.test_async_compaction import history from tests.support.environment_test_support import local_environment_factory as local_environment_factory +from tests.support.provider_sdk_http import completion_http_response, install_sdk_transport + + +class _LegacyPooledClient(LLMClient): + @wraps(LLMClient.complete) + async def complete(self, messages, tools=None, temperature=0.0, *, max_output_tokens=None): + return await super().complete(messages, tools, temperature, max_output_tokens=max_output_tokens) + + +class _ForwardingPooledClient(_LegacyPooledClient): + @wraps(_LegacyPooledClient.complete) + async def complete(self, *args, **kwargs): + return await super().complete(*args, **kwargs) + + +class _DelegatingPooledClient(_LegacyPooledClient): + @wraps(_LegacyPooledClient.complete) + async def complete(self, *args, **kwargs): + return await self.delegate(*args, **kwargs) + + +class _AdmissionMiddleware(_LegacyPooledClient): + supports_request_admission = True + + @wraps(_LegacyPooledClient.complete) + async def complete(self, *args, **kwargs): + on_admitted = kwargs.pop("on_admitted", None) + return await LLMClient.complete(self, *args, on_admitted=on_admitted, **kwargs) + + +class _PreparingPooledClient(_LegacyPooledClient): + @wraps(_LegacyPooledClient.complete) + async def complete(self, *args, **kwargs): + await asyncio.sleep(0.15) + return await LLMClient.complete(self, *args, **kwargs) + + +class _AdmissionProxy: + __slots__ = ("client",) + + def __init__(self, client): + self.client = client + + def __getattr__(self, name): + return getattr(self.client, name) + + +class _VariadicLegacyRoute(LLMClient): + async def complete(self, *args, **kwargs): + return await self.delegate(*args, **kwargs) + + +class _MetadataLegacyRoute(_VariadicLegacyRoute): + @wraps(LLMClient.complete) + async def complete(self, *args, **kwargs): + return await super().complete(*args, **kwargs) + + +class _ExplicitAdmissionRoute(_DelegatingPooledClient): + @wraps(_LegacyPooledClient.complete) + async def complete(self, *args, on_admitted=None, **kwargs): + return await self.delegate(*args, on_admitted=on_admitted, **kwargs) + + +def _legacy_completion_signature(self, messages, tools=None, temperature=0.0, *, max_output_tokens=None): + pass + + +class _MetadataOnlyPooledRoute(LLMClient): + @wraps(_legacy_completion_signature) + async def complete(self, *args, **kwargs): + return await self.delegate(*args, **kwargs) + + +@pytest.fixture( + params=[LLMClient, _AdmissionMiddleware, _ForwardingPooledClient], + ids=["native", "decorated", "inherited"], +) +def pooled_client_class(request): + return request.param @pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("proxy", [False, True], ids=["direct-client", "attribute-proxy"]) async def test_queue_wait_leaves_the_full_provider_timeout( - tmp_path, monkeypatch, local_environment_factory, summary, + tmp_path, monkeypatch, local_environment_factory, summary, proxy, pooled_client_class, ): calls = [] @@ -27,11 +113,11 @@ async def complete(*_args, **_kwargs): return LLMResponse(content="Earlier context." if summary and len(calls) == 1 else "answer") monkeypatch.setattr(transport, "complete_openai", complete) - async with LLMClient( + async with pooled_client_class( api_key="local", request_pool=str(tmp_path), request_capacity=1, context_window=40_000, ) as client: session = build_session( - agent=Agent(name="queued", system_prompt="system"), llm=client, + agent=Agent(name="queued", system_prompt="system"), llm=_AdmissionProxy(client) if proxy else client, env=local_environment_factory(str(tmp_path)), llm_timeout=0.1, ) session.messages = history(summary) @@ -47,7 +133,7 @@ async def complete(*_args, **_kwargs): @pytest.mark.parametrize("summary", [False, True]) async def test_admitted_timeout_retains_slot_and_late_usage_until_cleanup( - tmp_path, monkeypatch, local_environment_factory, summary, + tmp_path, monkeypatch, local_environment_factory, summary, pooled_client_class, ): cancelled = asyncio.Event() release = asyncio.Event() @@ -64,7 +150,7 @@ async def complete(*_args, **_kwargs): return LLMResponse(content="Earlier context.", usage=Usage(13, 24)) monkeypatch.setattr(transport, "complete_openai", complete) - async with LLMClient( + async with pooled_client_class( api_key="local", request_pool=str(tmp_path), request_capacity=1, context_window=40_000, ) as client: session = build_session( @@ -99,7 +185,7 @@ async def complete(*_args, **_kwargs): @pytest.mark.parametrize("summary", [False, True]) @pytest.mark.parametrize("stop", ["cancel", "deadline"]) async def test_outer_cancellation_removes_the_waiter_before_admission( - tmp_path, monkeypatch, local_environment_factory, summary, stop, + tmp_path, monkeypatch, local_environment_factory, summary, stop, pooled_client_class, ): calls = [] @@ -108,7 +194,7 @@ async def complete(*_args, **_kwargs): return LLMResponse(content="answer") monkeypatch.setattr(transport, "complete_openai", complete) - async with LLMClient( + async with pooled_client_class( api_key="local", request_pool=str(tmp_path), request_capacity=1, context_window=40_000, ) as client: session = build_session( @@ -125,8 +211,9 @@ async def complete(*_args, **_kwargs): with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(owner, 1) else: - with pytest.raises(asyncio.TimeoutError): + with pytest.raises(asyncio.TimeoutError) as timeout: await asyncio.wait_for(owner, 0.12) + assert type(timeout.value) is asyncio.TimeoutError await asyncio.wait_for(asyncio.gather(*session.pending_cleanup_tasks, return_exceptions=True), 1) await asyncio.sleep(0.1) assert calls == [] @@ -134,3 +221,428 @@ async def complete(*_args, **_kwargs): assert session.pending_cleanup_tasks == () assert (await asyncio.wait_for(client.complete([]), 1)).content == "answer" await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("proxy", [False, True], ids=["direct-client", "attribute-proxy"]) +async def test_legacy_pooled_forwarder_keeps_its_wrapped_signature( + tmp_path, monkeypatch, local_environment_factory, summary, proxy, +): + calls = [] + + async def complete(*_args, **_kwargs): + calls.append(True) + return LLMResponse( + content="Earlier context." if summary and len(calls) == 1 else "answer", + usage=Usage(7, 3), + ) + + monkeypatch.setattr(transport, "complete_openai", complete) + async with _ForwardingPooledClient( + api_key="local", request_pool=str(tmp_path), request_capacity=1, context_window=40_000, + ) as client: + assert client.supports_request_admission is True + session = build_session( + agent=Agent(name="legacy", system_prompt="system"), llm=_AdmissionProxy(client) if proxy else client, + env=local_environment_factory(str(tmp_path)), llm_timeout=1, + ) + session.messages = history(summary) + try: + assert await session.run_loop() == "answer" + assert len(calls) == (2 if summary else 1) + assert session.used_tokens == 10 * len(calls) + finally: + await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +async def test_inherited_admission_allows_middleware_preparation_before_pool( + tmp_path, monkeypatch, local_environment_factory, summary, +): + calls = [] + + async def complete(*_args, **_kwargs): + calls.append(True) + return LLMResponse(content="Earlier context." if summary and len(calls) == 1 else "answer") + + monkeypatch.setattr(transport, "complete_openai", complete) + async with _PreparingPooledClient( + api_key="local", request_pool=str(tmp_path), request_capacity=1, context_window=40_000, + ) as client: + session = build_session( + agent=Agent(name="middleware", system_prompt="system"), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.08, + ) + session.messages = history(summary) + try: + assert await asyncio.wait_for(session.run_loop(), 2) == "answer" + assert len(calls) == (2 if summary else 1) + finally: + await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("route", ["same-pool", "different-pool", "unpooled"]) +async def test_routed_native_completion_keeps_the_generation_deadline( + tmp_path, monkeypatch, local_environment_factory, summary, route, +): + entered = asyncio.Event() + + async def complete(*_args, **_kwargs): + entered.set() + await asyncio.Event().wait() + + monkeypatch.setattr(transport, "complete_openai", complete) + origin_pool = str(tmp_path) + (tmp_path / "recipient").mkdir() + recipient_pool = { + "same-pool": origin_pool, + "different-pool": str(tmp_path / "recipient"), + "unpooled": None, + }[route] + recipient_capacity = None if recipient_pool is None else 1 + config = dict(api_key="local", context_window=40_000) + async with ( + _DelegatingPooledClient(**config, request_pool=origin_pool, request_capacity=1) as client, + LLMClient(**config, request_pool=recipient_pool, request_capacity=recipient_capacity) as recipient, + ): + async def delegate(messages, *args, **kwargs): + return await recipient.complete(deepcopy(messages), *args, **kwargs) + + client.delegate = delegate + session = build_session( + agent=Agent(name="routed", system_prompt="system"), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.08, + ) + session.messages = history(summary) + try: + async with request_slot(recipient_pool, recipient_capacity): + owner = asyncio.create_task(session.run_loop()) + if recipient_pool is not None: + await asyncio.sleep(0.13) + assert not owner.done() and not entered.is_set() + await asyncio.wait_for(entered.wait(), 1) + with pytest.raises(GenerationTimeoutError): + await asyncio.wait_for(owner, 1) + finally: + await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("stall", [False, True]) +@pytest.mark.parametrize( + "route_class", [_VariadicLegacyRoute, _DelegatingPooledClient, _ExplicitAdmissionRoute], + ids=["opaque", "legacy-metadata", "explicit-admission"], +) +async def test_routed_rewrite_preserves_deadline_and_separate_preflight_usage( + tmp_path, monkeypatch, local_environment_factory, summary, stall, route_class, +): + ready = asyncio.Event() + entered = asyncio.Event() + calls = [] + preflight_limits = [] + target_limit = 64 if route_class is _DelegatingPooledClient else 128 + reserved = 0 + + async def complete( + _client, model, messages, *_args, max_output_tokens, before_attempt, on_attempt_usage, **_kwargs, + ): + if model == "preflight" or messages[-1]["content"] == "Preflight": + preflight_limits.append(max_output_tokens) + return LLMResponse(content="ok", usage=Usage(20, 30)) + calls.append(True) + assert messages[-1]["content"] == "Routed request" + assert type(max_output_tokens) is int and max_output_tokens == target_limit + assert all(type(message) is dict for message in messages) + entered.set() + if stall: + await asyncio.Event().wait() + assert before_attempt() == target_limit + previously_used = session.used_tokens + on_attempt_usage(Usage(2, 1)) + assert session.used_tokens == previously_used + 3 + session.max_budget_tokens = session.used_tokens + reserved + 31 + try: + assert before_attempt() == 31 + finally: + session.max_budget_tokens = 20_000 + content = "Earlier context." if summary and len(calls) == 1 else "answer" + return LLMResponse(content=content, usage=Usage(7, 3)) + + monkeypatch.setattr(transport, "complete_openai", complete) + destination = tmp_path / "target" + destination.mkdir() + config = dict(api_key="local", context_window=40_000, request_capacity=1) + async with ( + route_class(**config, request_pool=str(tmp_path)) as client, + LLMClient(**config, request_pool=str(destination)) as recipient, + LLMClient(api_key=__name__, model="preflight") as helper, + ): + async def delegate(messages, *args, **kwargs): + nonlocal reserved + reserved = client.estimate_request_tokens(messages, kwargs.get("tools")) + if route_class is _ExplicitAdmissionRoute: + await helper.complete(deepcopy(messages)) + else: + await LLMClient.complete(client, [{"role": "user", "content": "Preflight"}]) + ready.set() + rewritten = deepcopy(messages) + [{"role": "system", "content": "Routed request"}] + if route_class is _DelegatingPooledClient: + kwargs.pop("temperature", None) + kwargs["max_output_tokens"] = min(int(kwargs["max_output_tokens"]), target_limit) + return await recipient.complete(rewritten, *args, **kwargs) + + client.delegate = delegate + session = build_session( + agent=Agent(name="routed", system_prompt="system", max_tokens_per_step=128), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.09, max_budget_tokens=20_000, + ) + session.messages = history(summary) + try: + async with request_slot(str(destination), 1): + owner = asyncio.create_task(session.run_loop()) + await asyncio.wait_for(ready.wait(), 1) + await asyncio.sleep(0.15) + assert not owner.done() and not entered.is_set() + assert session.used_tokens == 0 + assert preflight_limits == [None] + await asyncio.wait_for(entered.wait(), 1) + if stall: + with pytest.raises(GenerationTimeoutError): + await asyncio.wait_for(owner, 1) + else: + assert await asyncio.wait_for(owner, 2) == "answer" + assert len(calls) == (2 if summary else 1) + assert session.used_tokens == 10 * len(calls) + finally: + if not owner.done(): + owner.cancel() + await asyncio.gather(owner, return_exceptions=True) + await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("stall", [False, True]) +async def test_legacy_router_normalizes_messages_and_forwards_admission( + tmp_path, monkeypatch, local_environment_factory, summary, stall, +): + helper_limits = [] + target_limits = [] + ready = asyncio.Event() + reserved_tokens = 0 + + async def handler(request): + body = json.loads(request.content) + if body["model"] == "preflight": + helper_limits.append(body.get("max_tokens")) + return completion_http_response(request, output_tokens=55) + assert body["messages"][0]["content"].endswith("\nUse the answer format.") + target_limits.append(body.get("max_tokens")) + if stall: + await asyncio.Event().wait() + content = "Earlier context." if summary else "answer" + return completion_http_response(request, text=content) + + install_sdk_transport(monkeypatch, "openai", handler) + config = dict(api_key=__name__, context_window=40_000, max_retries=0) + pool = dict(request_pool=str(tmp_path), request_capacity=1) + async with ( + _MetadataOnlyPooledRoute(**config, **pool) as client, + LLMClient(**config, **pool) as recipient, + LLMClient(**config, model="preflight") as helper, + ): + async def delegate(messages, *, max_output_tokens=None, on_admitted=None, **kwargs): + nonlocal reserved_tokens + reserved_tokens = client.estimate_request_tokens(messages, kwargs.get("tools")) + await helper.complete(deepcopy(messages)) + rewritten = json.loads(json.dumps(messages)) + rewritten[0]["content"] += "\nUse the answer format." + ready.set() + return await recipient.complete( + rewritten, max_output_tokens=max_output_tokens, on_admitted=on_admitted, **kwargs, + ) + + client.delegate = delegate + session = build_session( + agent=Agent(name="router", system_prompt="system", max_tokens_per_step=128), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.10, max_budget_tokens=20_000, + ) + await session.add_user_message("Prepare an answer.") + owner = None + try: + async with request_slot(str(tmp_path), 1): + operation = ( + session._context_summarizer.asummarize(session.messages) if summary else session.run_loop() + ) + owner = asyncio.create_task(operation) + await asyncio.wait_for(ready.wait(), 2) + await asyncio.sleep(0.15) + assert not owner.done() + assert not target_limits + assert session.used_tokens == 0 + assert helper_limits == [None] + session.max_budget_tokens = reserved_tokens + 31 + if stall: + with pytest.raises(GenerationTimeoutError): + await asyncio.wait_for(owner, 2) + else: + expected = "Summary:\nEarlier context." if summary else "answer" + assert await asyncio.wait_for(owner, 2) == expected + assert session.used_tokens == 4 + assert target_limits == [31] + finally: + if owner is not None: + if not owner.done(): + owner.cancel() + await asyncio.gather(owner, return_exceptions=True) + await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("client_class", [_VariadicLegacyRoute, _MetadataLegacyRoute]) +async def test_routed_legacy_completion_uses_response_usage( + tmp_path, local_environment_factory, summary, client_class, +): + calls = [] + + async def complete(messages, tools=None, temperature=0.0, *, max_output_tokens=None): + calls.append(messages) + content = "Earlier context." if summary and len(calls) == 1 else "answer" + return LLMResponse(content=content, usage=Usage(7, 3)) + + config = dict(api_key="local", context_window=40_000) + async with client_class(**config) as client: + client.delegate = complete + session = build_session( + agent=Agent(name="routed", system_prompt="system"), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=1, + ) + session.messages = history(summary) + try: + assert await session.run_loop() == "answer" + assert len(calls) == (2 if summary else 1) + assert session.used_tokens == 10 * len(calls) + finally: + await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("bypass", ["returns", "hangs", "unrelated", "nonpool"]) +async def test_inherited_admission_waiting_preserves_completion_and_caller_deadline( + tmp_path, monkeypatch, local_environment_factory, summary, bypass, +): + calls = [] + provider_calls = [] + + async def complete(*_args, **_kwargs): + provider_calls.append(True) + await asyncio.Event().wait() + + async def delegate(messages, tools=None, temperature=0.0, *, max_output_tokens=None): + calls.append(True) + if bypass == "returns": + content = "Earlier context." if summary and len(calls) == 1 else "answer" + return LLMResponse(content=content, usage=Usage(7, 3)) + if bypass == "unrelated": + return await other.complete([{"role": "user", "content": "Unrelated request"}]) + await asyncio.Event().wait() + + monkeypatch.setattr(transport, "complete_openai", complete) + config = dict(api_key="local", context_window=40_000) + pool_config = dict(request_pool=str(tmp_path), request_capacity=1) + client_pool = {} if bypass == "nonpool" else pool_config + async with _DelegatingPooledClient(**config, **client_pool) as client, LLMClient(**config, **pool_config) as other: + client.delegate = delegate + session = build_session( + agent=Agent(name="legacy", system_prompt="system"), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.1, + ) + session.messages = history(summary) + try: + if bypass == "returns": + assert await asyncio.wait_for(session.run_loop(), 2) == "answer" + assert len(calls) == (2 if summary else 1) + assert session.used_tokens == 10 * len(calls) + else: + expected_timeout = GenerationTimeoutError if bypass == "nonpool" else asyncio.TimeoutError + with pytest.raises(expected_timeout) as timeout: + await asyncio.wait_for(session.run_loop(), 0.25) + assert type(timeout.value) is expected_timeout + assert calls == [True] + assert session.used_tokens == 0 + await asyncio.gather(*session.pending_cleanup_tasks, return_exceptions=True) + assert provider_calls == ([True] if bypass == "unrelated" else []) + finally: + await session.aclose() + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("provider_error", [False, True]) +async def test_blocked_ledger_releases_slot_and_preserves_late_session_usage( + tmp_path, monkeypatch, local_environment_factory, summary, provider_error, +): + writing, release = threading.Event(), threading.Event() + second_started = asyncio.Event() + events, calls = [], [] + log_path = tmp_path / "usage.jsonl" + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(log_path)) + original_record = transport.record_api_usage + + def delayed_record(**kwargs): + if not writing.is_set(): + writing.set() + assert release.wait(5) + original_record(**kwargs) + + async def complete(*_args, **_kwargs): + calls.append(True) + if len(calls) == 2: + second_started.set() + return LLMResponse(content="next", usage=Usage(2, 2)) + usage = Usage(13, 24) + if provider_error: + exc = ValueError("provider disconnected") + exc.usage = usage + raise exc + return LLMResponse(content="Earlier context.", usage=usage) + + monkeypatch.setattr(transport, "record_api_usage", delayed_record) + monkeypatch.setattr(transport, "complete_openai", complete) + async with LLMClient( + api_key="local", request_pool=str(tmp_path), request_capacity=1, context_window=40_000, + ) as client: + session = build_session( + agent=Agent(name="blocked-ledger", system_prompt="system"), llm=client, + env=local_environment_factory(str(tmp_path)), llm_timeout=0.2, + run_control=RunControl(on_event=lambda event: events.append(event) if event.type == "usage" else None), + ) + session.messages = history(summary) + owner = asyncio.create_task(session.run_loop()) + following = None + try: + assert await asyncio.to_thread(writing.wait, 5) + assert session.used_tokens == 37 + following = asyncio.create_task(client.complete([])) + await asyncio.wait_for(second_started.wait(), 2) + with pytest.raises(GenerationTimeoutError): + await asyncio.wait_for(owner, 2) + assert session.pending_cleanup_tasks + assert len(calls) == 2 + finally: + release.set() + await asyncio.gather(owner, *session.pending_cleanup_tasks, return_exceptions=True) + if following is not None: + assert (await asyncio.wait_for(following, 2)).content == "next" + await session.aclose() + + assert session.used_tokens == 37 + assert session.runner.late_provider_usage == (37,) + assert not session.pending_cleanup_tasks + assert len(events) == 1 + assert events[0].data["purpose"] == ("summary" if summary else "completion") + assert events[0].data["total_tokens"] == 37 + assert events[0].data["late"] is True + rows = [json.loads(line) for line in log_path.read_text().splitlines()] + assert [(row["status"], row["usage"]["total_tokens"]) for row in rows] == [ + ("error" if provider_error else "success", 37), ("success", 4), + ] diff --git a/tests/runtime/test_pooled_request_budget_http.py b/tests/runtime/test_pooled_request_budget_http.py new file mode 100644 index 00000000..885ac0cb --- /dev/null +++ b/tests/runtime/test_pooled_request_budget_http.py @@ -0,0 +1,389 @@ +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters.llm import client as transport +from opencollab.adapters.llm.client import LLMClient +from opencollab.adapters.llm.request_admission import request_slot +from opencollab.application._session_run_shared import _TeamBudgetStop, _TokenBudgetStop +from opencollab.application.run_control import _RunCancelled +from opencollab.bootstrap import build_session +from opencollab.domain.agent import Agent +from tests.runtime.test_llm_chat_error_usage_http import ReportedUsageStream +from tests.support.environment_test_support import local_environment_factory as local_environment_factory +from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport + + +@pytest.fixture(autouse=True) +def isolated_configuration(monkeypatch, tmp_path): + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + + +async def _wait_until(predicate): + async def wait(): + while not predicate(): + await asyncio.sleep(0) + await asyncio.wait_for(wait(), 5) + + +@pytest.mark.parametrize("method", ["agent", "agent_run"]) +@pytest.mark.parametrize("first_tokens, total, slow_ledger", [ + (5_000, 20_000, False), + ("reduced", 20_000, False), + (19_990, 20_000, False), + (20_210, 20_000, False), + (20_210, 20_000, True), + (20_210, 20_000, "handoff"), + (20_210, 20_000, "error_usage"), + (20_210, None, False), +]) +async def test_native_queue_rechecks_shared_workflow_budget( + tmp_path, monkeypatch, method, first_tokens, total, slow_ledger, +): + started, release = asyncio.Event(), asyncio.Event() + contexts, requests, limits = [], [], [] + + async def handler(request): + body = json.loads(request.content) + requests.append((body, contexts[0].tokens_spent())) + if len(requests) == 1: + started.set() + await release.wait() + if slow_ledger == "error_usage": + return http_response( + request, 200, headers={"content-type": "text/event-stream"}, + stream=ReportedUsageStream( + {"prompt_tokens": first_tokens - 10, "completion_tokens": 10, "total_tokens": first_tokens}, + disconnect=True, + ), + ) + response = completion_http_response(request, text="first answer", output_tokens=10) + payload = response.json() + payload["usage"].update(prompt_tokens=first_tokens - 10, total_tokens=first_tokens) + return http_response(request, 200, json=payload) + return completion_http_response(request, text="second answer") + + install_sdk_transport(monkeypatch, "openai", handler) + record = transport._record_api_usage_async + + async def delayed_record(**kwargs): + if slow_ledger is True or slow_ledger == "error_usage": + await asyncio.sleep(0.12) + await record(**kwargs) + + monkeypatch.setattr(transport, "_record_api_usage_async", delayed_record) + if slow_ledger == "handoff": + released_slot = asyncio.Event() + native_slot = transport.request_slot + entries = 0 + + @asynccontextmanager + async def ordered_slot(*args): + nonlocal entries + index = entries + entries += 1 + if index: + await released_slot.wait() + async with native_slot(*args): + yield + if not index: + released_slot.set() + + monkeypatch.setattr(transport, "request_slot", ordered_slot) + pool = tmp_path / "pool" + pool.mkdir() + async with LLMClient( + model="gpt-4o", api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", request_pool=str(pool), request_capacity=1, + max_retries=0, + stream_chat=slow_ledger == "error_usage", + ) as client: + async def flow(ctx, _args): + nonlocal first_tokens + contexts.append(ctx) + run = getattr(ctx, method) + tasks = [asyncio.create_task(run("first", budget=18_000, tools=[]))] + try: + await asyncio.wait_for(started.wait(), 5) + tasks.append(asyncio.create_task(run("second", budget=2_000, tools=[]))) + await _wait_until(lambda: len(ctx.sessions) == 2 and ctx.sessions[1].runner._llm_step_started) + assert len(requests) == 1 + second = ctx.sessions[1] + reserved = client.estimate_request_tokens(second.messages) + if first_tokens == "reduced": + first_tokens = total - reserved - 50 + expected = min(128, (2_000 if total is None else min(2_000, total - first_tokens)) - reserved) + limits.append(expected) + release.set() + values = await asyncio.wait_for(asyncio.gather(*tasks), 5) + if expected >= 1: + assert getattr(values[1], "output", values[1]) == "second answer" + assert requests[1][0]["max_tokens"] == expected + assert requests[1][1] == first_tokens + else: + assert getattr(values[1], "output", values[1]) == "" + assert "budget exhausted before model call" in second.state.terminal_reason + return "finished" + finally: + release.set() + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + result = await OpenCollab(tmp_path, model="gpt-4o", config={"max_output_tokens": 128}).workflow( + flow, llm=client, budget=total, concurrency=2, trace=False, limit_mode="explicit", + ) + assert result.status == "completed", result.reason + expected_calls = 2 if limits[0] >= 1 else 1 + assert len(requests) == expected_calls + assert result.tokens == first_tokens + (4 if expected_calls == 2 else 0) + assert contexts[0].budget._leases == [] + assert contexts[0].pending_cleanup_tasks == () + + +async def test_native_overspend_retains_requests_that_already_started(tmp_path, monkeypatch): + first_started, second_started = asyncio.Event(), asyncio.Event() + release_first, release_second = asyncio.Event(), asyncio.Event() + requests, contexts = [], [] + + async def handler(request): + body = json.loads(request.content) + prompt = body["messages"][-1]["content"].splitlines()[0] + requests.append(prompt) + if prompt == "first": + first_started.set() + await release_first.wait() + payload = completion_http_response(request, text="first answer", output_tokens=10).json() + payload["usage"].update(prompt_tokens=20_200, total_tokens=20_210) + return http_response(request, 200, json=payload) + second_started.set() + await release_second.wait() + return completion_http_response(request, text="second answer") + + install_sdk_transport(monkeypatch, "openai", handler) + pool = tmp_path / "pool" + pool.mkdir() + async with LLMClient( + model="gpt-4o", api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", request_pool=str(pool), request_capacity=2, + ) as client: + async def flow(ctx, _args): + contexts.append(ctx) + tasks = [asyncio.create_task(ctx.agent("first", tools=[], budget=18_000))] + try: + await asyncio.wait_for(first_started.wait(), 5) + tasks.append(asyncio.create_task(ctx.agent("second", tools=[], budget=2_000))) + await asyncio.wait_for(second_started.wait(), 5) + release_first.set() + assert await asyncio.wait_for(tasks[0], 5) == "first answer" + assert ctx.tokens_spent() == 20_210 + assert not tasks[1].done() + release_second.set() + assert await asyncio.wait_for(tasks[1], 5) == "second answer" + return "finished" + finally: + release_first.set() + release_second.set() + await asyncio.gather(*tasks, return_exceptions=True) + + result = await OpenCollab(tmp_path, model="gpt-4o", config={"max_output_tokens": 128}).workflow( + flow, llm=client, budget=20_000, concurrency=2, trace=False, limit_mode="explicit", + ) + assert result.status == "completed", result.reason + assert requests == ["first", "second"] + assert result.tokens == 20_214 + assert contexts[0].pending_cleanup_tasks == () + + +@pytest.mark.parametrize("configuration, field", [ + ({"provider": "openai", "wire_protocol": "chat_completions", "model": "gpt-4o"}, "max_tokens"), + ({"provider": "openai", "wire_protocol": "responses", "model": "gpt-4o"}, "max_output_tokens"), + ({"provider": "anthropic", "model": "claude-sonnet-4-6"}, "max_tokens"), +], ids=["chat", "responses", "anthropic"]) +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("change", ["reduced", "exhausted", "unbounded", "cancelled", "team"]) +async def test_admission_uses_current_session_limits_and_cancellation( + tmp_path, monkeypatch, local_environment_factory, configuration, field, summary, change, +): + requests = [] + + async def handler(request): + requests.append(json.loads(request.content)) + return completion_http_response(request) + + install_sdk_transport(monkeypatch, configuration["provider"], handler) + pool = tmp_path / "pool" + pool.mkdir() + team_exhausted, cancel = asyncio.Event(), asyncio.Event() + async with LLMClient( + **configuration, api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", request_pool=str(pool), request_capacity=1, + max_retries=0, + ) as client: + session = build_session( + agent=Agent(name="queued", system_prompt="system", max_tokens_per_step=128, **configuration), + llm=client, env=local_environment_factory(str(tmp_path)), max_budget_tokens=20_000, + team_budget_exhausted=team_exhausted.is_set, default_cancel_event=cancel, + ) + messages = [{"role": "user", "content": "Finish the task."}] + reserved = client.estimate_request_tokens(messages) + session.messages = messages + queued = asyncio.Event() + native_slot = transport.request_slot + + @asynccontextmanager + async def observed_slot(*args): + queued.set() + async with native_slot(*args): + yield + + monkeypatch.setattr(transport, "request_slot", observed_slot) + + async def complete_summary(messages, *, on_response, **kwargs): + response = await client.complete(messages, **kwargs) + on_response(response) + return response + + try: + async with request_slot(str(pool), 1): + operation = session.runner._invoke_summary( + complete_summary, messages, _summary_output_limit=lambda _: 80, + ) if summary else session.run_loop() + task = asyncio.create_task(operation) + await asyncio.wait_for(queued.wait(), 5) + assert requests == [] + if change == "reduced": + session.max_budget_tokens = reserved + 50 + elif change == "exhausted": + session.used_tokens = 20_000 + elif change == "unbounded": + session.max_budget_tokens = None + elif change == "cancelled": + cancel.set() + else: + team_exhausted.set() + if summary and change in {"exhausted", "cancelled", "team"}: + error = {"exhausted": _TokenBudgetStop, "cancelled": _RunCancelled, "team": _TeamBudgetStop}[change] + with pytest.raises(error): + await asyncio.wait_for(task, 5) + else: + answer = await asyncio.wait_for(task, 5) + if change in {"reduced", "unbounded"}: + assert getattr(answer, "content", answer) == "ok" + else: + assert answer == "" + if change in {"reduced", "unbounded"}: + expected = 50 if change == "reduced" else (80 if summary else 128) + assert len(requests) == 1 and requests[0][field] == expected + assert session.used_tokens == 4 + else: + assert requests == [] + assert session.used_tokens == (20_000 if change == "exhausted" else 0) + finally: + await session.aclose() + + +async def test_nested_team_workflow_charges_parent_before_other_native_admission( + tmp_path, monkeypatch, local_environment_factory, +): + from opencollab.application.event_bus import EventBus + from opencollab.application.scheduler import Scheduler + from opencollab.application.tool_execution import ToolRuntime + from opencollab.application.workflow_registry import Registry, WorkflowSpec + from opencollab.bootstrap.context_builder import SpawnConfig + from opencollab.bootstrap.team_workflow import TeamWorkflowTool + from opencollab.domain.scheduler import SessionControlBlock + + provider_started, release_provider, queued, ledger_started, release_ledger = ( + asyncio.Event() for _ in range(5) + ) + requests = [] + + async def reply(request): + requests.append(json.loads(request.content)) + provider_started.set() + await release_provider.wait() + payload = completion_http_response(request, text="child answer").json() + payload["usage"].update(prompt_tokens=20_198, total_tokens=20_200) + return http_response(request, 200, json=payload) + + async def flow(ctx, _args): + return await ctx.agent("child request", tools=[]) + + native_slot, record = transport.request_slot, transport._record_api_usage_async + entries = 0 + + @asynccontextmanager + async def observed_slot(*args): + nonlocal entries + entries += 1 + if entries == 2: + queued.set() + async with native_slot(*args): + yield + + async def delayed_record(**kwargs): + if kwargs["status"] == "success": + ledger_started.set() + await release_ledger.wait() + await record(**kwargs) + + install_sdk_transport(monkeypatch, "openai", reply) + monkeypatch.setattr(transport, "request_slot", observed_slot) + monkeypatch.setattr(transport, "_record_api_usage_async", delayed_record) + scheduler = Scheduler(session_factory=None, worktree_pool=None, event_sink=EventBus(), max_budget_tokens=20_000) + registry = Registry() + registry.register(WorkflowSpec(name="registered", description="Return a result", fn=flow)) + env = local_environment_factory(str(tmp_path)) + async with LLMClient( + api_key="local-test", request_pool=str(tmp_path), request_capacity=1, max_retries=0, + ) as client: + parent, sibling = [ + build_session( + env=env, llm=client, max_budget_tokens=30_000, + agent=Agent(name=name, system_prompt="system", max_tokens_per_step=128), + team_budget_exhausted=lambda: scheduler.budget_exhausted, + ) for name in ("parent", "sibling") + ] + for aid, session in enumerate((parent, sibling)): + scheduler.table.add(SessionControlBlock( + aid=aid, parent_aid=0 if aid else None, agent=session.agent, state=session.state, + )) + config = SpawnConfig(model="gpt-4o", provider="openai", api_key=None, base_url=None, + llm_timeout=5, tracer=None, event_bus=EventBus(), permission_policy=None) + tool = TeamWorkflowTool(lambda: parent, registry, config, None) + active = asyncio.create_task(tool.execute_with_runtime( + {"name": "registered", "args": {}}, ToolRuntime(env, None, None), + )) + following = None + try: + await asyncio.wait_for(provider_started.wait(), 5) + await sibling.add_user_message("queued request") + following = asyncio.create_task(sibling.run_loop()) + await asyncio.wait_for(queued.wait(), 5) + release_provider.set() + await asyncio.wait_for(ledger_started.wait(), 5) + assert parent.used_tokens == scheduler.used_tokens == 20_200 + assert scheduler.budget_exhausted + assert await asyncio.wait_for(following, 5) == "" + assert "team budget exceeded" in sibling.state.terminal_reason + assert not active.done() + assert len(requests) == 1 + release_ledger.set() + result = json.loads(await asyncio.wait_for(active, 5)) + assert result["tokens"] == parent.used_tokens == 20_200 + assert result["output"] == "child answer" + finally: + release_provider.set() + release_ledger.set() + await asyncio.gather(active, *([following] if following is not None else []), return_exceptions=True) + await sibling.aclose() + await parent.aclose() diff --git a/tests/runtime/test_provider_attempt_budget_http.py b/tests/runtime/test_provider_attempt_budget_http.py new file mode 100644 index 00000000..e76d8590 --- /dev/null +++ b/tests/runtime/test_provider_attempt_budget_http.py @@ -0,0 +1,441 @@ +from __future__ import annotations + +import asyncio +import json +import socket +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +from opencollab import OpenCollab, RunControl +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm import retry +from opencollab.adapters.llm.client import LLMClient +from opencollab.application.event_bus import EventBus +from opencollab.application.scheduler import Scheduler +from opencollab.bootstrap import build_session +from opencollab.domain.agent import Agent +from opencollab.domain.scheduler import SessionControlBlock +from opencollab.domain.session import SessionPhase, SessionState +from tests.interfaces.test_member_authored_workflow import _manager +from tests.support.environment_test_support import local_environment_factory as local_environment_factory +from tests.support.paths import REPO_ROOT + + +def _event(value): + return f"data: {json.dumps(value)}\n\n".encode() + + +@pytest.fixture +def attempt_server(monkeypatch, tmp_path): + servers = [] + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + monkeypatch.delenv("OPENCOLLAB_EXTERNAL_PROVIDER_ISOLATION", raising=False) + monkeypatch.setattr(retry, "extract_retry_after_seconds", lambda _error: 0.0) + monkeypatch.setattr(retry.random, "uniform", lambda _lo, _hi: 0.0) + + def start(protocol, observe=lambda: None): + requests = [] + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + requests.append({"body": body, "observed": observe()}) + first = len(requests) == 1 + total = 1500 if first else 4 + status = 200 + content_type = "text/event-stream" + if protocol == "chat": + common = {"id": "chat_attempt", "object": "chat.completion.chunk", + "created": 1, "model": body["model"]} + payload = b"".join([ + _event({**common, "choices": [{"index": 0, "delta": { + "role": "assistant", "content": "ok"}, "finish_reason": None}]}), + _event({**common, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}), + _event({**common, "choices": [], "usage": { + "prompt_tokens": total - 2, "completion_tokens": 2, "total_tokens": total}}), + ]) + if not first: + payload += b"data: [DONE]\n\n" + elif protocol == "responses": + item = {"id": "msg_attempt", "type": "message", "role": "assistant", + "status": "completed", "content": [{"type": "output_text", "text": "ok", + "annotations": []}]} + response = {"id": "resp_attempt", "object": "response", "model": body["model"], + "status": "failed" if first else "completed", "output": [] if first else [item], + "error": {"code": "server_error", "message": "Temporarily unavailable"} + if first else None, "incomplete_details": None, + "usage": {"input_tokens": total - 2, "output_tokens": 2, "total_tokens": total}} + payload = _event({"type": "response.failed", "response": response}) if first else b"".join([ + _event({"type": "response.output_item.done", "output_index": 0, "item": item}), + _event({"type": "response.completed", "response": response}), + ]) + else: + content_type = "application/json" + status = 503 if first else 200 + payload = json.dumps({"type": "error", "error": { + "type": "overloaded_error", "message": "Temporarily unavailable"}} + if first else {"id": "msg_attempt", "type": "message", "role": "assistant", + "model": body["model"], "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", "stop_sequence": None, + "usage": {"input_tokens": 2, "output_tokens": 2}}).encode() + self.send_response(status) + self.send_header("Content-Type", content_type) + if protocol == "chat" and first: + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + self.wfile.write(f"{len(payload):x}\r\n".encode() + payload + b"\r\n") + self.wfile.flush() + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + self.close_connection = True + else: + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + self.wfile.flush() + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + servers.append((server, thread)) + return f"http://127.0.0.1:{server.server_port}/v1", requests + + yield start + for server, thread in servers: + server.shutdown() + server.server_close() + thread.join() + + +def _client(protocol, url, *, retries=1, **kwargs): + return LLMClient( + provider="anthropic" if protocol == "anthropic" else "openai", + model="claude-sonnet-4-6" if protocol == "anthropic" else "gpt-4o", + wire_protocol="responses" if protocol == "responses" else "chat_completions", + api_key="controlled-test", base_url=url, # pragma: allowlist secret + stream_chat=protocol == "chat", max_retries=retries, + **kwargs, + ) + + +def _records(tmp_path): + return [json.loads(line) for line in (tmp_path / "usage.jsonl").read_text().splitlines()] + + +@pytest.mark.parametrize("protocol", ["chat", "responses"]) +@pytest.mark.parametrize("budget, retries, count", [(2000, 1, 1), (20000, 1, 2), (None, 1, 2), (2000, 0, 1)]) +async def test_public_agent_rechecks_reported_retry_spend(tmp_path, attempt_server, protocol, budget, retries, count): + url, requests = attempt_server(protocol) + events = [] + result = await OpenCollab( + tmp_path, model="routed-worker", model_routes={"routed-worker": { + "model": "gpt-4o", "provider": "openai", "api_key": "controlled-test", # pragma: allowlist secret + "base_url": url, "wire_protocol": "responses" if protocol == "responses" else "chat_completions", + }}, config={"max_output_tokens": 128, "llm_stream_chat": protocol == "chat", "llm_max_retries": retries}, + ).agent("Reply ok.", tools=[], budget=budget, trace=False, run_control=RunControl( + on_event=lambda event: events.append(event.data) if event.type == "usage" else None, + )) + assert len(requests) == count + assert result.tokens == 1500 + 4 * (count - 1) + assert result.status == ("failed" if retries == 0 else "stopped" if count == 1 else "completed") + if count == 1 and retries: + assert "budget exhausted before model call" in result.reason + records = _records(tmp_path) + assert len(records) == len(events) == 1 + assert records[0]["usage"]["total_tokens"] == events[0]["total_tokens"] == result.tokens + + +@pytest.mark.parametrize("protocol, field", [("chat", "max_tokens"), ("responses", "max_output_tokens"), + ("anthropic", "max_tokens")]) +@pytest.mark.parametrize("change", ["reduced", "exhausted", "team", "cancel"]) +@pytest.mark.parametrize("summary", [False, True]) +async def test_physical_retry_observes_current_session_state( + tmp_path, monkeypatch, local_environment_factory, attempt_server, protocol, field, change, summary, +): + sessions = [] + url, requests = attempt_server(protocol, lambda: sessions[0].used_tokens) + team_stop, cancel = asyncio.Event(), asyncio.Event() + known = 0 if protocol == "anthropic" else 1500 + async with _client(protocol, url) as client: + session = build_session( + env=local_environment_factory(), llm=client, max_budget_tokens=20000, + agent=Agent(name="worker", model=client.model, provider=client.provider, + system_prompt="Finish the task.", max_tokens_per_step=128), + team_budget_exhausted=team_stop.is_set, default_cancel_event=cancel, + ) + sessions.append(session) + await session.add_user_message("Reply ok.") + messages = list(session.messages) + reserved = client.estimate_request_tokens(messages) + + def before_backoff(_error): + assert session.used_tokens == known + if change in {"reduced", "exhausted"}: + session.max_budget_tokens = known + reserved + (37 if change == "reduced" else 0) + elif change == "team": + team_stop.set() + else: + cancel.set() + return 0.0 + + monkeypatch.setattr(retry, "extract_retry_after_seconds", before_backoff) + + async def complete_summary(messages, *, on_response, **kwargs): + response = await client.complete(messages, **kwargs) + on_response(response) + return response + + if summary: + operation = session.runner._invoke_summary(complete_summary, messages) + if change == "reduced": + assert (await operation).content == "ok" + else: + with pytest.raises(Exception) as error: + await operation + assert type(error.value).__name__ == { + "exhausted": "_TokenBudgetStop", "team": "_TeamBudgetStop", "cancel": "_RunCancelled", + }[change] + if known: + assert error.value.usage.total_tokens == known + else: + assert await session.run_loop() == ("ok" if change == "reduced" else "") + assert session.phase is (SessionPhase.DONE if change == "reduced" else SessionPhase.STOPPED) + assert len(requests) == (2 if change == "reduced" else 1) + if change == "reduced": + assert requests[1]["body"][field] == 37 + assert requests[1]["observed"] == known + assert session.used_tokens == known + (4 if change == "reduced" else 0) + records = _records(tmp_path) + assert len(records) == 1 + assert records[0]["usage"]["total_tokens"] == session.used_tokens + + +@pytest.mark.parametrize("protocol", ["chat", "responses"]) +@pytest.mark.parametrize("callback", ["attempt", "final", "revalidate"]) +async def test_callback_errors_stop_retry_and_retain_usage(tmp_path, attempt_server, protocol, callback): + url, requests = attempt_server(protocol) + attempts, final, admissions, checks = [], [], [], [] + + def failed_callback(): + raise RuntimeError("timeout in caller callback") + + def on_attempt(usage): + attempts.append(usage.total_tokens) + if callback == "attempt": + failed_callback() + + def on_usage(usage): + final.append(usage.total_tokens) + if callback == "final": + failed_callback() + + def before_request(): + checks.append(True) + if len(checks) == 2 and callback == "revalidate": + failed_callback() + return 128 + + async with _client(protocol, url, retries=3) as client: + with pytest.raises(RuntimeError, match="timeout in caller callback") as error: + await client.complete( + [{"role": "user", "content": "Reply ok."}], before_request=before_request, + on_attempt_usage=on_attempt, on_usage=on_usage, on_admitted=lambda: admissions.append(True), + ) + expected = 1504 if callback == "final" else 1500 + assert len(requests) == (2 if callback == "final" else 1) + assert attempts == [1500] + assert final == [expected] + assert admissions == [True] + assert error.value.usage.total_tokens == expected + records = _records(tmp_path) + assert len(records) == 1 + assert records[0]["usage"]["total_tokens"] == expected + + +@pytest.mark.parametrize("callback", ["before_first", "admitted", "before_retry", "attempt", "final"]) +@pytest.mark.parametrize("timeout", [None, 600.0]) +@pytest.mark.parametrize("retry_budget", [0, 30]) +async def test_responses_preserves_callback_timeout_identity( + tmp_path, attempt_server, callback, timeout, retry_budget, +): + url, requests = attempt_server("responses") + original = TimeoutError("owned caller deadline") + checks, attempts, finals = [], [], [] + + def before_request(): + checks.append(True) + if callback == "before_first" or callback == "before_retry" and len(checks) == 2: + raise original + return 128 + + def on_admitted(): + if callback == "admitted": + raise original + + def on_attempt(usage): + attempts.append(usage.total_tokens) + if callback == "attempt": + raise original + + def on_usage(usage): + finals.append(usage.total_tokens) + if callback == "final": + raise original + + async with _client("responses", url, request_timeout=timeout, provider_error_time_budget=retry_budget) as client: + with pytest.raises(TimeoutError) as error: + await client.complete( + [{"role": "user", "content": "Reply ok."}], before_request=before_request, + on_admitted=on_admitted, on_attempt_usage=on_attempt, on_usage=on_usage, + ) + assert error.value is original + count = 0 if callback in {"before_first", "admitted"} else 2 if callback == "final" else 1 + expected = 0 if not count else 1500 + 4 * (count - 1) + assert len(requests) == count + assert attempts == ([1500] if count else []) + assert finals == ([expected] if count else []) + assert getattr(getattr(error.value, "usage", None), "total_tokens", 0) == expected + records = _records(tmp_path) + assert len(records) == 1 + assert records[0]["usage"]["total_tokens"] == expected + + +@pytest.mark.parametrize("protocol", ["chat", "responses"]) +async def test_workflow_observes_failure_spend_during_retry(tmp_path, monkeypatch, attempt_server, protocol): + contexts, seen = [], [] + url, requests = attempt_server(protocol, lambda: contexts[0].tokens_spent()) + + def backoff(_error): + seen.append(contexts[0].tokens_spent()) + return 0.0 + + monkeypatch.setattr(retry, "extract_retry_after_seconds", backoff) + async with _client(protocol, url) as client: + async def flow(ctx, _args): + contexts.append(ctx) + return await ctx.agent("Reply ok.", tools=[], budget=10000) + + result = await OpenCollab(tmp_path, config={"max_output_tokens": 128}).workflow( + flow, llm=client, budget=20000, trace=False, limit_mode="explicit", + ) + assert result.status == "completed" + assert result.tokens == 1504 + assert seen == [1500] + assert [request["observed"] for request in requests] == [0, 1500] + + +@pytest.mark.parametrize("budget, retries, count, stopped_by", [ + (2000, 1, 1, "_TokenBudgetStop"), + (20000, 1, 1, "_TeamBudgetStop"), + (20000, 1, 1, "_RunCancelled"), + (20000, 1, 2, None), + (None, 1, 2, None), + (2000, 0, 1, None), +]) +async def test_authored_subprocess_accounts_failure_before_retry( + tmp_path, monkeypatch, attempt_server, budget, retries, count, stopped_by, +): + monkeypatch.setenv("PYTHONPATH", str(REPO_ROOT)) + hosts = [] + url, requests = attempt_server("chat", lambda: (hosts[0].used_tokens, hosts[1].bridge.tokens)) + (tmp_path / "authored.py").write_text( + 'from opencollab import RunControl\n' + 'async def main(ctx, args):\n' + ' events = []\n' + ' def observe(event):\n' + ' if event.type == "usage":\n' + ' events.append([event.data["total_tokens"], event.data["error_type"],\n' + ' event.data["cache_read_tokens"]])\n' + f' goal = {"context " * 300!r}\n' + f' await ctx.agent_run(goal, tools=[], budget={budget!r}, run_control=RunControl(on_event=observe))\n' + ' return {"workflow": ctx.tokens_spent(), "session": ctx.sessions[0].used_tokens, "events": events}\n', + encoding="utf-8", + ) + environment = LocalEnvironment(str(tmp_path)) + parent, tool, runtime = _manager(environment, budget=budget) + if stopped_by == "_TeamBudgetStop": + parent.runner._team_budget_exhausted = lambda: parent.used_tokens >= 1500 + if stopped_by == "_RunCancelled": + def request_stop(_error): + hosts[1].stop_requested = True + return 0.0 + monkeypatch.setattr(retry, "extract_retry_after_seconds", request_stop) + for target in (tool._config, parent.agent): + target.base_url = url + target.api_key = "controlled-test" # pragma: allowlist secret + target.llm_stream_chat = True + target.llm_max_retries = retries + parent.agent.max_tokens_per_step = 128 + hosts.append(parent) + try: + launched = json.loads(await tool.execute_with_runtime( + {"action": "launch", "source": "authored.py", "python": sys.executable, "budget": budget}, runtime, + )) + child = tool._children[launched["id"]] + hosts.append(child) + await asyncio.wait_for(child.task, 20) + assert child.result["status"] == "completed", child.result + assert len(requests) == count + expected = 1500 + 4 * (count - 1) + assert child.tokens == child.bridge.tokens == parent.used_tokens == expected + artifacts = Path(child.artifacts) + assert json.loads((artifacts / "usage.json").read_text()) == {"tokens": expected} + result = json.loads((artifacts / "result.json").read_text()) + assert result["tokens"] == expected + output = result["output"] + assert output["workflow"] == output["session"] == expected + assert [event[0] for event in output["events"]] == [expected] + if stopped_by is not None: + assert output["events"] == [[expected, stopped_by, None]] + if count == 2: + assert requests[1]["observed"] == (1500, 1500) + records = _records(tmp_path) + assert len(records) == 1 + assert records[0]["usage"]["total_tokens"] == parent.used_tokens + finally: + await parent.aclose() + await environment.cleanup() + + +@pytest.mark.parametrize("protocol", ["chat", "responses"]) +async def test_team_observes_failure_spend_before_retry( + tmp_path, monkeypatch, local_environment_factory, attempt_server, protocol, +): + scheduler = Scheduler(session_factory=None, worktree_pool=None, event_sink=EventBus(), max_budget_tokens=2000) + sibling = SessionState(messages=[], used_tokens=750) + scheduler.table.add(SessionControlBlock( + aid=1, parent_aid=0, agent=Agent(name="sibling", system_prompt="Finish."), state=sibling, + )) + url, requests = attempt_server(protocol) + + def backoff(_error): + assert scheduler.used_tokens == 2250 + assert scheduler.budget_exhausted + return 0.0 + + monkeypatch.setattr(retry, "extract_retry_after_seconds", backoff) + async with _client(protocol, url) as client: + session = build_session( + env=local_environment_factory(), llm=client, max_budget_tokens=20000, + agent=Agent(name="worker", model="gpt-4o", system_prompt="Finish.", max_tokens_per_step=128), + team_budget_exhausted=lambda: scheduler.budget_exhausted, + ) + scheduler.table.add(SessionControlBlock(aid=0, parent_aid=None, agent=session.agent, state=session.state)) + await session.add_user_message("Reply ok.") + assert await session.run_loop() == "" + assert len(requests) == 1 + assert session.used_tokens == 1500 + assert session.phase is SessionPhase.STOPPED + assert "team budget exceeded" in session.state.terminal_reason + assert _records(tmp_path)[0]["usage"]["total_tokens"] == 1500 diff --git a/tests/runtime/test_responses_error_usage_http.py b/tests/runtime/test_responses_error_usage_http.py index 79c448cb..308609c1 100644 --- a/tests/runtime/test_responses_error_usage_http.py +++ b/tests/runtime/test_responses_error_usage_http.py @@ -529,6 +529,7 @@ def retry_after(_error): await session.add_user_message("Give a short answer") turn = asyncio.create_task(session.run_loop()) await asyncio.wait_for(backoff_started.wait(), 1) + assert session.used_tokens == 37 turn.cancel() with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(turn, 1) diff --git a/tests/runtime/test_routed_completion_usage.py b/tests/runtime/test_routed_completion_usage.py new file mode 100644 index 00000000..b57270fe --- /dev/null +++ b/tests/runtime/test_routed_completion_usage.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import json +from copy import deepcopy +from dataclasses import replace + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.llm import retry +from opencollab.adapters.llm.client import LLMClient +from opencollab.adapters.llm.responses_errors import ResponsesProtocolError +from opencollab.application.run_control import RunControl +from opencollab.bootstrap import build_session +from opencollab.domain.agent import Agent +from opencollab.tools import builtin_tools +from tests.runtime.test_responses_error_usage_http import _payload +from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport + + +@pytest.mark.parametrize("summary", [False, True]) +@pytest.mark.parametrize("forward_fallback", [False, True]) +@pytest.mark.parametrize("scenario", [ + "primary", "copied-primary", "scrubbed-primary", "replaced-primary", + "fallback", "equal-fallback", "copied-fallback", "scrubbed-fallback", "replaced-fallback", + "annotated-primary", "annotated-fallback", "annotated-primary-after-helper", + "annotated-primary-after-helper-without-timing", + "cache-redacted-primary-after-helper", "cache-redacted-primary-after-helper-without-timing", + "cache-redacted-fallback", "fallback-with-primary-timing", "manual-primary-settlement", + "failed-primary", "failed-primary-attempt-callback-only", +]) +async def test_routed_completion_accounts_for_each_provider( + tmp_path, monkeypatch, summary, forward_fallback, scenario, +): + requests = [] + events = [] + answer = "Answer." if summary else "Answer." + independent_helper = "-primary-after-helper" in scenario + cache_redacted = scenario.startswith("cache-redacted-") + annotated = scenario.startswith("annotated-") or cache_redacted + retains_primary_timing = scenario == "fallback-with-primary-timing" + manual_primary_settlement = scenario == "manual-primary-settlement" + usable_primary = independent_helper or scenario in { + "primary", "copied-primary", "scrubbed-primary", "replaced-primary", + "annotated-primary", + } + equal_usage = independent_helper or retains_primary_timing or scenario in { + "equal-fallback", "copied-fallback", "scrubbed-fallback", "replaced-fallback", + "annotated-fallback", "cache-redacted-fallback", "manual-primary-settlement", + } + failed_primary = scenario.startswith("failed-primary") + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + monkeypatch.setattr(retry, "extract_retry_after_seconds", lambda _error: 0.0) + monkeypatch.setattr(retry.random, "uniform", lambda _lo, _hi: 0.0) + + async def handle(request): + route = request.url.path.split("/")[1] + requests.append(route) + if route == "primary": + if failed_primary: + content_type, payload = _payload(json.loads(request.content), {"kind": "failed"}) + return http_response(request, 200, headers={"content-type": content_type}, content=payload) + text = answer if route != "primary" or usable_primary else "" + output_tokens = 55 if route == "primary" or equal_usage else 2 + payload = completion_http_response(request, text=text, output_tokens=output_tokens).json() + payload["usage"]["prompt_tokens_details"] = { + "cached_tokens": 0 if retains_primary_timing and route == "fallback" else 1, + } + return http_response(request, 200, json=payload) + + def copy_response(response): + if scenario.startswith("replaced-"): + return replace(response, usage=replace(response.usage, raw_usage={})) + if annotated: + removes_timing = scenario.endswith("-without-timing") + usage = replace( + response.usage, + context_tokens=7, + context_output_tokens=5, + context_tokens_estimated=True, + cache_read_tokens=None if cache_redacted else response.usage.cache_read_tokens, + raw_usage=response.usage.raw_usage if removes_timing else {}, + ) + timing = None if removes_timing else response.transport_timing + return replace(response, usage=usage, transport_timing=timing) + copied = deepcopy(response) + if scenario.startswith("scrubbed-"): + copied.usage.raw_usage.clear() + return copied + + class FallbackClient(LLMClient): + async def complete( + self, messages, *, on_admitted=None, before_request=None, + on_attempt_usage=None, on_usage=None, **kwargs, + ): + controls = dict(on_admitted=on_admitted, before_request=before_request, + on_attempt_usage=on_attempt_usage, on_usage=on_usage) + primary_controls = dict(controls) + if manual_primary_settlement or scenario == "failed-primary-attempt-callback-only": + primary_controls.pop("on_usage") + try: + first = await super().complete(messages, **kwargs, **primary_controls) + except ResponsesProtocolError: + assert failed_primary + else: + if manual_primary_settlement and on_usage is not None: + on_usage(replace(first.usage, cache_read_tokens=None)) + if first.content and first.content.strip(): + if independent_helper: + await fallback.complete(messages, **kwargs) + return copy_response(first) if scenario != "primary" else first + final = await fallback.complete(messages, **kwargs, **(controls if forward_fallback else {})) + if retains_primary_timing: + return replace(final, usage=replace(final.usage), transport_timing=first.transport_timing) + if failed_primary or annotated or scenario in { + "copied-fallback", "scrubbed-fallback", "replaced-fallback", + }: + return copy_response(final) + return final + + install_sdk_transport(monkeypatch, "openai", handle) + options = dict(api_key=__name__, context_window=40_000, request_pool=str(tmp_path), request_capacity=1) + control = RunControl(on_event=events.append) + async with ( + FallbackClient( + **options, base_url="https://test.invalid/primary/v1", + wire_protocol="responses" if failed_primary else "chat_completions", + max_retries=1 if failed_primary else 0, + ) as client, + LLMClient(**options, base_url="https://test.invalid/fallback/v1", max_retries=0) as fallback, + ): + if summary: + env = LocalEnvironment(str(tmp_path)) + session = build_session( + agent=Agent(name="routed", system_prompt="System.", max_tokens_per_step=128), llm=client, + env=env, llm_timeout=2, max_budget_tokens=20_000, run_control=control, + ) + try: + await session.add_user_message("Return a brief summary.") + text = await session._context_summarizer.asummarize(session.messages) + assert text == "Summary:\nAnswer." + assert session._context_summarizer.last_call_cacheable + if annotated: + assert session.state.context_tokens == 7 + tokens = session.used_tokens + finally: + await session.aclose() + await env.cleanup() + else: + result = await OpenCollab(tmp_path, config={"llm_timeout": 2, "max_output_tokens": 128}).agent( + "Return a brief answer.", llm=client, tools=(), budget=20_000, trace=False, + run_control=control, system_prompt="System.", + ) + assert result.status == "completed", result.reason + assert result.output == "Answer." + tokens = result.tokens + + expected_usage = [74 if failed_primary else 57] + if not usable_primary: + expected_usage.append(57 if equal_usage else 4) + expected_purpose = "summary" if summary else "completion" + ledger = [json.loads(line) for line in (tmp_path / "usage.jsonl").read_text().splitlines()] + expected_requests = ["primary", "primary"] if failed_primary else ["primary"] + if not usable_primary or independent_helper: + expected_requests.append("fallback") + assert requests == expected_requests + assert tokens == sum(expected_usage) + assert [entry["usage"]["total_tokens"] for entry in ledger] == expected_usage + ([57] if independent_helper else []) + usage_events = [event for event in events if event.type == "usage"] + assert [event.data["total_tokens"] for event in usage_events] == expected_usage + assert {event.data["purpose"] for event in usage_events} == {expected_purpose} + if failed_primary: + assert [entry["status"] for entry in ledger] == ["error", "success"] + assert [event.data["error_type"] for event in usage_events] == ["ResponsesTransientEventError", None] + + +@pytest.mark.parametrize("cache_read_tokens", [None, 0]) +@pytest.mark.parametrize("redact_at", ["response", "settlement", "wrapper"]) +async def test_cache_redaction_preserves_budget_for_tool_followup(tmp_path, monkeypatch, cache_read_tokens, redact_at): + (tmp_path / "facts.txt").write_text("The verified value is 42.\n", encoding="utf-8") + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + requests = [] + events = [] + + async def handle(request): + body = json.loads(request.content) + tool_results = [message["content"] for message in body["messages"] if message["role"] == "tool"] + requests.append(tool_results) + input_tokens, output_tokens = (315, 22) if tool_results else (270, 30) + assert body["max_tokens"] >= output_tokens + payload = completion_http_response(request, text="The value is 42.", output_tokens=output_tokens).json() + payload["usage"].update( + prompt_tokens=input_tokens, total_tokens=input_tokens + output_tokens, + prompt_tokens_details={"cached_tokens": 1}, + ) + if not tool_results: + choice = payload["choices"][0] + choice["finish_reason"] = "tool_calls" + choice["message"].update(content=None, tool_calls=[{ + "id": "read-facts", + "type": "function", + "function": {"name": "file_read", "arguments": json.dumps({"path": "facts.txt"})}, + }]) + return http_response(request, 200, json=payload) + + class CacheRedactingClient(LLMClient): + async def complete( + self, messages, *, on_admitted=None, before_request=None, on_attempt_usage=None, on_usage=None, **kwargs, + ): + def redact_usage(usage): + on_usage(replace(usage, cache_read_tokens=cache_read_tokens)) + + settlement_callback = on_usage + if redact_at == "settlement" and on_usage is not None: + settlement_callback = redact_usage + elif redact_at == "wrapper": + settlement_callback = None + controls = dict(on_admitted=on_admitted, before_request=before_request, + on_attempt_usage=on_attempt_usage, on_usage=settlement_callback) + kwargs.update((name, callback) for name, callback in controls.items() if callback is not None) + response = await super().complete(messages, **kwargs) + assert response.usage.cache_read_tokens == 1 + if redact_at == "wrapper" and on_usage is not None: + redact_usage(response.usage) + if redact_at == "response": + return replace(response, usage=replace(response.usage, cache_read_tokens=cache_read_tokens)) + return response + + install_sdk_transport(monkeypatch, "openai", handle) + async with CacheRedactingClient( + api_key=__name__, base_url="https://test.invalid/v1", max_retries=0, + context_window=40_000, request_pool=str(tmp_path), request_capacity=1, + ) as client: + result = await OpenCollab(tmp_path, config={"llm_timeout": 3, "max_output_tokens": 900}).agent( + "Read facts.txt and report the verified value.", llm=client, tools=builtin_tools("file_read"), + budget=1_250, max_steps=4, trace=False, system_prompt="System.", + run_control=RunControl(on_event=events.append), + ) + + assert result.status == "completed", result.reason + assert result.output == "The value is 42." + assert result.tokens == 637 + assert len(requests) == 2 + assert requests[0] == [] + assert "The verified value is 42." in requests[1][0] + ledger = [json.loads(line) for line in (tmp_path / "usage.jsonl").read_text().splitlines()] + assert [entry["usage"]["total_tokens"] for entry in ledger] == [300, 337] + usage_events = [event.data for event in events if event.type == "usage"] + assert [event["total_tokens"] for event in usage_events] == [300, 337] + if redact_at != "response": + assert [event["cache_read_tokens"] for event in usage_events] == [cache_read_tokens] * 2 diff --git a/tests/runtime/test_session_output_limit_http.py b/tests/runtime/test_session_output_limit_http.py index c18738d6..6e7423eb 100644 --- a/tests/runtime/test_session_output_limit_http.py +++ b/tests/runtime/test_session_output_limit_http.py @@ -2,18 +2,21 @@ from __future__ import annotations +import asyncio import json +from functools import wraps import pytest -from opencollab import OpenCollab +from opencollab import OpenCollab, RunControl from opencollab.adapters.llm.client import LLMClient from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.application._session_run_shared import _TokenBudgetStop from opencollab.bootstrap import container from opencollab.bootstrap.session_factory import build_session from opencollab.domain.agent import Agent from tests.support.environment_test_support import local_environment_factory as local_environment_factory -from tests.support.provider_sdk_http import completion_http_response, install_sdk_transport +from tests.support.provider_sdk_http import completion_http_response, http_response, install_sdk_transport @pytest.fixture(autouse=True) @@ -176,3 +179,235 @@ async def complete(self, messages, tools=None, temperature=0.0): assert await summarizer.asummarize([{"role": "user", "content": "Keep this context"}]) == ( "Summary:\nPreserved context." ) + + +class _LegacyNativeClient(LLMClient): + @wraps(LLMClient.complete) + async def complete(self, messages, tools=None, temperature=0.0, *, max_output_tokens=None, on_admitted=None): + self.received_admission_callback = on_admitted is not None + return await super().complete( + messages, tools=tools, temperature=temperature, + max_output_tokens=max_output_tokens, on_admitted=on_admitted, + ) + + +class _SettlementWrapper(_LegacyNativeClient): + @wraps(_LegacyNativeClient.complete) + async def complete(self, *args, on_usage=None, **kwargs): + self.received_usage_callback = on_usage is not None + response = await super().complete(*args, **kwargs) + if on_usage is not None: + on_usage(response.usage) + return response + + +class _RevalidatingWrapper(_LegacyNativeClient): + wait_for_prompt = None + + @wraps(_LegacyNativeClient.complete) + async def complete(self, *args, **kwargs): + messages = args[0] if args else kwargs["messages"] + if self.wait_for_prompt is None or messages[-1]["content"] == self.wait_for_prompt: + self.arrived.set() + await self.release.wait() + before_request = kwargs.pop("before_request", None) + if before_request is not None: + kwargs["max_output_tokens"] = min(kwargs["max_output_tokens"], before_request()) + return await super().complete(*args, **kwargs) + + +class _ClassOptInWrapper(_RevalidatingWrapper): + supports_request_revalidation = True + + +class _SlotOptInWrapper(_RevalidatingWrapper): + __slots__ = ("supports_request_revalidation",) + + +class _PropertyOptInWrapper(_RevalidatingWrapper): + @property + def supports_request_revalidation(self): + return True + + +@pytest.mark.parametrize("purpose", ["completion", "summary"]) +@pytest.mark.parametrize( + "client_class", + [_RevalidatingWrapper, _ClassOptInWrapper, _SlotOptInWrapper, _PropertyOptInWrapper], + ids=["instance", "class", "slot", "property"], +) +async def test_decorated_client_revalidates_budget_before_sending( + tmp_path, monkeypatch, local_environment_factory, purpose, client_class, +): + requests = [] + + async def handler(request): + requests.append(json.loads(request.content)) + return completion_http_response(request, text="Preserved context.") + + install_sdk_transport(monkeypatch, "openai", handler) + async with client_class( + api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + ) as client: + if client_class in {_RevalidatingWrapper, _SlotOptInWrapper}: + client.supports_request_revalidation = True + client.arrived, client.release = asyncio.Event(), asyncio.Event() + session = build_session( + agent=Agent(name="queued", system_prompt="system", max_tokens_per_step=128), + llm=client, env=local_environment_factory(str(tmp_path)), max_budget_tokens=20_000, + ) + await session.add_user_message("Keep this context") + operation = ( + session._context_summarizer.asummarize(session.messages) + if purpose == "summary" else session.run_loop() + ) + task = asyncio.create_task(operation) + try: + await asyncio.wait_for(client.arrived.wait(), 5) + session.max_budget_tokens = 1 + client.release.set() + if purpose == "summary": + with pytest.raises(_TokenBudgetStop): + await asyncio.wait_for(task, 5) + else: + assert await asyncio.wait_for(task, 5) == "" + assert "budget exhausted before model call" in session.state.terminal_reason + assert requests == [] + assert session.used_tokens == 0 + finally: + client.release.set() + await asyncio.gather(task, return_exceptions=True) + await session.aclose() + + +async def test_decorated_slot_client_revalidates_shared_workflow_spend(tmp_path, monkeypatch): + requests = [] + + async def handler(request): + body = json.loads(request.content) + prompt = body["messages"][-1]["content"] + requests.append((prompt, body["max_tokens"])) + response = completion_http_response(request) + if prompt == "spender": + usage = {"prompt_tokens": 1997, "completion_tokens": 2, "total_tokens": 1999} + return http_response(request, 200, json=response.json() | {"usage": usage}) + return response + + install_sdk_transport(monkeypatch, "openai", handler) + async with _SlotOptInWrapper( + api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + ) as client: + client.supports_request_revalidation = True + client.wait_for_prompt = "waiter" + client.arrived, client.release = asyncio.Event(), asyncio.Event() + + async def flow(ctx, _inputs): + waiter = asyncio.create_task(ctx.agent_run("waiter", tools=[], system_prompt="System.", budget=1000)) + try: + await asyncio.wait_for(client.arrived.wait(), 5) + spender = await ctx.agent_run("spender", tools=[], system_prompt="System.", budget=1000) + assert spender.tokens == ctx.tokens_spent() == 1999 + client.release.set() + return await asyncio.wait_for(waiter, 5) + finally: + client.release.set() + await asyncio.gather(waiter, return_exceptions=True) + + result = await OpenCollab(tmp_path, config={"max_output_tokens": 128}).workflow( + flow, llm=client, budget=2000, trace=False, limit_mode="explicit", concurrency=2, + ) + + assert result.status == "completed" + assert result.output.status == "stopped" + assert "budget exhausted before model call" in result.output.reason + assert result.output.tokens == 0 + assert result.tokens == 1999 + assert requests == [("spender", 128)] + + +class _ForwardingWrapper(_LegacyNativeClient): + @wraps(_LegacyNativeClient.complete) + async def complete(self, *args, **kwargs): + return await super().complete(*args, **kwargs) + + +class _ForwardingProxy: + def __init__(self, client): + self.client = client + + def __getattr__(self, name): + return getattr(self.client, name) + + @wraps(_ForwardingWrapper.complete) + async def complete(self, *args, **kwargs): + return await self.client.complete(*args, **kwargs) + + +@pytest.mark.parametrize("failed", [False, True]) +@pytest.mark.parametrize("client_class", [_LegacyNativeClient, _SettlementWrapper, _ForwardingWrapper]) +async def test_public_agent_charges_legacy_native_wrapper_usage_once(tmp_path, monkeypatch, failed, client_class): + requests = [] + events = [] + + async def handler(request): + requests.append(json.loads(request.content)) + response = completion_http_response(request) + if failed: + return http_response(request, 200, json=response.json() | {"choices": []}) + return response + + install_sdk_transport(monkeypatch, "openai", handler) + pool = tmp_path / "pool" + pool.mkdir() + async with client_class( + api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + request_pool=str(pool), request_capacity=1, + ) as client: + llm = _ForwardingProxy(client) if client_class is _ForwardingWrapper else client + result = await OpenCollab(tmp_path).agent( + "Return ok", llm=llm, tools=(), budget=20_000, trace=False, + run_control=RunControl(on_event=events.append), + ) + + assert client.received_admission_callback is True + if client_class is _SettlementWrapper: + assert client.received_usage_callback is True + assert len(requests) == 1 and requests[0]["max_tokens"] == 8192 + assert result.status == ("failed" if failed else "completed") + assert result.tokens == 4 + usage = [event.data for event in events if event.type == "usage"] + assert [event["total_tokens"] for event in usage] == [4] + assert usage[0]["error_type"] == ("IndexError" if failed else None) + + +async def test_summary_charges_legacy_native_wrapper_usage_once(tmp_path, monkeypatch, local_environment_factory): + requests = [] + + async def handler(request): + requests.append(json.loads(request.content)) + return completion_http_response(request, text="Preserved context.") + + install_sdk_transport(monkeypatch, "openai", handler) + async with _LegacyNativeClient( + api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + ) as client: + agent = Agent(name="legacy", system_prompt="system") + session = build_session( + agent=agent, llm=client, env=local_environment_factory(str(tmp_path)), max_budget_tokens=None, + ) + summarizer = container._build_summarizer( + agent, client, client, 600.0, None, completion_handler=session.runner._invoke_summary, + ) + try: + assert await summarizer.asummarize([{"role": "user", "content": "Keep this context"}]) == ( + "Summary:\nPreserved context." + ) + assert session.used_tokens == 4 + finally: + await session.aclose() + + assert len(requests) == 1 and requests[0]["max_tokens"] == 8192 diff --git a/tests/runtime/test_vm_mutation_locks.py b/tests/runtime/test_vm_mutation_locks.py new file mode 100644 index 00000000..7042bcb9 --- /dev/null +++ b/tests/runtime/test_vm_mutation_locks.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from opencollab.adapters._env_local import LocalEnvironment +from opencollab.adapters.tools.base import host_write_locks +from opencollab.adapters.tools.fs import FileWriteTool +from opencollab.application.tool_execution import ToolRuntime +from tests.runtime.test_vm_environment import _env + + +@pytest.mark.parametrize("cancel", [False, True]) +async def test_vm_grouped_mutation_uses_one_native_workspace_lock(tmp_path, monkeypatch, cancel): + (tmp_path / "nested").mkdir() + (tmp_path / "nested/value.py").write_text("VALUE = 1\n") + native = LocalEnvironment(str(tmp_path)) + environment = _env() + environment.workspace = str(tmp_path) + environment._identity = {"workspace": str(tmp_path)} + lock = asyncio.Lock() + owners = set() + acquisitions = [] + + async def rpc(payload): + operation = payload["operation_id"] + if payload["method"] == "cancel": + if operation in owners: + owners.remove(operation) + lock.release() + return {"status": "cancelled"} + assert payload["method"] == "start" + action = payload["request"]["action"] + arguments = payload["request"]["args"] + if action == "lock": + await lock.acquire() + owners.add(operation) + acquisitions.append(operation) + return {"status": "running", "lock_acquired": True} + assert lock.locked() + if action == "read": + value = await native.read_file(arguments["path"]) + else: + assert action == "write" + value = await native.write_file_with_change(arguments["path"], arguments["content"]) + return {"status": "completed", "value": value} + + monkeypatch.setattr(environment, "_rpc", rpc) + entered, release = asyncio.Event(), asyncio.Event() + + async def publish(): + async with host_write_locks(( + str(tmp_path / ".candidate-adoption"), str(tmp_path / "nested/value.py"), + str(tmp_path / "unrelated.py"), + ), environment): + await environment.write_file("nested/value.py", "VALUE = 42\n") + entered.set() + await release.wait() + + publication = asyncio.create_task(publish()) + writer = None + try: + await asyncio.wait_for(entered.wait(), 2) + assert len(acquisitions) == 1 + writer = asyncio.create_task(FileWriteTool().execute_with_runtime({ + "path": "nested/value.py", "mode": "create", "content": "VALUE = 100\n", + }, ToolRuntime(environment, None, None))) + await asyncio.sleep(0.05) + assert not writer.done() + if cancel: + publication.cancel() + with pytest.raises(asyncio.CancelledError): + await publication + else: + release.set() + await publication + assert "Created/wrote" in await asyncio.wait_for(writer, 2) + assert (tmp_path / "nested/value.py").read_text() == "VALUE = 100\n" + assert len(acquisitions) == 2 + assert not owners + assert not environment.pending_operations + assert not environment.revoked + finally: + release.set() + await asyncio.gather(publication, *([writer] if writer is not None else []), return_exceptions=True) + await environment.cleanup() + await native.cleanup() diff --git a/tests/support/docker_native_edits_support.py b/tests/support/docker_native_edits_support.py index 6478f2ac..0fe3ef6b 100644 --- a/tests/support/docker_native_edits_support.py +++ b/tests/support/docker_native_edits_support.py @@ -80,6 +80,18 @@ async def __call__(self, *argv, timeout=120, input_bytes=None): pass return ProcessResult(proc.returncode, out, err) +class RemoteDockerTransport(LocalDockerTransport): + def __init__(self, workspace: Path, *, pair_reads: bool = False): + super().__init__(workspace, pair_reads=pair_reads) + self.remote = f"/oc-native-test-{uuid.uuid4().hex}" + + async def __call__(self, *argv, **kwargs): + actual = str(self.workspace) + mapped = tuple(arg.replace(self.remote, actual) if isinstance(arg, str) else arg for arg in argv) + result = await super().__call__(*mapped, **kwargs) + return ProcessResult(result.returncode, result.stdout.replace(actual.encode(), self.remote.encode()), + result.stderr.replace(actual.encode(), self.remote.encode())) + def docker_environment(workspace: Path, transport: LocalDockerTransport, *, container_id=None): env = DockerEnvironment( workspace=str(workspace), diff --git a/tests/tools/test_docker_edit_path_identity.py b/tests/tools/test_docker_edit_path_identity.py index 6ecc95ad..742aa6d4 100644 --- a/tests/tools/test_docker_edit_path_identity.py +++ b/tests/tools/test_docker_edit_path_identity.py @@ -191,6 +191,21 @@ async def test_new_parent_directories_resolve_through_existing_alias(tmp_path): assert (directory / "new/nested/file.txt").read_text() == "created" +@pytest.mark.parametrize("path", ["config/default.toml", "missing/../module.py", "config/../module.py"]) +async def test_docker_edit_requires_a_directory_before_parent_traversal_or_file_creation(tmp_path, path): + (tmp_path / "config").write_text("config") + (tmp_path / "module.py").write_text("original") + environment = docker_environment(tmp_path, LocalDockerTransport(tmp_path, pair_reads=False)) + try: + with pytest.raises(OSError, match="execution directory"): + await FileWriteTool().execute_with_runtime({ + "path": path, "mode": "create", "content": "changed", + }, ToolRuntime(environment, None, None)) + assert (tmp_path / "config").read_text() == "config" + assert (tmp_path / "module.py").read_text() == "original" + finally: + await environment.cleanup() + @pytest.mark.parametrize("result", [ ExecResult(1, "", "directory unavailable"), ExecResult(0, "/unframed/path\n", ""), diff --git a/tests/tools/test_local_file_lock_workers.py b/tests/tools/test_local_file_lock_workers.py new file mode 100644 index 00000000..b18657de --- /dev/null +++ b/tests/tools/test_local_file_lock_workers.py @@ -0,0 +1,313 @@ +from __future__ import annotations + +import asyncio +import errno +import fcntl +import os +import resource +import signal +import subprocess +import sys +from contextlib import ExitStack +from pathlib import Path + +import pytest +from filelock import FileLock, Timeout + +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.tools import base +from opencollab.adapters.tools._file_lock_worker import _try_file_lock +from opencollab.adapters.tools.fs import FileWriteTool +from opencollab.application.tool_execution import ToolRuntime + + +@pytest.mark.parametrize("use_workers", [False, True], ids=["direct", "worker"]) +async def test_native_sibling_writes_share_ancestors_and_exclude_transactions(tmp_path, monkeypatch, use_workers): + if use_workers: + monkeypatch.setattr(base, "_local_file_lock_capacity", lambda: 2) + monkeypatch.setattr(base, "HOST_WRITE_LOCK_TIMEOUT_SECONDS", 0.25) + directory = tmp_path / "nested" + directory.mkdir() + for name in ("one.txt", "two.txt", "three.txt"): + (directory / name).write_text("before\n") + env = LocalEnvironment(str(tmp_path)) + runtime = ToolRuntime(environment=env, safety_policy=None, permission_policy=None) + transaction_paths = ("nested", "other.txt", "unrelated.txt") + started, finish = asyncio.Event(), asyncio.Event() + read_file = env.read_file + + async def pause_first_read(path): + if Path(path).name == "one.txt" and not started.is_set(): + started.set() + await finish.wait() + return await read_file(path) + + monkeypatch.setattr(env, "read_file", pause_first_read) + + async def write(name): + return await FileWriteTool().execute_with_runtime( + {"path": f"nested/{name}", "mode": "str_replace", "old_str": "before", "new_str": "after"}, runtime, + ) + + first = asyncio.create_task(write("one.txt")) + try: + try: + await asyncio.wait_for(started.wait(), 5) + await asyncio.wait_for(write("two.txt"), 5) + assert (directory / "two.txt").read_text() == "after\n" + assert (directory / "one.txt").read_text() == "before\n" + with pytest.raises(TimeoutError, match="one.txt"): + await write("one.txt") + with pytest.raises(TimeoutError, match="nested$"): + async with base.host_write_locks(transaction_paths, env): + pytest.fail("transaction acquired a directory with an active child edit") + finally: + finish.set() + await first + + assert (directory / "one.txt").read_text() == "after\n" + async with base.host_write_locks(transaction_paths, env): + with pytest.raises(TimeoutError, match="nested$"): + await write("three.txt") + assert (directory / "three.txt").read_text() == "before\n" + await write("three.txt") + assert (directory / "three.txt").read_text() == "after\n" + finally: + await env.cleanup() + + +def test_shared_flock_rejects_inode_unlinked_before_acquisition(tmp_path, monkeypatch): + path = tmp_path / "directory.lock" + acquire_exclusive = FileLock(path, timeout=0) + acquire_exclusive.acquire() + open_file = os.open + + def open_then_release(*args, **kwargs): + fd = open_file(*args, **kwargs) + path.unlink() + acquire_exclusive.release() + return fd + + try: + with monkeypatch.context() as patch: + patch.setattr(os, "open", open_then_release) + with pytest.raises(Timeout), _try_file_lock(str(path), shared=True): + pytest.fail("shared lock acquired an unlinked inode") + with _try_file_lock(str(path), shared=True): + with pytest.raises(Timeout), acquire_exclusive: + pytest.fail("legacy writer acquired an active shared inode") + finally: + acquire_exclusive.release() + + +def test_unavailable_shared_flock_uses_exclusive_file_lock(tmp_path, monkeypatch): + path = tmp_path / "directory.lock" + flock = fcntl.flock + + def exclusive_only(fd, mode): + if mode & fcntl.LOCK_SH: + raise OSError(errno.ENOSYS, "shared flock unavailable") + return flock(fd, mode) + + monkeypatch.setattr(fcntl, "flock", exclusive_only) + acquire_exclusive = FileLock(path, timeout=0) + with _try_file_lock(str(path), shared=True): + with pytest.raises(Timeout), acquire_exclusive: + pytest.fail("legacy writer acquired a fallback lock") + with acquire_exclusive: + pass + + +async def test_large_group_acquires_existing_flocks_in_path_order(tmp_path, monkeypatch): + monkeypatch.setattr(base, "_local_file_lock_capacity", lambda: 2) + monkeypatch.setattr(base, "HOST_WRITE_LOCK_TIMEOUT_SECONDS", 0.2) + env = LocalEnvironment(str(tmp_path)) + first, blocked, last = ( + FileLock(base._host_lock_path(path, env), timeout=0) for path in ("a.txt", "b.txt", "c.txt") + ) + other = FileLock(base._host_lock_path("other.txt", env), timeout=0) + paths = ("c.txt", "a.txt", "b.txt") + try: + with blocked, pytest.raises(TimeoutError, match="b.txt"): + async with base.host_write_locks(paths, env): + pytest.fail("group acquired a lock already held by an existing writer") + with first, last: + pass + async with base.host_write_locks(paths, env): + for lock in (first, blocked, last): + with pytest.raises(Timeout): + lock.acquire() + with other: + pass + finally: + await env.cleanup() + + +async def test_cancelled_acquisition_releases_the_partial_group(tmp_path, monkeypatch): + monkeypatch.setattr(base, "_local_file_lock_capacity", lambda: 2) + env = LocalEnvironment(str(tmp_path)) + paths = ("a.txt", "b.txt", "c.txt") + first, blocked, last = (FileLock(base._host_lock_path(path, env), timeout=0) for path in paths) + + async def acquire_group(): + async with base.host_write_locks(paths, env): + pytest.fail("group acquired a lock already held by an existing writer") + + async def wait_until_first_is_held(): + while True: + try: + with first: + pass + except Timeout: + return + await asyncio.sleep(0.02) + + try: + with blocked: + owner = asyncio.create_task(acquire_group()) + try: + await asyncio.wait_for(wait_until_first_is_held(), 5) + with last: + pass + finally: + owner.cancel() + with pytest.raises(asyncio.CancelledError): + await owner + with first, last: + pass + finally: + await env.cleanup() + + +def test_nested_file_write_with_four_free_descriptors(tmp_path): + completed = subprocess.run( + [sys.executable, "-c", + "import asyncio, sys; from tests.tools.test_local_file_lock_workers import _write_nested_file; " + "asyncio.run(_write_nested_file(sys.argv[1]))", str(tmp_path)], + cwd=Path(__file__).resolve().parents[2], + close_fds=True, capture_output=True, text=True, timeout=20, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + + +async def _write_nested_file(workspace): + target = Path(workspace, "nested/subdirectory/note.txt") + target.parent.mkdir(parents=True) + target.write_text("before\n") + env = LocalEnvironment(workspace) + try: + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + limit = 256 if soft == resource.RLIM_INFINITY else min(soft, 256) + resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) + opened = len(os.listdir("/dev/fd")) - 1 + with ExitStack() as descriptors: + for _ in range(limit - opened - 4): + descriptors.enter_context(open(os.devnull, "rb")) + await FileWriteTool().execute_with_runtime( + {"path": str(target.relative_to(workspace)), "mode": "str_replace", + "old_str": "before", "new_str": "after"}, + ToolRuntime(environment=env, safety_policy=None, permission_policy=None), + ) + assert target.read_text() == "after\n" + finally: + await env.cleanup() + + +_OWNER = """ +import asyncio +import sys + +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.tools import base +from opencollab.application.async_timeout import await_owned_operation + + +async def main(): + base._local_file_lock_capacity = lambda: 2 + stop, workspace, *paths = sys.argv[1:] + env = LocalEnvironment(workspace) + entered = asyncio.Event() + + async def hold(): + async with base.host_write_locks(paths, env): + entered.set() + try: + await asyncio.Event().wait() + finally: + if stop == 'terminal': + print('recovering', flush=True) + await await_owned_operation(asyncio.to_thread(sys.stdin.readline)) + + task = asyncio.create_task(hold()) + try: + await asyncio.wait_for(entered.wait(), 5) + print('ready', flush=True) + if stop == 'terminal': + await task + return + await asyncio.to_thread(sys.stdin.readline) + task.cancel() + try: + await task + except asyncio.CancelledError: + print('cancelled', flush=True) + await asyncio.to_thread(sys.stdin.readline) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await env.cleanup() + + +try: + asyncio.run(main()) +except KeyboardInterrupt: + sys.exit(130) +""" + + +@pytest.mark.parametrize("stop", ["cancel", "crash", "terminal"]) +async def test_large_group_releases_existing_flocks_after_owner_stops(tmp_path, stop): + paths = ("a.txt", "b.txt", "c.txt") + env = LocalEnvironment(str(tmp_path)) + old_locks = [FileLock(base._host_lock_path(path, env), timeout=2) for path in paths] + owner = await asyncio.create_subprocess_exec( + sys.executable, "-c", _OWNER, stop, str(tmp_path), *paths, + stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, start_new_session=True, + ) + + def confirm_released(): + for lock in old_locks: + with lock: + pass + + def confirm_held_during_recovery(): + with pytest.raises(Timeout), old_locks[0].acquire(timeout=0.25): + pass + + try: + assert await asyncio.wait_for(owner.stdout.readline(), 10) == b"ready\n" + for lock in old_locks: + with pytest.raises(Timeout): + lock.acquire(timeout=0) + if stop == "crash": + owner.kill() + await owner.wait() + elif stop == "terminal": + os.killpg(owner.pid, signal.SIGINT) + assert await asyncio.wait_for(owner.stdout.readline(), 10) == b"recovering\n" + await asyncio.to_thread(confirm_held_during_recovery) + owner.stdin.write(b"finish\n") + await owner.stdin.drain() + assert await asyncio.wait_for(owner.wait(), 10) == 130 + else: + owner.stdin.write(b"cancel\n") + await owner.stdin.drain() + assert await asyncio.wait_for(owner.stdout.readline(), 10) == b"cancelled\n" + assert owner.returncode is None + await asyncio.to_thread(confirm_released) + finally: + owner.stdin.close() + if owner.returncode is None: + owner.kill() + await owner.wait() + await env.cleanup() diff --git a/tests/tools/test_remote_search_paths.py b/tests/tools/test_remote_search_paths.py new file mode 100644 index 00000000..5abe0ee0 --- /dev/null +++ b/tests/tools/test_remote_search_paths.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from dataclasses import asdict + +from opencollab.adapters import _vm_remote +from opencollab.adapters.env import DockerEnvironment, LocalEnvironment, VMEnvironment +from opencollab.adapters.session_environments import SessionEnvironmentPool +from opencollab.adapters.tools.fs import GrepTool +from opencollab.adapters.vm_config import VMConnection +from opencollab.application.tool_execution import ToolRuntime +from opencollab.bootstrap.authored_workflow_bridge import AuthoredWorkflowBridge, BridgeEnvironment +from opencollab.bootstrap.runtime_context import build_workspace_safety_policy +from tests.support.docker_native_edits_support import RemoteDockerTransport + + +async def test_vm_search_path_is_resolved_in_the_guest(tmp_path, monkeypatch): + workspace = tmp_path / "workspace" + (workspace / "nested").mkdir(parents=True) + (workspace / "module.py").write_text("search marker\n") + guest = LocalEnvironment(str(workspace)) + vm = VMEnvironment( + VMConnection("guest", "worker"), str(tmp_path / "guest"), state_directory=str(tmp_path / "state"), + ) + info = workspace.stat() + identity = {"device": info.st_dev, "inode": info.st_ino} + vm._identity = identity + monkeypatch.setattr(_vm_remote, "_identity", lambda path: identity) + calls = [] + + async def rpc(payload): + request = payload["request"] + action, args = request["action"], request["args"] + calls.append(action) + if action == "resolve_search_path": + path = args["path"].replace(vm.workspace, str(workspace), 1) + value = await _vm_remote._perform({ + **request, "workspace": str(workspace), "args": {"path": path}, + }, tmp_path) + value = value.replace(str(workspace), vm.workspace, 1) + else: + assert action == "exec" + command = args["cmd"].replace(vm.workspace, str(workspace), 1) + value = {**asdict(await guest.exec_cmd(command)), "stdout_bytes": None} + return {"status": "completed", "value": value} + + monkeypatch.setattr(vm, "_rpc", rpc) + runtime = ToolRuntime(vm, build_workspace_safety_policy(vm), None) + try: + result = await GrepTool().execute_with_runtime( + {"path": "nested/../module.py", "pattern": "search marker"}, runtime, + ) + assert result == "1:search marker" + assert calls[:2] == ["resolve_search_path", "exec"] + finally: + await guest.cleanup() + await vm.cleanup() + + +async def test_authored_bridge_preserves_remote_search_resolution(tmp_path): + (tmp_path / "nested").mkdir() + (tmp_path / "module.py").write_text("search marker\n") + transport = RemoteDockerTransport(tmp_path) + environment = DockerEnvironment( + workspace=transport.remote, exec_workdir=transport.remote, container_id="b" * 64, + ) + transport.attach(environment) + pool = SessionEnvironmentPool(environment.workspace, isolation="shared", base_environment=environment) + bridge = AuthoredWorkflowBridge( + environment, environment.workspace + "/artifacts", pool, {}, ToolRuntime(environment, None, None), + allow_unisolated_shell=True, + ) + + class BridgeConnection: + async def call(self, action, arguments, handle, *, edit=None): + return await bridge._dispatch("local-search", { + "action": action, "arguments": arguments, "environment": handle, "edit": edit, + }) + + client = BridgeConnection() + handle = await client.call("acquire", {"strategy": "shared", "label": "child"}, 0) + proxy = BridgeEnvironment(client, handle) + try: + runtime = ToolRuntime(proxy, build_workspace_safety_policy(proxy), None) + result = await GrepTool().execute_with_runtime( + {"path": "nested/..", "pattern": "search marker"}, runtime, + ) + assert "module.py:1:search marker" in result + finally: + await proxy.cleanup() + await bridge.aclose() + await environment.cleanup() diff --git a/tests/workflows/test_candidate_adoption_cancellation.py b/tests/workflows/test_candidate_adoption_cancellation.py index 6e202bbb..2fb6b65c 100644 --- a/tests/workflows/test_candidate_adoption_cancellation.py +++ b/tests/workflows/test_candidate_adoption_cancellation.py @@ -7,8 +7,12 @@ import pytest +from opencollab.adapters._candidate_increment import _apply_increment +from opencollab.adapters._env_local import LocalEnvironment from opencollab.application.workflow import WorkflowContext from opencollab.application.workflow_candidates import CandidateRun +from tests.workflows.test_candidate_adoption_recovery import _repository +from tests.workflows.test_workflow_candidate_workspace import _git @pytest.mark.parametrize("modern", [False, True]) @@ -71,3 +75,38 @@ async def deliver(candidate, preserve_paths): with pytest.raises(OSError) as raised: await context.adopt_candidate(selected) assert raised.value is failure + + +async def test_backend_cancellation_cleans_recovery_directory_created_before_receipt(tmp_path, monkeypatch): + repo = _repository(tmp_path) + original = (repo / "other.py").read_text() + (repo / "other.py").write_text("VALUE = 42\n") + patch = _git(repo, "diff", "--binary", "--", "other.py") + _git(repo, "restore", "--worktree", "--", "other.py") + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + allocated, release_receipt = asyncio.Event(), asyncio.Event() + + async def hold_recovery_directory_receipt(command, *args, **kwargs): + result = await execute(command, *args, **kwargs) + if command.startswith("mktemp -d ") and "candidate-recovery-" in command: + allocated.set() + await release_receipt.wait() + return result + + monkeypatch.setattr(environment, "exec_cmd", hold_recovery_directory_receipt) + task = asyncio.create_task(_apply_increment(environment, str(repo), patch)) + try: + await asyncio.wait_for(allocated.wait(), 5) + assert list((repo / ".git").glob("candidate-recovery-*")) + task.cancel() + await asyncio.sleep(0) + release_receipt.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 5) + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert (repo / "other.py").read_text() == original + finally: + release_receipt.set() + await asyncio.gather(task, return_exceptions=True) + await environment.cleanup() diff --git a/tests/workflows/test_candidate_adoption_command_size.py b/tests/workflows/test_candidate_adoption_command_size.py new file mode 100644 index 00000000..0029058d --- /dev/null +++ b/tests/workflows/test_candidate_adoption_command_size.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import asyncio +import errno +import os +import resource +import subprocess +import sys +from contextlib import ExitStack +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters._env_local import LocalEnvironment +from tests.workflows.test_candidate_adoption_recovery import _detached_files +from tests.workflows.test_workflow_candidate_workspace import _git + + +class _ArgumentBoundEnvironment(LocalEnvironment): + async def exec_cmd(self, command, timeout=120.0, **kwargs): + if len(command.encode()) > 65_536: + raise OSError(errno.E2BIG, "transport command argument is too long") + return await super().exec_cmd(command, timeout, **kwargs) + + +@pytest.mark.parametrize(("blocked", "file_count", "already_open"), [ + pytest.param(False, 900, 0, id="bulk-False"), + pytest.param(True, 900, 0, id="bulk-True"), + pytest.param(False, 120, 120, id="busy-process-False"), + pytest.param(True, 120, 120, id="busy-process-True"), + pytest.param(False, 900, 232, id="near-descriptor-limit"), +]) +def test_bulk_candidate_adoption_keeps_recovery_below_process_limits(tmp_path, blocked, file_count, already_open): + soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + limit = 256 if soft == resource.RLIM_INFINITY else min(soft, 256) + if already_open and limit < 256: + pytest.skip("the busy process case uses a 256 descriptor limit") + completed = subprocess.run( + [ + sys.executable, "-m", __name__, str(tmp_path), + str(int(blocked)), str(file_count), str(already_open), str(limit), + ], + cwd=Path(__file__).resolve().parents[2], + close_fds=True, capture_output=True, text=True, timeout=180, + ) + if completed.returncode: + pytest.fail( + f"candidate child exited with status {completed.returncode}\n{completed.stdout}{completed.stderr}", + pytrace=False, + ) + + +async def _exercise_bulk_candidate(tmp_path: Path, blocked: bool, file_count: int, already_open: int) -> None: + # Long absolute paths expand the recovery script past the temp-file limit + # even though the repository-relative patch fits under one megabyte. + repo = tmp_path.joinpath(*("nested-" + "workspace-" * 8,) * 5, "repo") + (repo / "src").mkdir(parents=True) + (repo / "settings").mkdir() + names = [f"{number:04d}_" + "module_" * 10 + ".py" for number in range(file_count)] + for name in names: + (repo / "src" / name).write_text("VALUE = 1\n") + (repo / "settings/value.txt").write_text("original settings\n") + (repo / ".gitignore").write_text("*.cache\n") + _git(repo, "init", "-q") + _git(repo, "config", "user.name", "Test") + _git(repo, "config", "user.email", "test@example.invalid") + _git(repo, "add", ".") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare a bulk candidate") + if blocked: + (repo / "settings/session.cache").write_text("runtime cache\n") + index = (repo / ".git/index").read_bytes() + first_module = repo / "src" / names[0] + original_inode = first_module.stat().st_ino + environment = _ArgumentBoundEnvironment(str(repo)) + adoption_failure: RuntimeError | None = None + + async def edit(child, inputs): + folder = Path(child.workspace_root) + for name in names: + (folder / "src" / name).write_text("VALUE = 2\n") + (folder / "settings/value.txt").unlink() + (folder / "settings").rmdir() + (folder / "settings").write_text("new settings\n") + return "updated" + + async def flow(ctx, inputs): + nonlocal adoption_failure + candidate = await ctx.candidate_workflow(edit, {}, label="bulk-rewrite") + assert len(candidate.effective_diff.encode()) < 1_000_000 + with ExitStack() as descriptors: + for _ in range(already_open): + descriptors.enter_context(open(os.devnull, "rb")) + if blocked: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory") as failed: + await ctx.adopt_candidate(candidate) + adoption_failure = failed.value + else: + await ctx.adopt_candidate(candidate) + return "finished" + + try: + result = await OpenCollab(repo, environment=environment).workflow(flow, {}, trace=False) + assert result.ok, result.reason + if environment.revoked: + raise AssertionError("candidate adoption revoked the source environment") from adoption_failure + expected = "VALUE = 1\n" if blocked else "VALUE = 2\n" + assert all((repo / "src" / name).read_text() == expected for name in names) + if blocked: + assert (repo / "settings/value.txt").read_text() == "original settings\n" + assert (repo / "settings/session.cache").read_text() == "runtime cache\n" + originals, _ = _detached_files(repo / ".git") + assert originals[repo / "settings/value.txt"] == b"original settings\n" + assert all(originals[repo / "src" / name] == b"VALUE = 1\n" for name in names) + record, = ( + path for path in (repo / ".git").glob("candidate-recovery-*/original-inodes/*.path") + if Path(path.read_text()) == first_module + ) + assert record.with_suffix("").stat().st_ino == original_inode + else: + assert (repo / "settings").read_text() == "new settings\n" + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").rglob(".candidate-command-*.sh")) + finally: + await environment.cleanup() + + +if __name__ == "__main__": + directory, blocked, file_count, already_open, limit = sys.argv[1:] + resource.setrlimit(resource.RLIMIT_NOFILE, (int(limit), int(limit))) + asyncio.run(_exercise_bulk_candidate(Path(directory), bool(int(blocked)), int(file_count), int(already_open))) diff --git a/tests/workflows/test_candidate_adoption_concurrency.py b/tests/workflows/test_candidate_adoption_concurrency.py new file mode 100644 index 00000000..5999d23b --- /dev/null +++ b/tests/workflows/test_candidate_adoption_concurrency.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from contextvars import ContextVar +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters._env_local import LocalEnvironment +from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace +from tests.workflows.test_candidate_adoption_recovery import _repository, _run + +_ADOPTER = ContextVar("adopter", default="setup") + + +class _OrderedEnvironment(LocalEnvironment): + file_write_lock_scope = "workspace" + + def __init__(self, root, *, remote): + super().__init__(str(root)) + self.local_filesystem = not remote + self.first_entered = asyncio.Event() + self.release_first = asyncio.Event() + self.publications = [] + self.checks = [] + self.remote_lock = asyncio.Lock() + self.remote_lock_held = False + self.lock_paths = [] + + @asynccontextmanager + async def file_write_lock(self, path): + assert Path(path).parent == Path(self.workspace) + assert not Path(path).exists() + self.lock_paths.append(path) + async with self.remote_lock: + self.remote_lock_held = True + try: + yield path + finally: + self.remote_lock_held = False + + async def exec_cmd(self, command, timeout=120.0, **kwargs): + if " apply --check --binary " in command or " apply --binary " in command: + if not self.local_filesystem: + assert self.remote_lock_held + label = _ADOPTER.get() + if " apply --check --binary " in command: + self.checks.append(label) + else: + self.publications.append(label) + if label == "A": + self.first_entered.set() + await self.release_first.wait() + return await super().exec_cmd(command, timeout, **kwargs) + + +class _QueuedCandidates(EnvCandidateWorkspace): + def __init__(self, environment): + super().__init__(environment) + self.second_requested = asyncio.Event() + + async def adopt_run(self, candidate, preserve_paths=()): + if candidate.label == "B": + self.second_requested.set() + await super().adopt_run(candidate, preserve_paths) + + +@pytest.mark.parametrize("remote", [False, True]) +@pytest.mark.parametrize("entry,cancel,overlap", [ + ("sdk", False, False), ("sdk", False, True), + ("direct", True, False), ("sdk", True, False), +]) +async def test_candidate_publication_serializes_snapshots_and_queued_cancellation( + tmp_path, remote, entry, cancel, overlap, +): + repo = _repository(tmp_path) + index = (repo / ".git/index").read_bytes() + environment = _OrderedEnvironment(repo, remote=remote) + backend = _QueuedCandidates(environment) + + async def edit(child, inputs): + (Path(child.workspace_root) / inputs["path"]).write_text(inputs["contents"]) + return "prepared" + + async def flow(ctx, inputs): + first = await ctx.candidate_workflow(edit, {"path": "app.py", "contents": "print(42)\n"}, label="A") + second_path = "app.py" if overlap else "other.py" + second = await ctx.candidate_workflow( + edit, {"path": second_path, "contents": "print(99)\n"}, label="B", + ) + + async def adopt(candidate, selected_entry): + token = _ADOPTER.set(candidate.label) + try: + if selected_entry == "direct": + await backend.adopt_run(candidate) + else: + await ctx.adopt_candidate(candidate) + finally: + _ADOPTER.reset(token) + + first_task = asyncio.create_task(adopt(first, "sdk")) + second_task = None + try: + await asyncio.wait_for(environment.first_entered.wait(), 10) + second_task = asyncio.create_task(adopt(second, entry)) + await asyncio.wait_for(backend.second_requested.wait(), 10) + await asyncio.sleep(0.05) + assert not second_task.done() + assert environment.checks == environment.publications == ["A"] + if cancel: + second_task.cancel() + await asyncio.sleep(0) + second_task.cancel() + environment.release_first.set() + await first_task + assert _run(repo) == "42\n" + if cancel: + with pytest.raises(asyncio.CancelledError): + await second_task + elif overlap: + with pytest.raises(RuntimeError, match="patch does not apply"): + await second_task + else: + await second_task + assert _run(repo) == "42\n" + expected_second = "VALUE = 1\n" if overlap or cancel and entry == "direct" else "print(99)\n" + assert (repo / "other.py").read_text() == expected_second + assert environment.publications == (["A"] if overlap or cancel and entry == "direct" else ["A", "B"]) + return "serialized" + finally: + environment.release_first.set() + await asyncio.gather(first_task, *(task for task in (second_task,) if task is not None), + return_exceptions=True) + + try: + result = await OpenCollab(repo, environment=environment).workflow( + flow, candidate_workspace=backend, trace=False, timeout=30, + ) + assert result.ok, result.reason + assert result.output == "serialized" + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not environment.revoked + if remote: + assert environment.lock_paths + finally: + await environment.cleanup() diff --git a/tests/workflows/test_candidate_adoption_external_writer.py b/tests/workflows/test_candidate_adoption_external_writer.py new file mode 100644 index 00000000..fd9b67bd --- /dev/null +++ b/tests/workflows/test_candidate_adoption_external_writer.py @@ -0,0 +1,344 @@ +from __future__ import annotations + +import asyncio +import os +import shlex +import stat +import sys +import tempfile +from dataclasses import replace +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters.env import LocalEnvironment +from opencollab.adapters.tools.bash import BashTool +from opencollab.application.tool_execution import ToolRuntime +from tests.workflows.test_candidate_adoption_recovery import _partial_publication +from tests.workflows.test_workflow_candidate_workspace import _git + +_WRITER = """ +import os +import sys + +with open(sys.argv[1], "ab") as stream: + opened = os.fstat(stream.fileno()) + print(opened.st_dev, opened.st_ino, flush=True) + data = sys.stdin.buffer.read() + if not data: + raise RuntimeError("writer exited before release") + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) +print("written") +""" + + +def _identity(path: Path) -> tuple[int, int]: + status = path.stat() + return status.st_dev, status.st_ino + + +def _delete_original(candidate): + _, replacement = candidate.diff.split("diff --git a/settings", 1) + return replace(candidate, diff=( + "diff --git a/record.txt b/record.txt\ndeleted file mode 100644\n" + "--- a/record.txt\n+++ /dev/null\n@@ -1 +0,0 @@\n-original\n" + "diff --git a/settings" + replacement + )) + + +async def test_candidate_adopts_with_readonly_common_git_directory(tmp_path): + source, candidate = _partial_publication(tmp_path) + repo = tmp_path / "linked" + _git(source, "worktree", "add", "--detach", str(repo)) + (repo / "user-note.txt").write_text("staged user note\n") + _git(repo, "add", "--", "user-note.txt") + index = Path(_git(repo, "rev-parse", "--absolute-git-dir").removesuffix("\n")) / "index" + index_before = index.read_bytes() + common = source / ".git" + common_mode = stat.S_IMODE(common.stat().st_mode) + target = repo / "record.txt" + environment = LocalEnvironment(str(repo)) + + async def flow(ctx, _): + await ctx.adopt_candidate(candidate) + + try: + common.chmod(0o555) + if os.access(common, os.W_OK): + pytest.skip("requires an account subject to directory permissions") + assert os.access(index.parent, os.W_OK) and os.access(repo, os.W_OK) + with target.open("rb") as original: + result = await OpenCollab(repo, environment=environment, model="offline").workflow( + flow, trace=False, timeout=30, + ) + assert result.ok, result.reason + assert target.read_bytes() == b"candidate\n" + assert (repo / "settings").read_bytes() == b"replacement settings\n" + assert index.read_bytes() == index_before + assert original.read() == b"original\n" + assert os.fstat(original.fileno()).st_nlink == 0 + assert stat.S_IMODE(common.stat().st_mode) == 0o555 + for storage in (common, index.parent, tmp_path, repo): + assert not list(storage.glob("*candidate-recovery-*")) + assert not list(index.parent.glob("opencollab-candidate-*")) + finally: + common.chmod(common_mode) + await environment.cleanup() + + +@pytest.fixture +def worktree_parent(tmp_path, request): + if not request.param: + yield tmp_path + return + shared_memory = Path("/dev/shm") + if not shared_memory.is_dir() or not os.access(shared_memory, os.W_OK): + pytest.skip("requires writable shared memory on a separate filesystem") + if shared_memory.stat().st_dev == tmp_path.stat().st_dev: + pytest.skip("requires Git metadata and worktree on different filesystems") + with tempfile.TemporaryDirectory(prefix="oc-worktree-", dir=shared_memory) as path: + yield Path(path) + + +@pytest.mark.parametrize("opened_before_apply,linked,worktree_parent,hard_links,readonly_common", [ + pytest.param(True, False, False, True, False, id="original-inode"), + pytest.param(False, True, False, True, False, id="candidate-inode-after-worktree-removal"), + pytest.param(False, True, True, True, False, id="separate-worktree-filesystem"), + pytest.param(True, True, False, False, False, id="original-inode-without-hard-links"), + pytest.param(True, True, False, True, True, id="readonly-common-original-inode"), + pytest.param(False, True, False, True, True, id="readonly-common-candidate-inode"), +], indirect=["worktree_parent"]) +async def test_candidate_rollback_retains_inode_for_external_writer( + tmp_path, monkeypatch, caplog, opened_before_apply, linked, worktree_parent, hard_links, readonly_common, +): + source, candidate = _partial_publication(tmp_path) + repo = source + if linked: + repo = worktree_parent / "linked" + _git(source, "worktree", "add", "--detach", str(repo)) + (repo / "settings").mkdir() + (repo / "settings/notes.txt").write_text("existing settings\n") + (repo / "user-note.txt").write_text("staged user note\n") + _git(repo, "add", "--", "user-note.txt") + target = repo / "record.txt" + original = target.read_bytes() + original_inode = _identity(target) + published = b"candidate\n" + appended = b"external write after rollback\n" + index = Path(_git(repo, "rev-parse", "--absolute-git-dir").removesuffix("\n")) / "index" + index_before = index.read_bytes() + common = source / ".git" + common_mode = stat.S_IMODE(common.stat().st_mode) + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + writer = None + writer_inode = None + + async def start_writer(expected: bytes) -> None: + nonlocal writer, writer_inode + assert writer is None + assert target.read_bytes() == expected + writer = await asyncio.create_subprocess_exec( + sys.executable, "-c", _WRITER, str(target), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + assert writer.stdout is not None + ready = await asyncio.wait_for(writer.stdout.readline(), 10) + writer_inode = _identity(target) + assert ready.split() == [str(value).encode() for value in writer_inode] + + async def observe(command, timeout=120.0, **kwargs): + if not hard_links and "hold_originals()" in command: + command = "ln() { return 1; }; " + command + if " apply --binary " not in command or ".candidate-adopt-" not in command: + return await execute(command, timeout, **kwargs) + outcome = await execute(command, timeout, **kwargs) + assert outcome.returncode == 128, outcome.stderr + assert target.read_bytes() == published + assert _identity(target) != original_inode + if not opened_before_apply: + await start_writer(published) + return outcome + + monkeypatch.setattr(environment, "exec_cmd", observe) + + async def flow(ctx, _): + await ctx.adopt_candidate(candidate) + + try: + if readonly_common: + common.chmod(0o555) + if os.access(common, os.W_OK): + pytest.skip("requires an account subject to directory permissions") + assert os.access(index.parent, os.W_OK) and os.access(repo, os.W_OK) + if opened_before_apply: + await start_writer(original) + result = await OpenCollab(repo, environment=environment, model="offline").workflow( + flow, trace=False, timeout=30, + ) + assert type(result.error) is RuntimeError, result.reason + assert "candidate adoption failed" in result.reason + assert "settings" in result.reason + assert target.read_bytes() == original + assert (repo / "settings/notes.txt").read_bytes() == b"existing settings\n" + assert index.read_bytes() == index_before + assert writer is not None, result.reason + assert writer_inode is not None + + if linked: + _git(source, "worktree", "remove", "--force", str(repo)) + assert not repo.exists() + assert not index.exists() + assert writer.returncode is None + stdout, stderr = await asyncio.wait_for(writer.communicate(appended), 10) + assert writer.returncode == 0, stderr + assert stdout == b"written\n" + assert (source / "record.txt").read_bytes() == original + + storage = common if worktree_parent == tmp_path and not readonly_common else worktree_parent + backups = list(storage.glob("candidate-recovery-*")) + assert backups, "candidate rollback left no recovery storage for the open descriptor" + backup, = backups + assert all(path.name == "original-inodes" or path.name.startswith("observed-") for path in backup.iterdir()) + if opened_before_apply: + path_record, = (path for path in backup.glob("original-inodes/*.path") if path.read_text() == str(target)) + retained = path_record.with_suffix("") + initial = original + else: + path_record, = (path for path in backup.glob("observed-*/path") if path.read_text() == str(target)) + retained = path_record.with_name("entry") + initial = published + assert _identity(retained) == writer_inode + assert retained.read_bytes() == initial + appended + assert any(str(backup) in message for message in caplog.messages) + assert any(str(backup) in note for note in getattr(result.error, "__notes__", ())) + finally: + if readonly_common: + common.chmod(common_mode) + if writer is not None and writer.returncode is None: + writer.kill() + await writer.communicate() + await environment.cleanup() + + +@pytest.mark.parametrize("external_delete", [False, True], ids=["interrupted-move", "external-delete"]) +async def test_failed_preparation_recovers_only_owned_moves(tmp_path, monkeypatch, external_delete): + repo, candidate = _partial_publication(tmp_path) + if external_delete: + candidate = _delete_original(candidate) + target = repo / "record.txt" + identity = _identity(target) + index = repo / ".git/index" + index_before = index.read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + reached_activation = False + published = False + + async def interrupt(command, timeout=120.0, **kwargs): + nonlocal reached_activation, published + if "hold_originals()" in command: + command = "ln() { return 1; }; " + command + if 'restore_saved "$replacement" "$live"' in command: + reached_activation = True + if external_delete: + writer = await asyncio.create_subprocess_exec( + sys.executable, "-c", "import os, sys; os.unlink(sys.argv[1])", str(target), + ) + assert await asyncio.wait_for(writer.wait(), 10) == 0 + else: + command = r''' +mv() { + case "$3" in + */replacements/*) printf 'replacement move interrupted\n' >&2; return 1;; + *) command mv "$@";; + esac +} +''' + command + if " apply --binary " in command and ".candidate-adopt-" in command: + published = True + return await execute(command, timeout, **kwargs) + + monkeypatch.setattr(environment, "exec_cmd", interrupt) + + async def flow(ctx, _): + await ctx.adopt_candidate(candidate) + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow(flow, trace=False) + assert reached_activation and not published + assert type(result.error) is RuntimeError + if external_delete: + assert "concurrent change" in result.reason + assert not target.exists() + else: + assert "replacement move interrupted" in result.reason + assert _identity(target) == identity + assert target.read_bytes() == b"original\n" + assert index.read_bytes() == index_before + assert not list((repo / ".git").glob("candidate-recovery-*")) + finally: + await environment.cleanup() + + +async def test_detached_original_only_preserves_publication_error_during_shell_execution(tmp_path, monkeypatch): + repo, candidate = _partial_publication(tmp_path) + candidate = _delete_original(candidate) + target = repo / "record.txt" + original_inode = _identity(target) + index = repo / ".git/index" + index_before = index.read_bytes() + ready = tmp_path / "shell-ready" + release = tmp_path / "shell-release" + command = ( + f"touch {shlex.quote(str(ready))} && " + f"while [ ! -f {shlex.quote(str(release))} ]; do sleep 0.01; done" + ) + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + + async def observe(command, timeout=120.0, **kwargs): + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command and ".candidate-adopt-" in command: + assert result.returncode == 128, result.stderr + assert not target.exists() + return result + + monkeypatch.setattr(environment, "exec_cmd", observe) + + async def wait_for_shell(): + while not ready.exists(): + await asyncio.sleep(0.01) + + async def flow(ctx, _): + shell = asyncio.create_task(BashTool().execute_with_runtime( + {"command": command, "timeout": 20}, ToolRuntime(environment, None, None), + )) + try: + await asyncio.wait_for(wait_for_shell(), 10) + assert not shell.done() + await ctx.adopt_candidate(candidate) + finally: + release.touch() + assert await shell == "Exit code: 0" + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow( + flow, trace=False, timeout=30, + ) + assert type(result.error) is RuntimeError, result.reason + assert "candidate adoption failed" in result.reason + assert target.read_bytes() == b"original\n" + assert index.read_bytes() == index_before + backup, = (repo / ".git").glob("candidate-recovery-*") + record, = (backup / "original-inodes").glob("*.path") + assert record.read_text() == str(target) + assert _identity(record.with_suffix("")) == original_inode + assert not list(backup.glob("observed-*")) + finally: + await environment.cleanup() diff --git a/tests/workflows/test_candidate_adoption_mounts.py b/tests/workflows/test_candidate_adoption_mounts.py new file mode 100644 index 00000000..e797c060 --- /dev/null +++ b/tests/workflows/test_candidate_adoption_mounts.py @@ -0,0 +1,277 @@ +from __future__ import annotations + +import os +import shlex +import stat +from dataclasses import replace +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace +from opencollab.adapters.env import LocalEnvironment +from opencollab.application.workflow_candidates import CandidateWorkspaceTrackingError +from tests.workflows.test_candidate_adoption_recovery import _partial_publication +from tests.workflows.test_workflow_candidate_workspace import _git + + +def _mount_commands(repo: Path, mount: Path, *, bind: bool, hard_links: bool) -> str: + return ( + f"volume={shlex.quote(str(mount))}; root={shlex.quote(str(repo))}; " + f"volume_device={1 if bind else 2}; volume_links={int(hard_links)};\n" + + r''' +stat() { + case "$1:$2" in + -c:%d|-c:%d:%m) + mountpoint=$root; device_id=1 + case "$4" in "$volume"|"$volume"/*) mountpoint=$volume; device_id=$volume_device;; esac + if [ "$2" = %d ]; then printf '%s\n' "$device_id"; else printf '%s:%s\n' "$device_id" "$mountpoint"; fi + ;; + -c:%a) + case "$4" in + "$volume"/*) if [ "$volume_links" -eq 0 ]; then printf '755\n'; return; fi;; + esac + command stat "$@" + ;; + *) command stat "$@";; + esac +} +ln() { + if [ "$1" = -P ]; then + case "$3" in + "$volume"/*) + case "$4" in + "$volume"/*) [ "$volume_links" -eq 1 ] || return 1;; + *) return 1;; + esac + ;; + esac + fi + command ln "$@" +} +''' + ) + + +@pytest.mark.parametrize("scenario,bind,hard_links", [ + pytest.param("success", False, True, id="mixed-device-success"), + pytest.param("success", True, True, id="bind-mount-success"), + pytest.param("modified", False, True, id="restore-mounted-file"), + pytest.param("modified", False, False, id="restore-mounted-file-without-hard-links"), + pytest.param("added", True, True, id="park-new-file-on-bind-mount"), +]) +async def test_candidate_adoption_with_nested_mount(tmp_path, monkeypatch, scenario, bind, hard_links): + repo, candidate = _partial_publication(tmp_path) + mount = repo / "mounted" + mount.mkdir(exist_ok=True) + target = mount / "record.txt" + if scenario == "added": + header = "new file mode 100644\n--- /dev/null\n+++ b/mounted/record.txt\n@@ -0,0 +1 @@\n" + original = b"" + else: + original = b"original mount\n" + target.write_bytes(original) + _git(repo, "add", "mounted/record.txt") + _git(repo, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "commit", "-qm", "test: track mounted file") + header = "--- a/mounted/record.txt\n+++ b/mounted/record.txt\n@@ -1 +1 @@\n-original mount\n" + local_patch, blocked = candidate.diff.split("diff --git a/settings", 1) + diff = local_patch + "diff --git a/mounted/record.txt b/mounted/record.txt\n" + header + "+candidate mount\n" + if scenario != "success": + diff += "diff --git a/settings" + blocked + candidate = replace(candidate, diff=diff) + (repo / "user-note.txt").write_text("staged by user\n") + _git(repo, "add", "user-note.txt") + index = repo / ".git/index" + index_before = index.read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + original_fd = os.open(target, os.O_WRONLY | os.O_APPEND) if scenario == "modified" else None + published_fd = None + statuses = [] + + async def observe(command, timeout=120, **kwargs): + nonlocal published_fd + if "filesystem()" in command or "device()" in command: + command = _mount_commands(repo, mount, bind=bind, hard_links=hard_links) + command + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command and ".candidate-adopt-" in command: + statuses.append(result.returncode) + if scenario != "success": + assert target.read_bytes() == b"candidate mount\n" + published_fd = os.open(target, os.O_WRONLY | os.O_APPEND) + return result + + monkeypatch.setattr(environment, "exec_cmd", observe) + + async def flow(ctx, _): + await ctx.adopt_candidate(candidate) + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow(flow, trace=False) + assert index.read_bytes() == index_before + assert (repo / "settings/notes.txt").read_bytes() == b"existing settings\n" + if scenario == "success": + assert result.ok, result.reason + assert statuses == [0] + assert target.read_bytes() == b"candidate mount\n" + assert (repo / "record.txt").read_bytes() == b"candidate\n" + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not list(mount.glob(".candidate-recovery-*")) + return + + assert type(result.error) is RuntimeError and "candidate adoption failed" in result.reason, result.reason + assert statuses == [128] + assert (repo / "record.txt").read_bytes() == b"original\n" + if original: + assert target.read_bytes() == original + else: + assert not target.exists() + recovery, = (repo / ".git").glob("candidate-recovery-*") + location, = (recovery / "volumes").iterdir() + store = Path(location.read_text()) + assert store.parent == mount + assert not (recovery / "replay").exists() + assert not list(store.glob("restore-*")) + assert published_fd is not None + parked, = store.glob("observed-*/entry") + assert parked.with_name("path").read_text() == str(target) + retained = [(published_fd, parked, b"candidate mount\n")] + if original_fd is not None: + record, = store.glob("original-inodes/[0-9]*.path") + assert record.read_text() == str(target) + retained.append((original_fd, record.with_suffix(""), original)) + for fd, saved, initial in retained: + os.write(fd, b"late write\n") + os.fsync(fd) + assert os.fstat(fd).st_ino == saved.stat().st_ino + assert saved.read_bytes() == initial + b"late write\n" + finally: + if original_fd is not None: + os.close(original_fd) + if published_fd is not None: + os.close(published_fd) + await environment.cleanup() + + +@pytest.mark.parametrize(("source_edit", "linked", "readonly_common"), [ + (None, False, False), + (None, True, False), + pytest.param(None, True, True, id="linked-permissions-restored"), + ("tracked", False, False), + ("untracked", False, False), +]) +async def test_followup_candidate_ignores_only_registered_mount_recovery( + tmp_path, monkeypatch, source_edit, linked, readonly_common, +): + source, candidate = _partial_publication(tmp_path) + (source / "app.py").write_text("value = 1\n") + mounted_file = "mounted/record.txt" + (source / "mounted").mkdir() + (source / mounted_file).write_text("original\n") + (source / ".gitignore").write_text("settings/\n") + if linked: + (source / ".gitattributes").write_text("*.txt text eol=lf\n") + _git(source, "add", ".") + _git(source, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare mounted source") + repo = source + if linked: + repo = tmp_path / "linked" + _git(source, "worktree", "add", "--detach", str(repo)) + (repo / "settings").mkdir() + (repo / "settings/notes.txt").write_text("existing settings\n") + + mount = repo / "mounted" + target = repo / mounted_file + candidate = replace(candidate, diff=candidate.diff.replace("record.txt", mounted_file)) + user_data = mount / ".candidate-recovery-userdata" + user_data.mkdir() + user_note = user_data / "note.txt" + user_note.write_text("user data\n") + index = Path(_git(repo, "rev-parse", "--absolute-git-dir").removesuffix("\n")) / "index" + index_before = index.read_bytes() + common = source / ".git" + common_mode = stat.S_IMODE(common.stat().st_mode) + environment = LocalEnvironment(str(repo)) + workspace = EnvCandidateWorkspace(environment) + execute = environment.exec_cmd + + async def with_mount(command, timeout=120, **kwargs): + if "filesystem()" in command: + command = _mount_commands(repo, mount, bind=False, hard_links=True) + command + return await execute(command, timeout, **kwargs) + + monkeypatch.setattr(environment, "exec_cmd", with_mount) + store_path = None + retained = None + with target.open("ab", buffering=0) as old_writer: + async def edit(child, _): + worktree = Path(child.workspace_root) + assert not (worktree / store_path).exists() + assert (worktree / user_note.relative_to(repo)).read_text() == "user data\n" + (worktree / "app.py").write_text("value = 2\n") + old_writer.write(b"late original write\n") + if source_edit == "tracked": + (repo / "record.txt").write_text("tracked source edit\n") + elif source_edit == "untracked": + user_note.write_text("untracked source edit\n") + + async def flow(ctx, _): + nonlocal store_path, retained + try: + if readonly_common: + common.chmod(0o555) + if os.access(common, os.W_OK): + pytest.skip("requires an account subject to directory permissions") + with pytest.raises(RuntimeError, match="candidate adoption failed"): + await ctx.adopt_candidate(candidate) + finally: + if readonly_common: + common.chmod(common_mode) + _git(repo, "restore", "--source=HEAD", "--worktree", mounted_file) + primary = repo.parent if readonly_common else common + recovery, = primary.glob("candidate-recovery-*") + location, = (recovery / "volumes").iterdir() + store = Path(location.read_text()) + store_path = store.relative_to(repo) + retained, = (store / "original-inodes").glob("[0-9]") + assert retained.stat().st_ino == os.fstat(old_writer.fileno()).st_ino + + before = await workspace.source_diff() + assert str(store_path) not in before + assert "+user data\n" in before + if linked: + lease = await workspace._acquire("team follow-up", strict_capture=True) + try: + tree = Path(lease.candidate_workspace) + assert not (tree / store_path).exists() + assert (tree / user_note.relative_to(repo)).read_text() == "user data\n" + assert not await lease.source_contents_changed() + finally: + await lease.cleanup() + if source_edit is None: + followup = await ctx.candidate_workflow(edit, {}, label="unrelated candidate") + assert "+value = 2" in followup.diff + await ctx.adopt_candidate(followup) + else: + try: + with pytest.raises(CandidateWorkspaceTrackingError, match="source worktree changed"): + await ctx.candidate_workflow(edit, {}, label="unrelated candidate") + finally: + registered = _git(repo, "worktree", "list", "--porcelain").splitlines() + for entry in registered: + if entry.startswith("worktree ") and Path(entry[9:]) != repo: + _git(repo, "worktree", "remove", "--force", entry[9:]) + assert await workspace.source_diff() != before + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow(flow, trace=False) + assert result.ok, result.reason + assert target.read_text() == "original\n" + assert retained.read_bytes() == b"original\nlate original write\n" + assert (repo / "app.py").read_text() == ("value = 1\n" if source_edit else "value = 2\n") + assert index.read_bytes() == index_before + finally: + await environment.cleanup() diff --git a/tests/workflows/test_candidate_adoption_recovery.py b/tests/workflows/test_candidate_adoption_recovery.py new file mode 100644 index 00000000..dfaf2a3b --- /dev/null +++ b/tests/workflows/test_candidate_adoption_recovery.py @@ -0,0 +1,853 @@ +from __future__ import annotations + +import errno +import json +import os +import shlex +import shutil +import stat +import subprocess +import sys +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters._candidate_increment import _apply_increment +from opencollab.adapters._env_base import ExecResult +from opencollab.adapters._env_local import LocalEnvironment +from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace +from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError +from opencollab.workflows import CandidateRun +from tests.workflows.test_workflow_candidate_workspace import _git + + +def _run(repo, *, cache=False): + return subprocess.run( + [sys.executable, *([] if cache else ["-B"]), "app.py"], cwd=repo, + capture_output=True, text=True, check=True, + ).stdout + + +def _contents(root): + result = {} + for path in sorted(root.rglob("*")): + relative = path.relative_to(root) + if ".git" in relative.parts: + continue + mode = path.lstat().st_mode + result[str(relative)] = (stat.S_IFMT(mode), stat.S_IMODE(mode), + os.readlink(path) if path.is_symlink() else + path.read_bytes() if path.is_file() else None) + return result + + +def _detached_files(storage: Path) -> tuple[dict[Path, bytes], dict[Path, bytes]]: + backup, = storage.glob("candidate-recovery-*") + assert not (backup / "replay").exists() + originals = { + Path(record.read_text()): record.with_suffix("").read_bytes() + for record in backup.glob("original-inodes/*.path") + } + parked = { + Path(record.read_text()): record.with_name("entry").read_bytes() + for record in backup.glob("observed-*/path") + if not record.with_name("entry").is_symlink() + } + return originals, parked + + +def _repository(tmp_path, *, physical=False, cache=False): + repo = tmp_path / "repo" + repo.mkdir() + (repo / "settings").mkdir() + (repo / "settings/defaults.py").write_text("VALUE = 1\n") + (repo / "loader.py").write_text("from settings.defaults import VALUE\ndef value():\n return VALUE\n") + (repo / "app.py").write_text("from loader import value\nprint(value())\n") + (repo / "other.py").write_text("VALUE = 1\n") + (repo / "note.txt").write_text("committed\n") + (repo / ".gitignore").write_text("__pycache__/\nignored-link\n") + if physical: + (repo / ".gitattributes").write_text("*.py text eol=crlf\n") + _git(repo, "init", "-q") + _git(repo, "config", "user.name", "Test") + _git(repo, "config", "user.email", "test@example.invalid") + _git(repo, "add", ".") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare candidate recovery") + for name in ("note.txt", "loader.py"): + with (repo / name).open("a") as stream: + stream.write("# staged user change\n") + _git(repo, "add", name) + with (repo / name).open("a") as stream: + stream.write("# unstaged user change\n") + (repo / "draft.txt").write_text("untracked user draft\n") + (repo / "draft.txt").chmod(0o600) + (repo / "loader.py").chmod(0o750) + (repo / "settings").chmod(0o750) + (repo / "settings/defaults.py").chmod(0o640) + assert _run(repo, cache=cache) == "1\n" + if cache: + (repo / "settings/ignored-link").symlink_to("../../outside-user-file") + assert list((repo / "settings/__pycache__").glob("*.pyc")) + return repo + + +def _flatten(folder): + (folder / "settings/defaults.py").unlink() + (folder / "settings").rmdir() + (folder / "settings").write_text("42\n") + (folder / "loader.py").write_text( + 'from pathlib import Path\ndef value():\n return int(Path("settings").read_text())\n' + ) + (folder / "draft.txt").write_text("candidate draft\n") + + +def _partial_publication(tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q") + (repo / "record.txt").write_text("original\n") + _git(repo, "add", "record.txt") + _git(repo, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare partial publication") + (repo / "settings").mkdir() + (repo / "settings/notes.txt").write_text("existing settings\n") + patch = ( + "diff --git a/record.txt b/record.txt\n--- a/record.txt\n+++ b/record.txt\n" + "@@ -1 +1 @@\n-original\n+candidate\n" + "diff --git a/settings b/settings\nnew file mode 100644\n--- /dev/null\n+++ b/settings\n" + "@@ -0,0 +1 @@\n+replacement settings\n" + ) + return repo, CandidateRun("record update", None, patch, (), ()) + + +@pytest.mark.parametrize("physical", [False, True]) +@pytest.mark.parametrize("cache", [False, True]) +async def test_public_sdk_candidate_recovers_directory_replacement(tmp_path, physical, cache): + repo = _repository(tmp_path, physical=physical, cache=cache) + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + + async def edit(child, inputs): + _flatten(Path(child.workspace_root)) + assert _run(Path(child.workspace_root)) == "42\n" + return "tested" + + async def flow(ctx, inputs): + candidate = await ctx.candidate_workflow(edit, {}, label="flatten") + assert "+42" in candidate.effective_diff + if cache: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await ctx.adopt_candidate(candidate) + assert _contents(repo) == before + else: + await ctx.adopt_candidate(candidate) + return "finished" + + result = await OpenCollab(repo).workflow(flow, trace=False) + assert result.ok, result.reason + assert _run(repo) == ("1\n" if cache else "42\n") + assert (repo / ".git/index").read_bytes() == index + if cache: + originals, parked = _detached_files(repo / ".git") + assert originals[repo / "settings/defaults.py"] == before["settings/defaults.py"][2] + assert parked[repo / "draft.txt"] == b"candidate draft\n" + else: + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert _git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1 + + +@pytest.mark.parametrize("physical", [False, True]) +@pytest.mark.parametrize("pruned", ["both", "nested", "none"]) +@pytest.mark.parametrize("cache", [False, True]) +async def test_public_sdk_recovers_pruned_ancestors_and_modes(tmp_path, physical, pruned, cache): + repo = _repository(tmp_path, physical=physical, cache=cache) + (repo / "old/nested").mkdir(parents=True) + (repo / "old/nested/helper.py").write_text("VALUE = 1\n") + _git(repo, "add", "old/nested/helper.py") + (repo / "old").chmod(0o750) + (repo / "old/nested").chmod(0o710) + (repo / "old/nested/helper.py").chmod(0o640) + if pruned != "both": + sibling = "old/sibling.txt" if pruned == "nested" else "old/nested/sibling.txt" + (repo / sibling).write_text("user sibling\n") + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + + async def edit(child, inputs): + folder = Path(child.workspace_root) + (folder / "old/nested/helper.py").unlink() + _flatten(folder) + assert _run(folder) == "42\n" + return "tested" + + async def flow(ctx, inputs): + candidate = await ctx.candidate_workflow(edit, {}, label="pruned-parent") + if cache: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await ctx.adopt_candidate(candidate) + assert _contents(repo) == before + else: + await ctx.adopt_candidate(candidate) + assert not (repo / "old/nested/helper.py").exists() + assert (repo / "old").exists() is (pruned != "both") + assert (repo / "old/nested").exists() is (pruned == "none") + assert not environment.revoked + assert (await environment.exec_cmd("printf recovered")).stdout == "recovered" + return "recovered" + + try: + result = await OpenCollab(repo, environment=environment).workflow(flow, trace=False) + assert result.ok, result.reason + assert result.output == "recovered" + assert _run(repo) == ("1\n" if cache else "42\n") + assert (repo / ".git/index").read_bytes() == index + if cache: + originals, _ = _detached_files(repo / ".git") + assert originals[repo / "old/nested/helper.py"] == before["old/nested/helper.py"][2] + else: + assert not list((repo / ".git").glob("candidate-recovery-*")) + finally: + await environment.cleanup() + + +@pytest.mark.parametrize("physical", [False, True]) +@pytest.mark.parametrize("change,sibling", [ + ("rename", False), ("rename", True), ("readonly", True), ("readonly-root", True), +]) +async def test_public_sdk_restores_changed_ancestor_permissions(tmp_path, monkeypatch, physical, change, sibling): + repo = _repository(tmp_path, physical=physical, cache=True) + (repo / "old/nested").mkdir(parents=True) + helper = repo / "old/nested/helper.py" + helper.write_text("VALUE = 1\n") + _git(repo, "add", "old/nested/helper.py") + (repo / "old").chmod(0o750) + (repo / "old/nested").chmod(0o500 if change == "readonly" else 0o710) + if sibling: + (repo / "old/sibling.txt").write_text("shared contents\n") + if change == "readonly-root": + repo.chmod(0o500) + root_mode = stat.S_IMODE(repo.stat().st_mode) + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + commands = tmp_path / "commands" + commands.mkdir() + log = tmp_path / "chmod.log" + chmod = shutil.which("chmod") + assert chmod is not None + (commands / "chmod").write_text( + f'#!/bin/sh\nprintf \'%s\\0\' "$@" >> {shlex.quote(str(log))}\n' + f'exec {shlex.quote(chmod)} "$@"\n' + ) + (commands / "chmod").chmod(0o755) + execute = environment.exec_cmd + + async def recorded(command, timeout=120.0, **kwargs): + if command.startswith("failed=0;"): + command = f"PATH={shlex.quote(str(commands) + os.pathsep + os.environ['PATH'])}; " + command + return await execute(command, timeout, **kwargs) + + monkeypatch.setattr(environment, "exec_cmd", recorded) + + async def edit(child, inputs): + folder = Path(child.workspace_root) + if change == "rename": + (folder / "old/nested/helper.py").rename(folder / "old/nested/renamed.py") + else: + (folder / "old/nested/helper.py").write_text("VALUE = 42\n") + _flatten(folder) + assert _run(folder) == "42\n" + return "tested" + + async def flow(ctx, inputs): + candidate = await ctx.candidate_workflow(edit, {}, label="ancestor-permissions") + if change == "rename": + assert "rename from old/nested/helper.py" in candidate.effective_diff + with pytest.raises(RuntimeError, match="Directory not empty|Permission denied|Is a directory"): + await ctx.adopt_candidate(candidate) + assert _contents(repo) == before + assert not environment.revoked + return "recovered" + + try: + result = await OpenCollab(repo, environment=environment).workflow(flow, trace=False) + assert result.ok, result.reason + assert (repo / ".git/index").read_bytes() == index + originals, _ = _detached_files(repo / ".git") + assert originals[repo / "settings/defaults.py"] == before["settings/defaults.py"][2] + chmod_paths = [Path(value.decode()) for value in log.read_bytes().split(b"\0")[2::3] if value] + assert stat.S_IMODE(repo.stat().st_mode) == root_mode + assert (repo / "old/nested" in chmod_paths) is (change != "readonly-root") + assert (repo / "old" in chmod_paths) is not sibling + assert (repo in chmod_paths) is (change == "readonly-root") + finally: + repo.chmod(0o700) + (repo / "old/nested").chmod(0o700) + await environment.cleanup() + + +async def test_git_write_failure_after_successful_check_restores_partial_application(tmp_path): + repo = _repository(tmp_path) + (repo / "z").mkdir(mode=0o555) + if os.access(repo / "z", os.W_OK): + pytest.skip("requires directory write permissions to be enforced") + patch = ( + "diff --git a/other.py b/other.py\n--- a/other.py\n+++ b/other.py\n" + "@@ -1 +1 @@\n-VALUE = 1\n+VALUE = 42\n" + "diff --git a/z/new.txt b/z/new.txt\nnew file mode 100644\n" + "--- /dev/null\n+++ b/z/new.txt\n@@ -0,0 +1 @@\n+new contents\n" + ) + checked = subprocess.run( + ["git", "-C", str(repo), "apply", "--check"], input=patch, capture_output=True, text=True, + ) + assert checked.returncode == 0, checked.stderr + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + try: + with pytest.raises(RuntimeError, match="unable to write file|Permission denied"): + await EnvCandidateWorkspace(environment).adopt(patch) + assert _contents(repo) == before + assert (repo / ".git/index").read_bytes() == index + originals, parked = _detached_files(repo / ".git") + assert originals[repo / "other.py"] == b"VALUE = 1\n" + assert parked[repo / "other.py"] == b"VALUE = 42\n" + assert not environment.revoked + assert (await environment.exec_cmd("printf usable")).stdout == "usable" + finally: + (repo / "z").chmod(0o755) + await environment.cleanup() + + +@pytest.mark.parametrize(("rules", "blocked"), [ + ("tracked", True), + ("unstaged", True), + ("nested", True), + ("info", True), + ("configured", True), + ("tracked", False), +]) +async def test_public_sdk_replays_effective_crlf_attributes(tmp_path, rules, blocked): + repo = tmp_path + _git(repo, "init", "-q") + _git(repo, "config", "core.autocrlf", "false") + (repo / "nested").mkdir() + target = repo / "nested/a.txt" + target.write_bytes(b"original\n") + attributes = { + "nested": repo / "nested/.gitattributes", + "info": repo / ".git/info/attributes", + "configured": repo / "local-attributes", + }.get(rules, repo / ".gitattributes") + if rules == "configured": + _git(repo, "config", "core.attributesFile", attributes.name) + if rules in {"tracked", "unstaged"}: + eol = "lf" if rules == "unstaged" else "crlf" + attributes.write_text(f"*.txt text eol={eol}\n") + _git(repo, "add", ".gitattributes") + _git(repo, "add", "nested/a.txt") + _git(repo, "-c", "user.name=Test", "-c", "user.email=test@example.invalid", + "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare CRLF recovery") + if rules != "tracked": + attributes.write_text("*.txt text eol=crlf\n") + target.write_bytes(b"original\r\n") + patch = ( + "diff --git a/nested/a.txt b/nested/a.txt\n--- a/nested/a.txt\n+++ b/nested/a.txt\n" + "@@ -1 +1 @@\n-original\n+candidate\n" + ) + if blocked: + (repo / "settings").mkdir() + (repo / "settings/cache.txt").write_text("preserve cache\n") + patch += ( + "diff --git a/settings b/settings\nnew file mode 100644\n--- /dev/null\n+++ b/settings\n" + "@@ -0,0 +1 @@\n+new settings\n" + ) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + publications = [] + replays = [] + change_rules = rules in {"nested", "configured"} + + async def observe(command, timeout=120.0, **kwargs): + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command: + publications.append((result.returncode, target.read_bytes())) + if change_rules: + attributes.write_text("*.txt text eol=lf\n") + if "candidate-recovery-" in command and " apply " in command: + replays.append(result.returncode) + return result + + environment.exec_cmd = observe + + async def flow(ctx, inputs): + candidate = CandidateRun("text update", "ready", patch, (), ()) + await ctx.adopt_candidate(candidate) + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow(flow, trace=False) + assert publications == [(128 if blocked else 0, b"candidate\r\n")] + if blocked: + assert type(result.error) is RuntimeError + assert "Directory not empty" in result.reason + assert replays == [0, 128] + assert target.read_bytes() == b"original\r\n" + assert (repo / "settings/cache.txt").read_bytes() == b"preserve cache\n" + originals, parked = _detached_files(repo / ".git") + assert originals[target] == b"original\r\n" + assert parked[target] == b"candidate\r\n" + else: + assert result.ok, result.reason + assert target.read_bytes() == b"candidate\r\n" + assert replays == [] + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert attributes.read_text() == f"*.txt text eol={'lf' if change_rules else 'crlf'}\n" + assert (repo / ".git/index").read_bytes() == index + assert not environment.revoked + finally: + await environment.cleanup() + + +@pytest.mark.parametrize(("replay_status", "target_kind"), [ + (1, "existing"), (128, "existing"), (128, "new"), (128, "sibling"), +]) +async def test_unverified_candidate_replay_preserves_live_entries(tmp_path, replay_status, target_kind): + repo = _repository(tmp_path) + if target_kind == "existing": + target = repo / "other.py" + patch = ( + "diff --git a/other.py b/other.py\n--- a/other.py\n+++ b/other.py\n" + "@@ -1 +1 @@\n-VALUE = 1\n+VALUE = 42\n" + ) + else: + target = repo / "new/inner/file.py" + patch = ( + "diff --git a/new/inner/file.py b/new/inner/file.py\nnew file mode 100644\n" + "--- /dev/null\n+++ b/new/inner/file.py\n@@ -0,0 +1 @@\n+VALUE = 42\n" + ) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + + async def fault(command, timeout=120.0, **kwargs): + if " apply --whitespace=nowarn --binary " in command: + return ExecResult(replay_status, "", "replay interrupted before writing") + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command: + assert result.returncode == 0, result.stderr + if target_kind == "sibling": + target.unlink() + target.with_name("notes.txt").write_text("unrelated notes\n") + raise OSError("publication receipt unavailable") + return result + + environment.exec_cmd = fault + try: + if target_kind == "sibling": + with pytest.raises(OSError, match="publication receipt unavailable"): + await _apply_increment(environment, str(repo), patch) + assert target.with_name("notes.txt").read_text() == "unrelated notes\n" + assert not target.exists() + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not environment.revoked + else: + with pytest.raises(CandidateAdoptionRecoveryError, match="original files retained"): + await _apply_increment(environment, str(repo), patch) + assert target.read_text() == "VALUE = 42\n" + backup, = (repo / ".git").glob("candidate-recovery-*") + record, = backup.glob("*.path") + if target_kind == "existing": + assert record.with_suffix("").read_bytes() == b"VALUE = 1\n" + else: + assert Path(record.read_text()) == repo / "new" + assert not record.with_suffix("").exists() + assert environment.revoked + assert (repo / ".git/index").read_bytes() == index + finally: + await environment.cleanup() + + +@pytest.mark.parametrize("inside_repository", [False, True]) +async def test_directory_copy_candidate_replays_partial_apply_outside_git(tmp_path, monkeypatch, inside_repository): + monkeypatch.setenv("TMPDIR", str(tmp_path)) + if inside_repository: + _git(tmp_path, "init", "-q") + repo = tmp_path / "copy" + (repo / "settings").mkdir(parents=True) + (repo / "settings/retained.txt").write_text("keep\n") + (repo / "a.txt").write_text("original\n") + patch = ( + "diff --git a/a.txt b/a.txt\n--- a/a.txt\n+++ b/a.txt\n" + "@@ -1 +1 @@\n-original\n+candidate\n" + "diff --git a/settings b/settings\nnew file mode 100644\n--- /dev/null\n+++ b/settings\n" + "@@ -0,0 +1 @@\n+candidate settings\n" + ) + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + intermediate = [] + + async def inspect(command, timeout=120.0, **kwargs): + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command: + intermediate.append((repo / "a.txt").read_text()) + return result + + environment.exec_cmd = inspect + try: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await _apply_increment(environment, str(repo), patch, repository=False) + assert intermediate == ["candidate\n"] + assert (repo / "a.txt").read_text() == "original\n" + assert (repo / "settings/retained.txt").read_text() == "keep\n" + originals, parked = _detached_files(tmp_path) + assert originals[repo / "a.txt"] == b"original\n" + assert parked[repo / "a.txt"] == b"candidate\n" + assert not environment.revoked + finally: + await environment.cleanup() + + +@pytest.mark.parametrize("replay_full", [False, True]) +async def test_recovery_preserves_saved_entries_when_copies_fail(tmp_path, replay_full): + repo = _repository(tmp_path) + target = repo / "other.py" + patch = ( + "diff --git a/other.py b/other.py\n--- a/other.py\n+++ b/other.py\n" + "@@ -1 +1 @@\n-VALUE = 1\n+VALUE = 42\n" + ) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + failures = [] + + async def fault(command, timeout=120.0, **kwargs): + if command.startswith("present()") and replay_full: + failures.append("replay") + raise OSError(errno.ENOSPC, "No space left on device") + if command.startswith("failed=0;"): + command = "cp() { echo 'restore copy: No space left on device' >&2; return 1; }; " + command + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command and ".candidate-adopt-" in command: + assert result.returncode == 0, result.stderr + assert target.read_text() == "VALUE = 42\n" + if replay_full: + target.write_text("VALUE = 100\n") + failures.append("receipt") + raise OSError("publication receipt unavailable") + return result + + environment.exec_cmd = fault + try: + if replay_full: + detail = "No space left on device.*original files retained" + with pytest.raises(CandidateAdoptionRecoveryError, match=detail): + await _apply_increment(environment, str(repo), patch) + backup, = (repo / ".git").glob("candidate-recovery-*") + record, = backup.glob("*.path") + assert record.with_suffix("").read_text() == "VALUE = 1\n" + assert target.read_text() == "VALUE = 100\n" + assert failures == ["receipt", "replay"] + else: + with pytest.raises(OSError, match="publication receipt unavailable"): + await _apply_increment(environment, str(repo), patch) + assert target.read_text() == "VALUE = 1\n" + originals, parked = _detached_files(repo / ".git") + assert originals[target] == b"VALUE = 1\n" + assert parked[target] == b"VALUE = 42\n" + assert failures == ["receipt"] + assert environment.revoked is replay_full + assert (repo / ".git/index").read_bytes() == index + finally: + await environment.cleanup() + + +async def test_recovery_preserves_a_write_after_comparing_the_live_file(tmp_path): + repo = _repository(tmp_path) + target = repo / "other.py" + marker = tmp_path / "compared" + patch = ( + "diff --git a/other.py b/other.py\n--- a/other.py\n+++ b/other.py\n" + "@@ -1 +1 @@\n-VALUE = 1\n+VALUE = 42\n" + ) + index = (repo / ".git/index").read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + + async def after_comparison(command, timeout=120.0, **kwargs): + if command.startswith("failed=0;"): + command = ( + 'cmp() { command cmp "$@" || return; ' + f'if [ "$3" = {shlex.quote(str(target))} ] && [ ! -e {shlex.quote(str(marker))} ]; then ' + f"printf 'VALUE = 100\\n' > {shlex.quote(str(target))} && touch {shlex.quote(str(marker))}; " + "fi; }; " + command + ) + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command and ".candidate-adopt-" in command: + assert result.returncode == 0, result.stderr + raise OSError("publication receipt unavailable") + return result + + environment.exec_cmd = after_comparison + try: + with pytest.raises((OSError, CandidateAdoptionRecoveryError)) as failed: + await _apply_increment(environment, str(repo), patch) + assert marker.is_file() + assert target.read_text() == "VALUE = 100\n" + assert isinstance(failed.value, CandidateAdoptionRecoveryError) + assert "concurrent change at " + str(target) in str(failed.value) + backup, = (repo / ".git").glob("candidate-recovery-*") + record, = backup.glob("*.path") + assert record.with_suffix("").read_text() == "VALUE = 1\n" + assert (repo / ".git/index").read_bytes() == index + finally: + await environment.cleanup() + + +@pytest.mark.parametrize("linked", [False, True]) +async def test_recovery_compares_parked_bytes_before_pruning_replay_files(tmp_path, linked): + source, candidate = _partial_publication(tmp_path) + repo = source + if linked: + repo = tmp_path / "linked" + _git(source, "worktree", "add", "--detach", str(repo)) + (repo / "settings").mkdir() + (repo / "settings/notes.txt").write_text("existing settings\n") + target = repo / "record.txt" + original = target.read_bytes() + index = Path(_git(repo, "rev-parse", "--absolute-git-dir").removesuffix("\n")) / "index" + index_before = index.read_bytes() + environment = LocalEnvironment(str(repo)) + execute = environment.exec_cmd + writer = None + appended = b"# independent write\n" + published = None + + async def observe(command, timeout=120.0, **kwargs): + nonlocal writer, published + if command.startswith("recovery="): + assert writer is not None + assert target.read_bytes() == original + writer.write(appended) + writer.flush() + os.fsync(writer.fileno()) + writer.close() + result = await execute(command, timeout, **kwargs) + if " apply --binary " in command and ".candidate-adopt-" in command: + assert result.returncode == 128, result.stderr + published = target.read_bytes() + assert published != original + writer = target.open("ab") + return result + + environment.exec_cmd = observe + + async def flow(ctx, _): + await ctx.adopt_candidate(candidate) + + try: + result = await OpenCollab(repo, environment=environment, model="offline").workflow(flow, trace=False) + assert isinstance(result.error, CandidateAdoptionRecoveryError) + assert f"concurrent change at {target}" in result.reason + backup, = (source / ".git").glob("candidate-recovery-*") + assert str(backup) in result.reason + record, = (record for record in backup.glob("observed-*/path") if record.read_text() == str(target)) + assert record.with_name("entry").read_bytes() == published + appended + assert target.read_bytes() == original + assert index.read_bytes() == index_before + finally: + if writer is not None: + writer.close() + await environment.cleanup() + if linked: + _git(source, "worktree", "remove", "--force", str(repo)) + assert not index.exists() + assert record.with_name("entry").read_bytes() == published + appended + + +@pytest.mark.parametrize("path", [".", ".gi\u200ct/config"]) +async def test_git_rejects_metadata_aliases_before_candidate_backup(tmp_path, path): + repo = _repository(tmp_path) + _git(repo, "config", "core.protectHFS", "true") + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + patch = (f"diff --git a/{path} b/{path}\nnew file mode 100644\n" + f"--- /dev/null\n+++ b/{path}\n@@ -0,0 +1 @@\n+unsafe\n") + environment = LocalEnvironment(str(repo)) + try: + with pytest.raises(RuntimeError, match="invalid path"): + await EnvCandidateWorkspace(environment).adopt(patch) + assert _contents(repo) == before + assert (repo / ".git/index").read_bytes() == index + assert not list((repo / ".git").glob("candidate-recovery-*")) + assert not environment.revoked + finally: + await environment.cleanup() + + +class _PublishingFailureEnvironment(LocalEnvironment): + def __init__(self, workspace, *, recovery_failure=False, publication_failure=True): + super().__init__(str(workspace)) + self.recovery_failure = recovery_failure + self.publication_failure = publication_failure + self.published = False + + async def exec_cmd(self, command, timeout=120.0, **kwargs): + if command.startswith("failed=0;") and self.recovery_failure: + raise OSError("recovery device unavailable") + result = await super().exec_cmd(command, timeout, **kwargs) + if (self.publication_failure and " apply --binary " in command + and ".candidate-adopt-" in command and result.returncode == 0): + self.published = True + raise OSError("publication receipt I/O failure") + return result + + +@pytest.mark.parametrize("physical", [False, True]) +@pytest.mark.parametrize("recover", [False, True]) +async def test_publication_error_restores_types_permissions_and_new_parents(tmp_path, physical, recover): + repo = _repository(tmp_path, physical=physical) + (repo / "link").symlink_to("draft.txt") + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + environment = _PublishingFailureEnvironment(repo, recovery_failure=not recover) + + async def edit(child, inputs): + folder = Path(child.workspace_root) + _flatten(folder) + (folder / "created/nested").mkdir(parents=True) + (folder / "created/nested/new.txt").write_text("new contents\n") + (folder / "link").unlink() + (folder / "link").write_text("regular replacement\n") + return "ready" + + async def flow(ctx, inputs): + candidate = await ctx.candidate_workflow(edit, {}, label="publication") + await ctx.adopt_candidate(candidate) + + try: + result = await OpenCollab(repo, environment=environment).workflow(flow, trace=False) + assert environment.published + assert not result.ok + if recover: + assert isinstance(result.error, OSError) + assert str(result.error) == "publication receipt I/O failure" + assert _contents(repo) == before + assert _run(repo) == "1\n" + assert not environment.revoked + else: + assert isinstance(result.error, CandidateAdoptionRecoveryError) + assert "publication receipt I/O failure" in result.reason + assert "recovery device unavailable" in result.reason + retained, = (repo / ".git").glob("candidate-recovery-*") + assert str(retained) in result.reason + originals = { + Path(path.read_text()).relative_to(repo).as_posix(): path.with_suffix("") + for path in retained.glob("*.path") + } + assert originals["loader.py"].read_bytes() == before["loader.py"][2] + assert originals["loader.py"].stat().st_mode & 0o777 == 0o750 + assert originals["settings"].is_dir() + assert originals["link"].is_symlink() + assert os.readlink(originals["link"]) == "draft.txt" + assert environment.revoked + with pytest.raises(RuntimeError, match="aborted"): + await environment.exec_cmd("printf unexpected > other.py") + assert (repo / ".git/index").read_bytes() == index + finally: + await environment.cleanup() + + +def _response(name, arguments, role, step): + return LLMResponse(tool_calls=[{ + "id": f"{role}-{step}", "type": "function", + "function": {"name": name, "arguments": json.dumps(arguments)}, + }], finish_reason="tool_calls", usage=Usage(1, 1)) + + +class _DuoModel: + def __init__(self, *, independent): + self.independent = independent + + async def complete(self, messages, **kwargs): + prompt = "\n".join(str(message.get("content") or "") for message in messages + if message.get("role") == "user") + role = "A" if "You are candidate A," in prompt else "B" if "You are candidate B," in prompt else "judge" + step = sum(message.get("role") == "tool" for message in messages) + if role in {"A", "B"}: + if step == 0: + if role == "A": + script = ( + 'from pathlib import Path; p=Path("settings"); (p/"defaults.py").unlink(); ' + 'p.rmdir(); p.write_text("42\\n"); ' + 'Path("loader.py").write_text("from pathlib import Path\\ndef value():\\n' + ' return int(Path(\\"settings\\").read_text())\\n")' + ) + else: + path = "other.py" if self.independent else "settings/defaults.py" + script = f"from pathlib import Path; Path({path!r}).write_text('VALUE = 42\\n')" + return _response("bash", {"command": shlex.join([sys.executable, "-B", "-c", script])}, role, step) + if step == 1: + return _response("bash", {"command": shlex.join([sys.executable, "-B", "app.py"])}, role, step) + return LLMResponse(content=f"Candidate {role} finished", usage=Usage(1, 1)) + return _response("structured_output", { + "winner": "A", "requirements_complete": True, + "requirements": [{ + "requirement": "Read a flat settings file and print 42", + "a_coverage": "covered", "b_coverage": "not_covered", + "a_evidence": ["loader.py reads the flat settings file containing 42"], + "b_evidence": ["settings/defaults.py still uses the original directory layout"], + }], + "rationale": "A replaces settings/defaults.py with settings and updates loader.py", + }, role, step) + + +@pytest.mark.parametrize("independent", [False, True]) +@pytest.mark.parametrize("recovery_failure", [False, True]) +async def test_duo_fallback_requires_recovered_source(tmp_path, independent, recovery_failure): + repo = _repository(tmp_path, cache=True) + before = _contents(repo) + index = (repo / ".git/index").read_bytes() + environment = _PublishingFailureEnvironment( + repo, recovery_failure=recovery_failure, publication_failure=False, + ) + backend = EnvCandidateWorkspace(environment) + attempted = [] + adopt = backend.adopt_run + + async def recorded(candidate, preserve_paths=()): + attempted.append(candidate.label) + await adopt(candidate, preserve_paths) + + backend.adopt_run = recorded + try: + result = await OpenCollab(repo, environment=environment, model="local-scripted").workflow( + "duo", {"goal": "Read a flat settings file and print 42", "allow_unisolated_shell": True, + "candidate_evidence_dir": str(tmp_path / "evidence")}, + llm=_DuoModel(independent=independent), candidate_workspace=backend, + trace=False, max_steps=5, budget=10000, limit_mode="explicit", timeout=30, + ) + if recovery_failure: + assert not result.ok + assert isinstance(result.error, CandidateAdoptionRecoveryError) + assert len(attempted) == 1 + assert "Directory not empty" in result.reason + assert "original files retained" in result.reason + assert list((repo / ".git").glob("candidate-recovery-*")) + else: + assert result.ok, result.reason + assert len(attempted) == 2 + assert result.output["status"] == "done" + assert result.output["adopted"] == "B" + target = "other.py" if independent else "settings/defaults.py" + after = _contents(repo) + assert after.pop(target)[2] == b"VALUE = 42\n" + before.pop(target) + assert after == before + assert _run(repo) == ("1\n" if independent else "42\n") + assert (repo / ".git/index").read_bytes() == index + finally: + await environment.cleanup() diff --git a/tests/workflows/test_candidate_docker_alias_mutations.py b/tests/workflows/test_candidate_docker_alias_mutations.py new file mode 100644 index 00000000..3912c39f --- /dev/null +++ b/tests/workflows/test_candidate_docker_alias_mutations.py @@ -0,0 +1,336 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters import _env_docker as docker_module +from opencollab.adapters._candidate_increment import _apply_increment +from opencollab.adapters._env_docker import DockerEnvironment +from opencollab.adapters.candidate_workspace import EnvCandidateWorkspace +from opencollab.adapters.tools.base import host_write_locks +from opencollab.adapters.tools.fs import FileWriteTool +from opencollab.application.tool_execution import ToolRuntime +from opencollab.tools import builtin_tools +from tests.support.docker_native_edits_support import LocalDockerTransport, RemoteDockerTransport, docker_environment +from tests.workflows.test_candidate_adoption_recovery import _flatten, _repository, _run +from tests.workflows.test_candidate_shared_mutations import _assert_flatten_recovery, _WriterModel +from tests.workflows.test_workflow_candidate_workspace import _git + + +class _RemoteCandidates(EnvCandidateWorkspace): + def __init__(self, environment, transport): + super().__init__(environment) + self.transport = transport + + async def _candidate_environment(self, path): + environment = await super()._candidate_environment(path) + self.transport.attach(environment) + return environment + + +def _remote_environment(repo): + transport = RemoteDockerTransport(repo) + environment = DockerEnvironment( + workspace=transport.remote, container_id="a" * 64, exec_workdir=transport.remote, + command_prefix="export OC_NATIVE_EDIT_PROBE=wrapped", + ) + transport.attach(environment) + return environment, transport + + +@pytest.mark.parametrize("tool", ["file_write", "apply_patch"]) +@pytest.mark.parametrize("path", ["alias/runtime.cache", "settings/runtime.cache", "unrelated.cache"]) +@pytest.mark.parametrize("cancel", [False, True]) +async def test_sdk_remote_alias_mutations_survive_directory_recovery(tmp_path, monkeypatch, tool, path, cancel): + repo = _repository(tmp_path) + (repo / ".gitignore").write_text("__pycache__/\n*.cache\n") + (repo / "alias").symlink_to("settings", target_is_directory=True) + _git(repo, "add", ".gitignore", "alias") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: retain workspace directory alias") + for cached in ("settings/runtime.cache", "unrelated.cache"): + (repo / cached).write_text("VALUE = 1\n") + loader = (repo / "loader.py").read_bytes() + index = (repo / ".git/index").read_bytes() + environment, transport = _remote_environment(repo) + entered, release = asyncio.Event(), asyncio.Event() + execute = environment.exec_cmd + publications = [] + + async def publication_barrier(cmd, timeout=120, **kwargs): + publication = " apply --binary " in cmd and ".candidate-adopt-" in cmd + if publication: + entered.set() + await release.wait() + result = await execute(cmd, timeout, **kwargs) + if publication: + publications.append(result.returncode) + return result + + monkeypatch.setattr(environment, "exec_cmd", publication_barrier) + model = _WriterModel(tool, path) + + async def edit(child, _): + _flatten(Path(child.workspace_root)) + return "prepared" + + async def flow(ctx, _): + candidate = await ctx.candidate_workflow(edit, {}, label="flatten-directory") + publication = asyncio.create_task(ctx.adopt_candidate(candidate)) + writer = None + try: + await asyncio.wait_for(entered.wait(), 15) + writer = asyncio.create_task(ctx.agent_run( + "Set the runtime cache value to 100", tools=list(builtin_tools(tool, headless=False)), + isolation="shared", budget=2000, system_prompt="Use the requested file tool", + )) + await asyncio.wait_for(model.requested.wait(), 10) + if path == "unrelated.cache": + assert (await asyncio.wait_for(asyncio.shield(writer), 10)).status == "completed" + else: + await asyncio.sleep(0.1) + assert not writer.done() + assert (repo / path).read_text() == "VALUE = 1\n" + if cancel: + publication.cancel() + await asyncio.sleep(0) + publication.cancel() + assert not publication.done() + release.set() + if cancel: + with pytest.raises(asyncio.CancelledError): + await publication + else: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await publication + assert (await asyncio.wait_for(writer, 10)).status == "completed" + assert (repo / path).read_text() == "VALUE = 100\n" + assert len(publications) == 1 and publications[0] != 0 + return "retained" + finally: + release.set() + await asyncio.gather(publication, *([writer] if writer is not None else []), return_exceptions=True) + + try: + result = await OpenCollab(repo, environment=environment, model="local-scripted").workflow( + flow, llm=model, candidate_workspace=_RemoteCandidates(environment, transport), trace=False, + timeout=60, budget=10000, concurrency=2, limit_mode="explicit", agent_profile="single2", + ) + assert result.ok, result.reason + assert result.output == "retained" + assert _run(repo) == "1\n" + assert (repo / ".git/index").read_bytes() == index + _assert_flatten_recovery(repo, loader) + assert not environment._temporary_files + assert not environment.revoked + finally: + release.set() + await environment.cleanup() + + +async def test_remote_alias_group_resolves_before_and_after_ordered_acquisition_and_releases_cancelled_waiter( + tmp_path, monkeypatch, +): + (tmp_path / "settings").mkdir() + (tmp_path / "settings/value.py").write_text("VALUE = 1\n") + (tmp_path / "alias").symlink_to("settings", target_is_directory=True) + environment, _ = _remote_environment(tmp_path) + resolve = environment._resolve_write_target + resolving_task = asyncio.current_task() + resolutions = [] + expected = [environment.workspace + suffix for suffix in ("/alias", "/settings", "/settings/value.py")] + + async def resolve_with_lock_check(target, **kwargs): + owned = [key.split("\0", 1)[1] for key, owner in docker_module._WRITE_LOCK_OWNERS.items() + if owner is resolving_task] + assert owned in ([], expected) + resolutions.append(bool(owned)) + return await resolve(target, **kwargs) + + monkeypatch.setattr(environment, "_resolve_write_target", resolve_with_lock_check) + waiting = None + try: + async with host_write_locks((environment.workspace + "/alias/value.py",), environment): + owned = [key.split("\0", 1)[1] for key, owner in docker_module._WRITE_LOCK_OWNERS.items() + if owner is resolving_task] + assert owned == expected + assert resolutions == sorted(resolutions) + assert set(resolutions) == {False, True} + monkeypatch.setattr(environment, "_resolve_write_target", resolve) + async with host_write_locks((environment.workspace + "/settings",), environment): + waiting = asyncio.create_task(FileWriteTool().execute_with_runtime({ + "path": "alias/value.py", "mode": "create", "content": "VALUE = 100\n", + }, ToolRuntime(environment, None, None))) + await asyncio.sleep(0.1) + assert not waiting.done() + alias_key = environment._container_id + "\0" + environment.workspace + "/alias" + assert alias_key in docker_module._WRITE_LOCK_OWNERS + waiting.cancel() + with pytest.raises(asyncio.CancelledError): + await waiting + assert alias_key not in docker_module._WRITE_LOCK_OWNERS + output = await asyncio.wait_for(FileWriteTool().execute_with_runtime({ + "path": "alias/value.py", "mode": "create", "content": "VALUE = 100\n", + }, ToolRuntime(environment, None, None)), 5) + assert "Created/wrote" in output + assert (tmp_path / "settings/value.py").read_text() == "VALUE = 100\n" + assert not docker_module._WRITE_LOCK_OWNERS + finally: + if waiting is not None: + await asyncio.gather(waiting, return_exceptions=True) + await environment.cleanup() + + +async def test_docker_edits_serialize_on_the_target_after_candidate_retargets_directory_alias( + tmp_path, monkeypatch, +): + repo = _repository(tmp_path) + for directory in ("old", "new"): + (repo / directory).mkdir() + (repo / directory / "module.py").write_text("alpha = 1\nbeta = 1\n") + alias = repo / "alias" + alias.symlink_to("old", target_is_directory=True) + _git(repo, "add", "old", "new", "alias") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: keep two module directories") + alias.unlink() + alias.symlink_to("new", target_is_directory=True) + patch = _git(repo, "diff", "--binary", "--", "alias") + _git(repo, "restore", "--", "alias") + + environment = docker_environment(repo, LocalDockerTransport(repo)) + runtime = ToolRuntime(environment, None, None) + entered, release = asyncio.Event(), asyncio.Event() + execute, read = environment.exec_cmd, environment.read_file + + async def pause_publication(cmd, *args, **kwargs): + if " apply --binary " in cmd and ".candidate-adopt-" in cmd: + entered.set() + await release.wait() + return await execute(cmd, *args, **kwargs) + + async def read_under_target_lock(path): + target_key = environment.container_reference + "\0" + str(repo / "new/module.py") + assert docker_module._WRITE_LOCK_OWNERS.get(target_key) is asyncio.current_task() + return await read(path) + + async def edit(path, name): + return await FileWriteTool().execute_with_runtime({ + "path": path, "mode": "str_replace", "old_str": f"{name} = 1", "new_str": f"{name} = 2", + }, runtime) + + async def wait_for_alias_editor(): + alias_key = environment.container_reference + "\0" + str(alias) + while getattr(docker_module._WRITE_LOCKS.get(alias_key), "users", 0) < 2: + await asyncio.sleep(0.005) + + monkeypatch.setattr(environment, "exec_cmd", pause_publication) + monkeypatch.setattr(environment, "read_file", read_under_target_lock) + publication = asyncio.create_task(_apply_increment(environment, str(repo), patch)) + writers = [] + try: + await asyncio.wait_for(entered.wait(), 5) + writers.append(asyncio.create_task(edit("alias/module.py", "alpha"))) + await asyncio.wait_for(wait_for_alias_editor(), 5) + assert not writers[0].done() + release.set() + await asyncio.wait_for(publication, 5) + writers.append(asyncio.create_task(edit("new/module.py", "beta"))) + receipts = await asyncio.wait_for(asyncio.gather(*writers), 5) + assert all(receipt.startswith("Replaced") for receipt in receipts), receipts + assert (repo / "old/module.py").read_text() == "alpha = 1\nbeta = 1\n" + assert (repo / "new/module.py").read_text() == "alpha = 2\nbeta = 2\n" + finally: + release.set() + await asyncio.gather(publication, *writers, return_exceptions=True) + await environment.cleanup() + + +@pytest.mark.parametrize("new_file", ["default.toml", "nested/default.toml"]) +@pytest.mark.parametrize("symlink", [False, True]) +async def test_sdk_docker_candidate_can_replace_a_file_with_a_directory(tmp_path, new_file, symlink): + repo = _repository(tmp_path) + if symlink: + (repo / "config-source").write_text("enabled = true\n") + (repo / "config").symlink_to("config-source") + _git(repo, "add", "config", "config-source") + else: + (repo / "config").write_text("enabled = true\n") + _git(repo, "add", "config") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: keep a flat config") + index = (repo / ".git/index").read_bytes() + environment, transport = _remote_environment(repo) + + async def edit(child, _): + config = Path(child.workspace_root) / "config" + config.unlink() + target = config / new_file + target.parent.mkdir(parents=True) + target.write_text("enabled = false\n") + + async def flow(ctx, _): + candidate = await ctx.candidate_workflow(edit, {}, label="nested-config") + await ctx.adopt_candidate(candidate) + return "adopted" + + try: + result = await OpenCollab(repo, environment=environment).workflow( + flow, candidate_workspace=_RemoteCandidates(environment, transport), trace=False, timeout=30, + ) + assert result.ok, result.reason + assert result.output == "adopted" + assert (repo / "config" / new_file).read_text() == "enabled = false\n" + if symlink: + assert (repo / "config-source").read_text() == "enabled = true\n" + assert (repo / ".git/index").read_bytes() == index + finally: + await environment.cleanup() + + +async def test_sdk_docker_candidate_uses_the_same_git_root_for_capture_and_adoption(tmp_path): + repo = _repository(tmp_path) + package = repo / "package" + package.mkdir() + (package / "module.py").write_text("VALUE = 1\n") + _git(repo, "add", "package") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare a subdirectory workflow") + index = (repo / ".git/index").read_bytes() + transport = LocalDockerTransport(repo, pair_reads=False) + environment = docker_environment(package, transport) + + async def edit(child, _): + (Path(child.workspace_root).parent / "other.py").write_text("VALUE = 2\n") + + async def flow(ctx, _): + candidate = await ctx.candidate_workflow(edit, {}, label="update-neighbor") + assert "b/other.py" in candidate.effective_diff + await ctx.adopt_candidate(candidate) + return "adopted" + + try: + result = await OpenCollab(package, environment=environment).workflow( + flow, candidate_workspace=_RemoteCandidates(environment, transport), trace=False, timeout=30, + ) + assert result.ok, result.reason + assert (repo / "other.py").read_text() == "VALUE = 2\n" + assert (repo / ".git/index").read_bytes() == index + assert environment._resolved_workspace in {None, str(package)} + finally: + await environment.cleanup() + + +async def test_docker_group_reuses_sibling_parent_resolution_in_each_pass(tmp_path): + source = tmp_path / "src" + source.mkdir() + transport = LocalDockerTransport(tmp_path, pair_reads=False) + environment = docker_environment(tmp_path, transport) + try: + paths = tuple(str(source / f"module_{index}.py") for index in range(48)) + async with environment.file_write_locks(paths): + resolutions = [row for row in transport.calls if "opencollab-file-parent" in row["command"]] + assert len(resolutions) == 4 + await environment.write_file(paths[0], "VALUE = 1\n") + assert (source / "module_0.py").read_text() == "VALUE = 1\n" + finally: + await environment.cleanup() diff --git a/tests/workflows/test_candidate_mutation_backends.py b/tests/workflows/test_candidate_mutation_backends.py new file mode 100644 index 00000000..099cbd35 --- /dev/null +++ b/tests/workflows/test_candidate_mutation_backends.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from opencollab.adapters._candidate_increment import _apply_increment +from opencollab.adapters._env_local import LocalEnvironment +from opencollab.adapters.session_environments import SessionEnvironmentPool +from opencollab.adapters.tools.base import host_write_locks +from opencollab.adapters.tools.fs import FileWriteTool +from opencollab.application.tool_execution import ToolRuntime +from opencollab.bootstrap.authored_workflow_bridge import AuthoredWorkflowBridge, BridgeClient, BridgeEnvironment +from tests.support.docker_native_edits_support import LocalDockerTransport, docker_environment +from tests.workflows.test_candidate_adoption_recovery import _detached_files, _flatten, _repository, _run +from tests.workflows.test_workflow_candidate_workspace import _git + + +@pytest.mark.parametrize("backend", ["local", "docker"]) +@pytest.mark.parametrize("bridge_entry", [False, True]) +@pytest.mark.parametrize("failure", [False, True]) +async def test_candidate_and_bridge_share_native_mutation_ownership( + tmp_path, monkeypatch, backend, bridge_entry, failure, +): + monkeypatch.chdir(tmp_path) + repo = _repository(tmp_path) + _git(repo, "add", "-A") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare native publication") + if failure: + _flatten(repo) + else: + (repo / "settings/defaults.py").write_text("VALUE = 42\n") + _git(repo, "add", "-A") + patch = _git(repo, "diff", "--cached", "--binary") + _git(repo, "reset", "--hard", "-q") + if failure: + assert _run(repo, cache=True) == "1\n" + index = (repo / ".git/index").read_bytes() + native = (LocalEnvironment(str(repo)) if backend == "local" else + docker_environment(repo, LocalDockerTransport(repo, pair_reads=False))) + entered, release, stop = asyncio.Event(), asyncio.Event(), asyncio.Event() + execute = native.exec_cmd + + async def publication_barrier(cmd, timeout=120, **kwargs): + command = cmd + if " apply --binary " in command and ".candidate-adopt-" in command: + entered.set() + await release.wait() + return await execute(command, timeout, **kwargs) + + monkeypatch.setattr(native, "exec_cmd", publication_barrier) + mailbox_root = tmp_path / "mailbox" + artifacts = mailbox_root / "artifacts" + artifacts.mkdir(parents=True) + mailbox = LocalEnvironment(str(mailbox_root)) + bridge = AuthoredWorkflowBridge( + mailbox, str(artifacts), + SessionEnvironmentPool(str(mailbox_root), isolation="shared", base_environment=mailbox), + {}, ToolRuntime(mailbox, None, None), allow_unisolated_shell=True, + ) + bridge._environments[1] = native + proxy = BridgeEnvironment(BridgeClient(artifacts, bridge.options()), { + "handle": 1, "workspace": native.workspace, "process_isolated": False, + }) + + async def serve(): + while not stop.is_set(): + await bridge.poll() + await asyncio.sleep(0.005) + + server = asyncio.create_task(serve()) + publication = asyncio.create_task(_apply_increment(proxy if bridge_entry else native, str(repo), patch)) + writer = None + try: + entered_task = asyncio.create_task(entered.wait()) + try: + await asyncio.wait((entered_task, publication), timeout=10, return_when=asyncio.FIRST_COMPLETED) + if publication.done(): + await publication + await asyncio.wait_for(entered_task, 1) + finally: + entered_task.cancel() + await asyncio.gather(entered_task, return_exceptions=True) + writer = asyncio.create_task(FileWriteTool().execute_with_runtime({ + "path": "settings/defaults.py", "mode": "create", "content": "VALUE = 100\n", + }, ToolRuntime(native if bridge_entry else proxy, None, None))) + await asyncio.sleep(0.05) + assert not writer.done() + assert (repo / "settings/defaults.py").read_text() == "VALUE = 1\n" + release.set() + if failure: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await asyncio.wait_for(publication, 10) + else: + await asyncio.wait_for(publication, 10) + assert "Created/wrote" in await asyncio.wait_for(writer, 10) + assert _run(repo) == "100\n" + assert (repo / ".git/index").read_bytes() == index + assert not native.revoked + assert not bridge._edits + finally: + release.set() + await asyncio.gather(publication, *([writer] if writer is not None else []), return_exceptions=True) + stop.set() + await server + await bridge.aclose() + await native.cleanup() + await mailbox.cleanup() + + +async def test_cancelled_publication_recovers_before_releasing_shared_writer(tmp_path, monkeypatch): + repo = _repository(tmp_path) + target = repo / "settings/defaults.py" + index = (repo / ".git/index").read_bytes() + target.write_text("VALUE = 42\n") + patch = _git(repo, "diff", "--binary", "--", "settings/defaults.py") + target.write_text("VALUE = 1\n") + environment = LocalEnvironment(str(repo)) + entered = asyncio.Event() + execute = environment.exec_cmd + + async def pause_publication(command, timeout=120, **kwargs): + if " apply --binary " in command and ".candidate-adopt-" in command: + # A real completed Git publication is cancelled before its caller + # receives the result, exercising recovery of changed bytes. + result = await execute(command, timeout, **kwargs) + assert result.returncode == 0 + entered.set() + await asyncio.Event().wait() + return await execute(command, timeout, **kwargs) + + monkeypatch.setattr(environment, "exec_cmd", pause_publication) + publication = asyncio.create_task(_apply_increment(environment, str(repo), patch)) + writer = None + try: + await asyncio.wait_for(entered.wait(), 10) + writer = asyncio.create_task(FileWriteTool().execute_with_runtime({ + "path": str(target), "mode": "create", "content": "VALUE = 100\n", + }, ToolRuntime(environment, None, None))) + await asyncio.sleep(0.05) + assert not writer.done() + publication.cancel() + with pytest.raises(asyncio.CancelledError): + await publication + assert "Created/wrote" in await asyncio.wait_for(writer, 10) + assert _run(repo) == "100\n" + assert (repo / ".git/index").read_bytes() == index + assert _detached_files(repo / ".git") == ( + {target: b"VALUE = 1\n"}, {target: b"VALUE = 42\n"}, + ) + finally: + if not publication.done(): + publication.cancel() + await asyncio.gather(publication, *([writer] if writer is not None else []), return_exceptions=True) + await environment.cleanup() + + +async def test_local_mutation_lock_normalizes_workspace_parent_aliases(tmp_path): + workspace = tmp_path / "workspace" + target = workspace / "nested/value.py" + target.parent.mkdir(parents=True) + target.write_text("VALUE = 1\n") + alias = tmp_path / "alias" + alias.symlink_to(workspace, target_is_directory=True) + environment = LocalEnvironment(str(workspace)) + writer = None + try: + async with host_write_locks((str(alias / "nested/value.py"),), environment): + writer = asyncio.create_task(FileWriteTool().execute_with_runtime({ + "path": str(target), "mode": "create", "content": "VALUE = 100\n", + }, ToolRuntime(environment, None, None))) + await asyncio.sleep(0.05) + assert not writer.done() + assert target.read_text() == "VALUE = 1\n" + assert "Created/wrote" in await asyncio.wait_for(writer, 2) + assert target.read_text() == "VALUE = 100\n" + finally: + if writer is not None: + await asyncio.gather(writer, return_exceptions=True) + await environment.cleanup() diff --git a/tests/workflows/test_candidate_shared_mutations.py b/tests/workflows/test_candidate_shared_mutations.py new file mode 100644 index 00000000..a21ebb95 --- /dev/null +++ b/tests/workflows/test_candidate_shared_mutations.py @@ -0,0 +1,352 @@ +from __future__ import annotations + +import asyncio +import json +import shlex +import sys +from pathlib import Path + +import pytest + +from opencollab import OpenCollab +from opencollab.adapters._env_local import LocalEnvironment +from opencollab.adapters.llm.types import LLMResponse, Usage +from opencollab.application.workflow_candidates import CandidateAdoptionRecoveryError +from opencollab.tools import builtin_tools +from tests.workflows.test_candidate_adoption_recovery import ( + _detached_files, + _flatten, + _partial_publication, + _repository, + _run, +) +from tests.workflows.test_workflow_candidate_workspace import _git + + +class _PublicationBarrier(LocalEnvironment): + def __init__(self, workspace, *, after_write=False): + super().__init__(str(workspace)) + self.after_write = after_write + self.entered = asyncio.Event() + self.release = asyncio.Event() + self.publications = [] + + async def exec_cmd(self, command, timeout=120, **kwargs): + publication = " apply --binary " in command and ".candidate-adopt-" in command + if publication and not self.after_write: + self.entered.set() + await self.release.wait() + result = await super().exec_cmd(command, timeout, **kwargs) + if publication: + self.publications.append(result.returncode) + if self.after_write: + self.entered.set() + await self.release.wait() + return result + + +class _WriterModel: + def __init__(self, tool, path, *, command=None): + self.tool, self.path, self.calls = tool, path, 0 + self.command = command + self.requested = asyncio.Event() + + async def complete(self, messages, tools=None, **kwargs): + self.calls += 1 + if self.calls == 1: + self.requested.set() + arguments = {"path": self.path} + if self.tool == "bash": + arguments = {"command": self.command or f"printf '%s\\n' 'shared replacement' > {self.path}"} + elif self.tool == "file_write": + arguments.update(mode="create", content="VALUE = 100\n") + else: + arguments.update(mode="line_replace", start_line=1, end_line=1, new_str="VALUE = 100") + return LLMResponse(tool_calls=[{ + "id": "shared-edit", "type": "function", "function": { + "name": self.tool, "arguments": json.dumps(arguments), + }, + }], usage=Usage(10, 5), finish_reason="tool_calls") + evidence = "Exit code: 0" if self.tool == "bash" else "100" + assert any(evidence in str(message) for message in messages) + return LLMResponse(content="Shared edit complete", usage=Usage(10, 5), finish_reason="stop") + + +def _assert_flatten_recovery(repo: Path, loader: bytes) -> None: + assert (repo / "loader.py").read_bytes() == loader + assert (repo / "draft.txt").read_bytes() == b"untracked user draft\n" + assert _detached_files(repo / ".git") == ( + { + repo / "loader.py": loader, + repo / "draft.txt": b"untracked user draft\n", + repo / "settings/defaults.py": b"VALUE = 1\n", + }, + { + repo / "loader.py": ( + b'from pathlib import Path\ndef value():\n return int(Path("settings").read_text())\n' + ), + repo / "draft.txt": b"candidate draft\n", + }, + ) + + +@pytest.mark.parametrize("tool", ["file_write", "apply_patch"]) +@pytest.mark.parametrize("relation,failure", [ + ("target", False), ("target", True), ("descendant", True), + ("unrelated", False), ("unrelated", True), +]) +async def test_sdk_shared_agent_owns_mutations_through_candidate_publication( + tmp_path, tool, relation, failure, +): + repo = _repository(tmp_path, cache=relation == "descendant") + if failure and relation != "descendant": + (repo / "blocked").mkdir() + (repo / "blocked/entry.py").write_text("VALUE = 1\n") + _git(repo, "add", "blocked/entry.py") + _git(repo, "-c", "commit.gpgsign=false", "commit", "-qm", "test: prepare directory replacement") + (repo / "blocked/__pycache__").mkdir() + (repo / "blocked/__pycache__/retained.pyc").write_bytes(b"retained ignored bytes") + environment = _PublicationBarrier(repo) + path = "other.py" if relation == "unrelated" else "settings/defaults.py" + model = _WriterModel(tool, path) + loader = (repo / "loader.py").read_bytes() + index = (repo / ".git/index").read_bytes() + + async def edit(child, _): + folder = Path(child.workspace_root) + if relation == "descendant": + _flatten(folder) + else: + (folder / "settings/defaults.py").write_text("VALUE = 42\n") + if failure: + (folder / "blocked/entry.py").unlink() + (folder / "blocked").rmdir() + (folder / "blocked").write_text("replacement\n") + return "prepared" + + async def flow(ctx, _): + candidate = await ctx.candidate_workflow(edit, {}, label="forty-two") + publication = asyncio.create_task(ctx.adopt_candidate(candidate)) + writer = None + try: + await asyncio.wait_for(environment.entered.wait(), 10) + writer = asyncio.create_task(ctx.agent_run( + "Set the shared value to 100", tools=list(builtin_tools(tool, headless=False)), + isolation="shared", budget=2000, system_prompt="Use the requested file tool", + )) + await asyncio.wait_for(model.requested.wait(), 10) + if relation == "unrelated": + result = await asyncio.wait_for(asyncio.shield(writer), 10) + assert result.status == "completed" + assert (repo / path).read_text() == "VALUE = 100\n" + else: + await asyncio.sleep(0.05) + assert not writer.done() + assert (repo / path).read_text() == "VALUE = 1\n" + environment.release.set() + if failure: + with pytest.raises(RuntimeError, match="Directory not empty|Is a directory"): + await publication + else: + await publication + result = await writer + assert result.status == "completed" + assert (repo / path).read_text() == "VALUE = 100\n" + assert len(environment.publications) == 1 + assert (environment.publications[0] != 0) is failure + return "retained" + finally: + environment.release.set() + await asyncio.gather(publication, *([writer] if writer is not None else []), return_exceptions=True) + + try: + result = await OpenCollab(repo, environment=environment, model="local-scripted").workflow( + flow, llm=model, trace=False, timeout=40, budget=10000, concurrency=2, + limit_mode="explicit", agent_profile="single2", + ) + assert result.ok, result.reason + assert result.output == "retained" + assert _run(repo) == ("100\n" if relation != "unrelated" else "1\n" if failure else "42\n") + assert (repo / ".git/index").read_bytes() == index + if not failure: + assert not list((repo / ".git").glob("candidate-recovery-*")) + elif relation == "descendant": + _assert_flatten_recovery(repo, loader) + else: + blocked = repo / "blocked/entry.py" + assert blocked.read_bytes() == b"VALUE = 1\n" + assert _detached_files(repo / ".git") == ( + {blocked: b"VALUE = 1\n", repo / "settings/defaults.py": b"VALUE = 1\n"}, + {}, + ) + assert not environment.revoked + finally: + environment.release.set() + await environment.cleanup() + + +@pytest.mark.parametrize("after_write,path", [ + (True, "loader.py"), (True, "settings/ignored.cache"), (True, "settings/notes.txt"), (True, "separate.txt"), + (False, "settings/notes.txt"), +]) +async def test_sdk_shared_bash_preserves_writes_around_git_publication_failure(tmp_path, after_write, path): + repo = _repository(tmp_path) + if after_write: + with (repo / ".gitignore").open("a") as ignore: + ignore.write("settings/ignored.cache\n") + (repo / "settings/ignored.cache").write_text("runtime cache\n") + loader = (repo / "loader.py").read_bytes() + index = (repo / ".git/index").read_bytes() + environment = _PublicationBarrier(repo, after_write=after_write) + overlap = path == "loader.py" + + async def edit(child, _): + _flatten(Path(child.workspace_root)) + + async def flow(ctx, _): + candidate = await ctx.candidate_workflow(edit, {}, label="flatten") + publication = asyncio.create_task(ctx.adopt_candidate(candidate)) + try: + await asyncio.wait_for(environment.entered.wait(), 10) + assert ((repo / "loader.py").read_bytes() != loader) is after_write + writer = await asyncio.wait_for(ctx.agent_run( + "Save this file in the shared workspace", tools=list(builtin_tools("bash", headless=False)), + isolation="shared", budget=2000, system_prompt="Use the requested Bash operation", + ), 10) + assert writer.status == "completed" + assert (repo / path).read_text() == "shared replacement\n" + environment.release.set() + error = CandidateAdoptionRecoveryError if overlap else RuntimeError + detail = "concurrent change at .*/loader.py" if overlap else "Directory not empty|Is a directory" + with pytest.raises(error, match=detail): + await publication + finally: + environment.release.set() + await asyncio.gather(publication, return_exceptions=True) + + try: + result = await OpenCollab(repo, environment=environment, model="local-scripted").workflow( + flow, llm=_WriterModel("bash", path), trace=False, timeout=30, budget=5000, + concurrency=2, limit_mode="explicit", agent_profile="single2", + ) + assert result.ok, result.reason + assert environment.publications == [128] + assert (repo / path).read_text() == "shared replacement\n" + assert (repo / "settings/defaults.py").read_text() == "VALUE = 1\n" + assert (repo / "draft.txt").read_text() == "untracked user draft\n" + assert (repo / ".git/index").read_bytes() == index + if overlap: + backup, = (repo / ".git").glob("candidate-recovery-*") + for record in backup.glob("*.path"): + if Path(record.read_text()).name == "loader.py": + assert record.with_suffix("").read_bytes() == loader + break + else: + pytest.fail("recovery did not retain the original loader") + else: + _assert_flatten_recovery(repo, loader) + finally: + await environment.cleanup() + + +class _ParkingBarrier(_PublicationBarrier): + def __init__(self, workspace): + super().__init__(workspace, after_write=True) + self.parking = asyncio.Event() + self.allow_parking = asyncio.Event() + self.parked = asyncio.Event() + self.return_recovery = asyncio.Event() + + async def exec_cmd(self, command, timeout=120, **kwargs): + recovery = command.startswith("failed=0;") + if recovery: + self.parking.set() + await self.allow_parking.wait() + result = await super().exec_cmd(command, timeout, **kwargs) + if recovery: + self.parked.set() + await self.return_recovery.wait() + return result + + +@pytest.mark.parametrize("finished_inside_recovery", [False, True]) +async def test_sdk_shared_bash_retains_writes_through_an_open_file(tmp_path, finished_inside_recovery): + repo, candidate = _partial_publication(tmp_path) + target = repo / "record.txt" + original = target.read_bytes() + index = (repo / ".git/index").read_bytes() + environment = _ParkingBarrier(repo) + ready = repo / "writer.ready" + release = repo / "writer.release" + appended = b"# shared shell bytes\n" + command = shlex.join((sys.executable, "-c", f""" +import os +import time +from pathlib import Path + +with Path({target.name!r}).open("ab") as stream: + Path({ready.name!r}).touch() + deadline = time.monotonic() + 10 + while not Path({release.name!r}).exists(): + if time.monotonic() >= deadline: + raise TimeoutError("waiting for recovery") + time.sleep(0.01) + stream.write({appended!r}) + stream.flush() + os.fsync(stream.fileno()) +""")) + + async def wait_for_writer(): + while not ready.exists(): + await asyncio.sleep(0.01) + + async def flow(ctx, _): + publication = asyncio.create_task(ctx.adopt_candidate(candidate)) + writer = None + try: + await asyncio.wait_for(environment.entered.wait(), 10) + published = target.read_bytes() + assert published != original + if finished_inside_recovery: + environment.release.set() + await asyncio.wait_for(environment.parking.wait(), 10) + writer = asyncio.create_task(ctx.agent_run( + "Save the shared Bash result", tools=list(builtin_tools("bash", headless=False)), + isolation="shared", budget=2000, system_prompt="Run the shell command", + )) + await asyncio.wait_for(wait_for_writer(), 10) + environment.release.set() + environment.allow_parking.set() + await asyncio.wait_for(environment.parked.wait(), 10) + if finished_inside_recovery: + release.touch() + assert (await asyncio.wait_for(writer, 10)).status == "completed" + environment.return_recovery.set() + with pytest.raises(CandidateAdoptionRecoveryError, match="concurrent shell command.*retained"): + await asyncio.wait_for(asyncio.shield(publication), 10) + release.touch() + assert (await asyncio.wait_for(writer, 10)).status == "completed" + return published + finally: + environment.release.set() + environment.allow_parking.set() + environment.return_recovery.set() + release.touch() + await asyncio.gather(publication, *([writer] if writer else []), return_exceptions=True) + + try: + result = await OpenCollab(repo, environment=environment, model="local-scripted").workflow( + flow, llm=_WriterModel("bash", target.name, command=command), trace=False, timeout=40, + budget=5000, concurrency=2, limit_mode="explicit", agent_profile="single2", + ) + assert result.ok, result.reason + assert environment.publications == [128] + assert target.read_bytes() == original + assert (repo / "settings/notes.txt").read_text() == "existing settings\n" + assert (repo / ".git/index").read_bytes() == index + retained, = (repo / ".git").glob("candidate-recovery-*") + record, = (record for record in retained.glob("observed-*/path") if record.read_text() == str(target)) + assert record.with_name("entry").read_bytes() == result.output + appended + assert not environment.revoked + finally: + await environment.cleanup() diff --git a/tests/workflows/test_candidate_symlink_delivery.py b/tests/workflows/test_candidate_symlink_delivery.py index f7de2725..d60ec425 100644 --- a/tests/workflows/test_candidate_symlink_delivery.py +++ b/tests/workflows/test_candidate_symlink_delivery.py @@ -181,7 +181,7 @@ async def test_link_probe_without_bash_keeps_executable_physical_delivery(tmp_pa binaries = tmp_path / "commands" binaries.mkdir() for name in ("git", "mktemp", "rm", "cat", "grep", "mkdir", "sh", "sed", "awk", "basename", - "dirname", "uname", "expr", "tr", "wc", "cut", "pwd", "sort", "cp"): + "dirname", "uname", "expr", "tr", "wc", "cut", "pwd", "sort", "cp", "stat", "mv", "chmod"): command = shutil.which(name) assert command is not None (binaries / name).symlink_to(command) diff --git a/tests/workflows/test_scheduler_cross_agent_budget_cleanup.py b/tests/workflows/test_scheduler_cross_agent_budget_cleanup.py new file mode 100644 index 00000000..3f090161 --- /dev/null +++ b/tests/workflows/test_scheduler_cross_agent_budget_cleanup.py @@ -0,0 +1,307 @@ +"""Accepted messages share progress with another agent's deferred budget owner.""" + +from __future__ import annotations + +import asyncio +import json +from types import SimpleNamespace + +import pytest + +from opencollab.adapters.llm.client import LLMClient +from opencollab.adapters.worktree_pool import WorktreePool +from opencollab.application.event_bus import EventBus +from opencollab.application.scheduler import Scheduler, SchedulerTurnError +from opencollab.bootstrap import build_session +from opencollab.domain.agent import Agent +from opencollab.domain.scheduler import SessionControlBlock +from opencollab.domain.session import SessionPhase, SessionState +from tests.support.provider_sdk_http import completion_http_response, install_sdk_transport + + +class _Factory: + def __init__(self, client, timeout): + self.client = client + self.timeout = timeout + + def build_spawn_session(self, *, role, budget, aid, task, **kwargs): + return build_session( + agent=Agent(name=role, system_prompt=role), llm=self.client, + env=kwargs["env"], aid=aid, max_budget_tokens=budget, llm_timeout=self.timeout, + seed_user_messages=[{"role": "user", "content": task}], + ) + + +@pytest.mark.parametrize("owner_aid", [0, 2]) +@pytest.mark.parametrize("ending", ["success", "cancelled", "late_success", "late_error"]) +async def test_accepted_inbox_waits_for_another_agents_budget_owner( + monkeypatch, tmp_path, owner_aid, ending, +): + started = asyncio.Event() + cancelled = asyncio.Event() + release = asyncio.Event() + waiting_for_budget_owner = asyncio.Event() + requests = [] + + async def handler(request): + body = json.loads(request.content) + requests.append(body) + latest_user = next(m["content"] for m in reversed(body["messages"]) if m["role"] == "user") + if latest_user == "owner operation": + started.set() + try: + await release.wait() + except asyncio.CancelledError: + cancelled.set() + if ending == "cancelled": + raise + await release.wait() + if ending == "late_error": + raise RuntimeError("late provider failure") + return completion_http_response(request, text="owner answer") + if latest_user == "lead warmup": + return completion_http_response(request, output_tokens=19_998, text="lead ready") + if latest_user == "owner warmup": + return completion_http_response(request, output_tokens=9_998, text="owner ready") + return completion_http_response(request, text="recipient processed") + + install_sdk_transport(monkeypatch, "openai", handler) + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", "/dev/null") + async with LLMClient( + model="gpt-4o", provider="openai", api_key="controlled-test", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + ) as client: + total = 400_000 if owner_aid == 0 else 40_000 + timeout = 1.0 if ending == "success" else 0.05 + scheduler = Scheduler( + session_factory=_Factory(client, 1.0), + worktree_pool=WorktreePool(str(tmp_path), use_worktrees=False), + event_sink=EventBus(), max_budget_tokens=total, + ) + lead = build_session( + agent=Agent(name="lead", system_prompt="lead"), llm=client, + max_budget_tokens=total, llm_timeout=timeout, + ) + scheduler.register_lead(lead) + if owner_aid: + assert await scheduler.run("lead warmup") == "lead ready" + recipient = await scheduler.spawn(0, "recipient", "recipient warmup") + await scheduler._tasks[recipient] + recipient_session = scheduler._sessions[recipient] + assert recipient_session.phase is SessionPhase.DONE, recipient_session.state.terminal_reason + if owner_aid: + assert await scheduler.spawn(0, "owner", "owner warmup") == owner_aid + await scheduler._tasks[owner_aid] + owner = scheduler._sessions[owner_aid] + owner.runner._per_call_timeout = timeout + original_active = scheduler._active_scheduler_tasks + + def observe_progress(): + pending = original_active() + if ( + scheduler._pending_provider_budget_leases + and not any(not task.done() for task in scheduler._tasks.values()) + and any(task in pending for task in owner.runner.pending_cleanup_tasks) + ): + waiting_for_budget_owner.set() + return pending + + monkeypatch.setattr(scheduler, "_active_scheduler_tasks", observe_progress) + active = asyncio.create_task(scheduler.run_turn(owner_aid, "owner operation")) + try: + await asyncio.wait_for(started.wait(), 2) + assert scheduler.allocated_tokens == total + assert await scheduler.send_message(0, recipient, "work", "accepted follow-up") == ( + "Message queued to aid 1." + ) + assert len(scheduler._message_inbox[recipient]) == 1 + if ending == "success": + release.set() + assert await asyncio.wait_for(active, 2) == "owner answer" + else: + await asyncio.wait_for(cancelled.wait(), 2) + if ending != "cancelled": + await asyncio.wait_for(waiting_for_budget_owner.wait(), 2) + assert not active.done() + assert scheduler.allocated_tokens == total + assert len(scheduler._message_inbox[recipient]) == 1 + release.set() + with pytest.raises(SchedulerTurnError) as failure: + await asyncio.wait_for(active, 2) + assert failure.value.aid == owner_aid + assert failure.value.phase is SessionPhase.ERROR + assert "GenerationTimeoutError" in failure.value.terminal_reason + + recipient_requests = [r for r in requests if r["messages"][0]["content"] == "recipient"] + assert len(requests) == (3 if owner_aid == 0 else 5) + assert len(recipient_requests) == 2 + assert "accepted follow-up" in str(recipient_requests[-1]) + assert not scheduler._message_inbox[recipient] + assert scheduler._sessions[recipient].phase is SessionPhase.DONE + assert not owner.runner.pending_cleanup_tasks + assert scheduler._pending_provider_budget_leases == {} + assert scheduler.allocated_tokens == scheduler.used_tokens + assert scheduler.used_tokens == ( + (0 if owner_aid == 0 else 30_000) + + 8 + (4 if ending in {"success", "late_success"} else 0) + ) + assert owner.runner.late_provider_usage == ((4,) if ending == "late_success" else ()) + if ending.startswith("late"): + assert "owner answer" not in str(owner.messages) + finally: + release.set() + await scheduler.cleanup() + await asyncio.gather(active, return_exceptions=True) + + +async def test_inbox_runs_after_first_of_two_deferred_grants_releases(monkeypatch, tmp_path): + started = [asyncio.Event(), asyncio.Event()] + release = [asyncio.Event(), asyncio.Event()] + both_deferred = asyncio.Event() + delivered = asyncio.Event() + recipient_requests = [] + + async def handler(request): + body = json.loads(request.content) + text = next(m["content"] for m in reversed(body["messages"]) if m["role"] == "user") + if text in {"owner 0 operation", "owner 1 operation"}: + index = int(text.split()[1]) + started[index].set() + try: + await release[index].wait() + except asyncio.CancelledError: + await release[index].wait() + if "accepted follow-up" in text: + recipient_requests.append(text) + delivered.set() + return completion_http_response(request, text="done", output_tokens=19_986 if text == "lead warmup" else 2) + + monkeypatch.setenv("OPENCOLLAB_API_USAGE_LOG", str(tmp_path / "usage.jsonl")) + monkeypatch.setenv("OPENCOLLAB_WRITE_NUDGE_MODE", "off") + monkeypatch.setenv("OPENCOLLAB_BUDGET_NUDGE_MODE", "off") + install_sdk_transport(monkeypatch, "openai", handler) + async with LLMClient( + model="gpt-4o", provider="openai", + api_key="local-controlled", # pragma: allowlist secret + base_url="https://controlled.invalid/v1", max_retries=0, + ) as client: + scheduler = Scheduler( + session_factory=_Factory(client, 1.0), + worktree_pool=WorktreePool(str(tmp_path), use_worktrees=False), + event_sink=EventBus(), max_budget_tokens=40_000, + ) + lead = build_session(agent=Agent(name="lead", system_prompt="lead"), llm=client, max_budget_tokens=40_000) + scheduler.register_lead(lead) + assert await scheduler.run("lead warmup") == "done" + for aid, role in enumerate(("recipient", "owner-one", "owner-two"), 1): + assert await scheduler.spawn(0, role, f"{role} warmup") == aid + await scheduler._tasks[aid] + assert scheduler.used_tokens == 20_000 + + original_active = scheduler._active_scheduler_tasks + + def observe(): + tasks = original_active() + if len(scheduler._pending_provider_budget_leases) == 2: + both_deferred.set() + return tasks + + monkeypatch.setattr(scheduler, "_active_scheduler_tasks", observe) + for aid in (2, 3): + scheduler._sessions[aid].runner._per_call_timeout = 0.5 + work = [asyncio.create_task(scheduler.run_turn(aid, f"owner {aid - 2} operation")) for aid in (2, 3)] + try: + await asyncio.wait_for(asyncio.gather(*(event.wait() for event in started)), 3) + assert scheduler.allocated_tokens == 40_000 + assert await scheduler.send_message(0, 1, "work", "accepted follow-up") == "Message queued to aid 1." + await asyncio.wait_for(both_deferred.wait(), 3) + release[0].set() + await asyncio.wait_for(delivered.wait(), 3) + + assert not scheduler._message_inbox[1] + assert {lease[0] for lease in scheduler._pending_provider_budget_leases.values()} == {3} + assert not work[1].done() + assert len(recipient_requests) == 1 + second_owner, = next(iter(scheduler._pending_provider_budget_leases.values()))[3] + settled = asyncio.Event() + second_owner.add_done_callback(lambda _: settled.set()) + release[1].set() + results = await asyncio.wait_for(asyncio.gather(*work, return_exceptions=True), 3) + assert all(isinstance(error, SchedulerTurnError) for error in results) + await asyncio.wait_for(settled.wait(), 3) + assert not scheduler._pending_provider_budget_leases + assert scheduler.allocated_tokens == scheduler.used_tokens + finally: + for event in release: + event.set() + await scheduler.cleanup() + await asyncio.gather(*work, return_exceptions=True) + + +def _budget_callback_scheduler(tmp_path, owner): + state = SessionState(messages=[]) + state.set_phase(SessionPhase.DONE) + lead = SimpleNamespace( + agent=Agent(name="lead", system_prompt="lead"), state=state, + runner=SimpleNamespace(pending_cleanup_tasks=(owner,)), used_tokens=0, + ) + scheduler = Scheduler( + session_factory=_Factory(None, 1.0), + worktree_pool=WorktreePool(str(tmp_path), use_worktrees=False), + event_sink=EventBus(), max_budget_tokens=100_000, + ) + scheduler.register_lead(lead) + scheduler._reserve_turn_lease(0) + recipient_state = SessionState(messages=[]) + recipient_state.set_phase(SessionPhase.DONE) + scheduler.table.add(SessionControlBlock( + aid=1, parent_aid=0, agent=Agent(name="recipient", system_prompt="recipient"), + state=recipient_state, + )) + return scheduler, state + + +async def test_done_provider_callback_releases_budget_before_the_next_inbox_drain(monkeypatch, tmp_path): + async def completed_owner(): + return None + + owner = asyncio.create_task(completed_owner()) + await owner + scheduler, state = _budget_callback_scheduler(tmp_path, owner) + owner.add_done_callback(lambda _task: state.add_used_tokens(17)) + assert scheduler._defer_budget_lease_for_provider_cleanup(0) + scheduler._message_inbox[1] = [object()] + observations = [] + + async def drain_after_budget_release(): + observations.append(scheduler.allocated_tokens) + if scheduler.allocated_tokens < scheduler._max_budget_tokens: + scheduler._message_inbox[1].clear() + + monkeypatch.setattr(scheduler, "_drain_ready_message_inboxes", drain_after_budget_release) + assert not scheduler._active_scheduler_tasks() + assert scheduler._pending_provider_budget_leases + await scheduler._wait_for_scheduler_progress(0) + assert observations == [100_000, 17] + assert not scheduler._message_inbox[1] + assert not scheduler._pending_provider_budget_leases + assert scheduler.allocated_tokens == scheduler.used_tokens == 17 + + +async def test_deferred_budget_owner_counts_as_progress_only_for_an_unheld_inbox(tmp_path): + release = asyncio.Event() + owner = asyncio.create_task(release.wait()) + scheduler, _state = _budget_callback_scheduler(tmp_path, owner) + assert scheduler._defer_budget_lease_for_provider_cleanup(0) + try: + assert not scheduler._active_scheduler_tasks() + scheduler._message_inbox[1] = [object()] + scheduler._cancelled_turn_inbox_holds.add(1) + assert not scheduler._active_scheduler_tasks() + scheduler._cancelled_turn_inbox_holds.clear() + assert scheduler._active_scheduler_tasks() == {owner} + finally: + release.set() + await owner diff --git a/tests/workflows/test_workflow_discovery_module_names.py b/tests/workflows/test_workflow_discovery_module_names.py index e0dc122b..a91f7c27 100644 --- a/tests/workflows/test_workflow_discovery_module_names.py +++ b/tests/workflows/test_workflow_discovery_module_names.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import sys import pytest @@ -9,6 +10,7 @@ from opencollab import OpenCollab from opencollab.adapters.cli.main import app +from opencollab.application.workflow import WorkflowContext from opencollab.bootstrap import _workflow_runtime_discovery as workflow_discovery @@ -102,3 +104,101 @@ async def test_multiple_workflow_files_keep_sibling_and_self_imports(tmp_path): assert _import_state() == before assert await registry.get("task-entry").fn(None, {}) == "shared task" assert _import_state() == before + + +async def test_concurrent_calls_share_one_loaded_package_until_both_finish(tmp_path): + directory = tmp_path / "workflows" + directory.mkdir() + (directory / "_helper.py").write_text("def answer():\n return 42\n", encoding="utf-8") + (directory / "task.py").write_text( + "from opencollab import workflow\n" + "@workflow\n" + "async def overlap(ctx, args):\n" + " args['entered'].set()\n" + " await args['release'].wait()\n" + " from ._helper import answer\n" + " return answer()\n", + encoding="utf-8", + ) + before = _import_state() + spec = workflow_discovery.discover_workflows(str(directory)).get("overlap") + client = OpenCollab(tmp_path, model="offline") + entered = [asyncio.Event(), asyncio.Event()] + release = [asyncio.Event(), asyncio.Event()] + calls = [asyncio.create_task(client.workflow( + spec, {"entered": entered[index], "release": release[index]}, trace=False, + )) for index in range(2)] + try: + await asyncio.wait_for(asyncio.gather(*(event.wait() for event in entered)), 5) + release[0].set() + assert (await asyncio.wait_for(calls[0], 5)).output == 42 + assert _import_state() != before + release[1].set() + assert (await asyncio.wait_for(calls[1], 5)).output == 42 + assert _import_state() == before + finally: + for event in release: + event.set() + await asyncio.gather(*calls, return_exceptions=True) + + +@pytest.mark.parametrize("awaited_first", [False, True]) +async def test_cleanup_callbacks_keep_imports_for_following_owned_tasks(tmp_path, awaited_first): + directory = tmp_path / "workflows" + directory.mkdir() + (directory / "_helper.py").write_text("def answer():\n return 42\n", encoding="utf-8") + (directory / "task.py").write_text('''import asyncio +from opencollab import workflow + +@workflow +async def cleanup(ctx, args): + async def finish(): + args["following_started"].set() + await args["release_following"].wait() + from ._helper import answer + args["answers"].append(answer()) + + def follow(_task): + asyncio.get_running_loop().call_soon( + lambda: ctx._track_pending_cleanup(asyncio.create_task(finish())), + ) + + first = asyncio.create_task(asyncio.sleep(0)) + first.add_done_callback(follow) + ctx._track_pending_cleanup(first) + if args["awaited_first"]: + await first + return "returned" +''', encoding="utf-8") + before = _import_state() + spec = workflow_discovery.discover_workflows(str(directory)).get("cleanup") + ctx = WorkflowContext(factory=object()) + args = {name: asyncio.Event() for name in ( + "following_started", "release_following", + )} + args["awaited_first"] = awaited_first + args["answers"] = [] + caller = asyncio.current_task() + ctx._active_call_tasks.add(caller) + try: + assert await spec.fn(ctx, args) == "returned" + assert _import_state() != before + await asyncio.wait_for(args["following_started"].wait(), 5) + assert _import_state() != before + args["release_following"].set() + + async def wait_for_package_release(): + while _import_state() != before: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_package_release(), 5) + assert args["answers"] == [42] + assert _import_state() == before + ctx._active_call_tasks.discard(caller) + await asyncio.wait_for(ctx.wait_for_pending_cleanup(), 5) + assert args["answers"] == [42] + assert _import_state() == before + finally: + args["release_following"].set() + ctx._active_call_tasks.discard(caller) + await ctx.wait_for_pending_cleanup()