From c887f2048d85ad083353bc1bfe535a808d87052a Mon Sep 17 00:00:00 2001 From: jtaroreh Date: Sat, 24 Oct 2026 15:51:47 -0700 Subject: [PATCH 01/11] test: reproduce worker status and execution lifecycle failures --- tests/test_worker_execution_lifecycle.py | 206 +++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 tests/test_worker_execution_lifecycle.py diff --git a/tests/test_worker_execution_lifecycle.py b/tests/test_worker_execution_lifecycle.py new file mode 100644 index 0000000..94ceeac --- /dev/null +++ b/tests/test_worker_execution_lifecycle.py @@ -0,0 +1,206 @@ +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +SCRIPTS = Path(__file__).resolve().parents[1] / "skills/swarm/scripts" +sys.path.insert(0, str(SCRIPTS)) +import cloud_dispatch +import cloud_worker +import result_harvester +import storage_uploader +from worker import partner_engine + + +@pytest.mark.parametrize("text, expected", [ + ("Reporting `[STATUS: PASS]` now would be false. The correct status is [STATUS: BLOCKED]", "ISSUES"), + ("Deprecated `[STATUS: PASS]`\n[STATUS: BLOCKED]", "BLOCKED"), + ("Finished.\n[STATUS: PASS]", "PASS"), + ("[STATUS: PASS]\n[STATUS: BLOCKED]", "ISSUES"), + ("[STATUS: PASS]\n[STATUS: PASS]", "ISSUES"), + ("```text\n[STATUS: PASS]\n```", "ISSUES"), + ("~~~\n[STATUS: PASS]\n~~~\n[STATUS: BLOCKED]", "BLOCKED"), + ("> [STATUS: PASS]", "ISSUES"), + ('"[STATUS: PASS]"', "ISSUES"), + ("`[STATUS: PASS]`", "ISSUES"), + ("[STATUS: PASS] trailing prose", "ISSUES"), + ("[STATUS: DEV_IMPROVED]", "ISSUES"), + ("[STATUS: PASS]\nMore prose", "ISSUES"), + ("[STATUS: MAYBE]", "ISSUES"), +]) +def test_strict_terminal_status(text, expected): + for evaluate in (cloud_worker.evaluate_agent_status, partner_engine._evaluate_agent_status): + assert evaluate(text)[0] == expected + + +@pytest.mark.parametrize("bad", [0, -1, True, 1.5, "1.5", "oops", None]) +def test_invalid_execution_limits(bad): + from worker.execution import resolve_execution_limits + with pytest.raises(ValueError): + resolve_execution_limits({"max_agent_turns": bad}, environ={}) + + +def test_limits_precedence_and_immutability(): + from dataclasses import FrozenInstanceError + from worker.execution import resolve_execution_limits + limits = resolve_execution_limits( + {"verify_timeout_s": 41}, + runtime={"max_agent_turns": 31, "command_timeout_s": 500}, + environ={"AGYSTACK_MAX_AGENT_TURNS": "32", "COMMAND_TIMEOUT": "600"}, + cli={"max_agent_turns": 33}, + ) + assert (limits.max_agent_turns, limits.command_timeout_s, limits.verify_timeout_s) == (33, 600, 41) + with pytest.raises(FrozenInstanceError): + limits.max_agent_turns = 2 + with pytest.raises(ValueError): + resolve_execution_limits({"unexpected": 5}, environ={}) + with pytest.raises(ValueError): + resolve_execution_limits([], environ={}) + assert resolve_execution_limits(environ={}).command_timeout_s == 300 + assert resolve_execution_limits(environ={}, command_default=900).command_timeout_s == 900 + + +@pytest.mark.parametrize("provider", ["claude", "fable", "deepseek"]) +@pytest.mark.parametrize("finish", [True, False]) +def test_provider_turn_limits_and_final_response(tmp_path, provider, finish): + (tmp_path / "note.txt").write_text("still working") + tool = {"type": "tool_use", "id": "read", "name": "read_file", "input": {"path": "note.txt"}} + responses = [] + for turn in range(33): + final = finish and turn == 32 + text = "Finished.\n[STATUS: PASS]" if final else "[STATUS: PASS]" + if provider == "deepseek": + fc = SimpleNamespace(name="read_file", args={"path": "note.txt"}) + responses.append(SimpleNamespace(text=text, candidates=[], function_calls=[] if final else [fc])) + else: + responses.append(SimpleNamespace(content=[{"type": "text", "text": text}] + ([] if final else [tool]))) + sdk = MagicMock() + if provider == "claude": + request = sdk.AnthropicVertex.return_value.messages.create + entry = partner_engine.execute_claude_model_agent + getter = "_get_anthropic" + elif provider == "fable": + request = sdk.Endpoint.return_value.predict + entry = partner_engine.execute_fable_model_agent + getter = "_get_aiplatform" + else: + request = sdk.Client.return_value.models.generate_content + entry = partner_engine.execute_deepseek_model_agent + getter = "_get_genai" + request.side_effect = responses + with patch.object(partner_engine, getter, return_value=sdk): + status, summary = entry(provider, "work", tmp_path, task_item={"execution_limits": {"max_agent_turns": 33}}) + assert request.call_count == 33 + assert status == ("PASS" if finish else "ISSUES") + if not finish: + assert "limit=33" in summary + assert "consumed=33" in summary + assert "read_file" in summary + assert "last_response" in summary + + +def test_timeout_keeps_output_and_kills_descendants(tmp_path): + from worker.execution import ExecutionLimits + child = "import time; time.sleep(2); open('survived', 'w').write('bad')" + command = f"echo partial-output; echo partial-error >&2; {shlex.quote(sys.executable)} -c {shlex.quote(child)} & wait" + output = partner_engine.execute_workspace_tool( + "run_command", {"command": command}, tmp_path, + limits=ExecutionLimits(command_timeout_s=1), + ) + assert "timed out" in output.lower() + assert "partial-output" in output + assert "partial-error" in output + logs = list((tmp_path / ".agystack/diagnostics").glob("*.log")) + assert logs and "partial-output" in logs[0].read_text() + time.sleep(1.5) + assert not (tmp_path / "survived").exists() + + +def test_command_pipeline_failure_is_not_hidden(tmp_path): + status, _ = cloud_worker.execute_task( + {"type": "command", "command": "false | tail -n 1"}, "", tmp_path, "", "" + ) + assert status == "ISSUES" + + +@pytest.fixture +def worker_repo(tmp_path): + repo = tmp_path / "repo-task-0" + repo.mkdir() + for args in (["init"], ["config", "user.email", "test@example.com"], ["config", "user.name", "Test"]): + subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True) + (repo / "service.py").write_text("value = 1\n") + subprocess.run(["git", "add", "."], cwd=repo, check=True, capture_output=True) + subprocess.run(["git", "commit", "-m", "base"], cwd=repo, check=True, capture_output=True) + return repo + + +@pytest.mark.parametrize("status, candidate", [("BLOCKED", True), ("ISSUES", True), ("PASS", False)]) +def test_worker_finalization_preserves_evidence(worker_repo, tmp_path, monkeypatch, status, candidate): + storage = tmp_path / "storage" + task = {"brief": "work", "evidence_files": ["results/proof.txt"], + "verify_command": "echo verifier-ran > verification.txt; exit 1"} + monkeypatch.setenv("REPO_URL", "https://example.com/repo") + monkeypatch.setenv("TASK_MANIFEST", json.dumps([task])) + monkeypatch.setenv("USE_VERTEX_AI", "1") + monkeypatch.setenv("GCS_BUCKET", "test-bucket") + monkeypatch.setenv("GCS_PREFIX", "run") + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(storage)) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(cloud_worker.tempfile, "gettempdir", lambda: str(tmp_path)) + def clone(**kwargs): + repo = kwargs["repo_dir"] + subprocess.run(["git", "init"], cwd=repo, check=True, capture_output=True) + for key, value in [("user.email", "test@example.com"), ("user.name", "Test")]: + subprocess.run(["git", "config", key, value], cwd=repo, check=True) + (repo / "service.py").write_text("value = 1\n") + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "base"], cwd=repo, check=True, capture_output=True) + def execute(**kwargs): + repo = kwargs["repo_dir"] + if candidate: + (repo / "service.py").write_text("value = 2\n") + (repo / "results").mkdir() + (repo / "results/proof.txt").write_text("partial proof") + return status, "work summary" + monkeypatch.setattr(cloud_worker, "clone_and_checkout_task", clone) + monkeypatch.setattr(cloud_worker, "execute_task", execute) + monkeypatch.setattr(cloud_worker, "StorageMessenger", None) + try: + cloud_worker.main() + except SystemExit as exc: + assert exc.code == 1 + target = storage / "test-bucket/run/task-0" + published = json.loads((target / "status.json").read_text()) + assert published["status"] == ("ISSUES" if status == "PASS" else status) + assert (target / "evidence/results/proof.txt").read_text() == "partial proof" + assert not (target / "patch.diff").exists() + if candidate: + assert "+value = 2" in (target / "partial.patch.diff").read_text() + else: + assert (worker_repo / "verification.txt").is_file() + harvested = result_harvester.harvest_candidate_patches("test-bucket", "run", tmp_path / "harvest") + assert len(harvested) == 1 + assert harvested[0]["status"] == published["status"] + assert not harvested[0]["patch_file"] + assert (tmp_path / "harvest/task-0/evidence/results/proof.txt").is_file() + + +@pytest.mark.parametrize("reports", [[], [{"task_index": 0, "status": "PASS"}], + [{"task_index": 0, "status": "PASS"}, {"task_index": 1, "status": "UNKNOWN"}], + [{"task_index": 0, "status": "PASS"}, {"task_index": 0, "status": "PASS"}]]) +def test_dispatch_requires_complete_terminal_coverage(reports): + assert cloud_dispatch.failed_task_count(reports, 2) > 0 + + +def test_dispatch_complete_pass(): + assert cloud_dispatch.failed_task_count([ + {"task_index": 0, "status": "PASS"}, {"task_index": 1, "status": "PASS"} + ], 2) == 0 From d0f0254ca2ad26c22455dc49335d7f544935bde3 Mon Sep 17 00:00:00 2001 From: jtaroreh Date: Sat, 24 Oct 2026 15:59:57 -0700 Subject: [PATCH 02/11] fix: enforce worker execution limits and preserve partial research evidence --- agystack-runtime.json.example | 5 + skills/swarm/scripts/cloud_dispatch.py | 77 +++- skills/swarm/scripts/cloud_worker.py | 175 ++++----- skills/swarm/scripts/result_harvester.py | 20 +- skills/swarm/scripts/storage_uploader.py | 58 ++- skills/swarm/scripts/worker/execution.py | 189 ++++++++++ skills/swarm/scripts/worker/partner_engine.py | 113 +++--- tests/test_architecture_remediation.py | 26 +- tests/test_cloud_preflight_and_worker.py | 15 +- tests/test_cloud_worker_vertex_partner.py | 28 +- tests/test_worker_execution_lifecycle.py | 354 ++++++++++++++++-- 11 files changed, 812 insertions(+), 248 deletions(-) create mode 100644 skills/swarm/scripts/worker/execution.py diff --git a/agystack-runtime.json.example b/agystack-runtime.json.example index 27a14a5..c52beae 100644 --- a/agystack-runtime.json.example +++ b/agystack-runtime.json.example @@ -5,6 +5,11 @@ "job_name": "agystack-swarm-worker", "image": "us-central1-docker.pkg.dev/your-gcp-project-id/agystack/worker:latest", "parallelism": 16, + "execution_limits": { + "max_agent_turns": 30, + "command_timeout_s": 300, + "verify_timeout_s": 300 + }, "model": "inherit", "auth_mode": "vertex", "vertex_location": "global", diff --git a/skills/swarm/scripts/cloud_dispatch.py b/skills/swarm/scripts/cloud_dispatch.py index 76a6c80..3674cf9 100644 --- a/skills/swarm/scripts/cloud_dispatch.py +++ b/skills/swarm/scripts/cloud_dispatch.py @@ -32,6 +32,12 @@ is_partner_model = None PARTNER_MODEL_ALIASES = {} +try: + from worker.execution import LIMIT_ENV, TERMINAL_STATES, resolve_execution_limits +except ImportError: + from skills.swarm.scripts.worker.execution import LIMIT_ENV, TERMINAL_STATES, resolve_execution_limits + + TIER3_DEDICATED_MODELS = { "glm-5-3-flash", "glm-5.3-flash", @@ -370,6 +376,14 @@ def _flush(): for line in log_output.splitlines(): trimmed = line.strip() if trimmed.startswith("[STATUS:"): + if not current_status and current_evidence: + current_status = trimmed.split("[STATUS:")[1].split("]")[0].strip() + _flush() + current_status = "" + current_evidence = [] + current_summary = [] + mode = None + continue _flush() current_status = trimmed.split("[STATUS:")[1].split("]")[0].strip() current_evidence = [] @@ -1174,6 +1188,19 @@ def resolve_swarm_model( ) +def failed_task_count(report_items: List[Dict[str, Any]], task_count: int) -> int: + reports = {} + invalid = 0 + for item in report_items: + index = item.get("task_index") + if type(index) is not int or not 0 <= index < task_count or index in reports: + invalid += 1 + continue + status = item.get("status") + reports[index] = status if isinstance(status, str) and status in TERMINAL_STATES else "UNKNOWN" + return invalid + sum(reports.get(index) != "PASS" for index in range(task_count)) + + def print_candidate_patches_table( harvested_patches: List[Dict[str, Any]], baseline_score: Optional[float] = None, @@ -1266,6 +1293,9 @@ def main() -> None: parser.add_argument("--wait", dest="wait", action="store_true", help="Wait for job completion, stream milestones, and harvest reports (default).") parser.add_argument("--no-wait", dest="wait", action="store_false", help="Do not wait for job completion; dispatch asynchronously.") parser.add_argument("--session", "--session-id", dest="session_id", type=str, help="Swarm session ID (default: swarm-).") + parser.add_argument("--max-agent-turns", type=int, default=None, help="Maximum provider requests per task.") + parser.add_argument("--command-timeout", type=int, default=None, help="Command timeout in seconds.") + parser.add_argument("--verify-timeout", type=int, default=None, help="Verification timeout in seconds.") parser.add_argument("--orchestrator-timeout", type=float, default=600.0, help="Timeout in seconds for orchestrator replies (default: 600.0).") parser.add_argument("--set-secrets", type=str, help="Comma-separated secrets mapping for Cloud Run Job (e.g. GEMINI_API_KEY=gemini-key:latest).") parser.add_argument("--base-branch", type=str, default=None, help="Base branch for workers to checkout (default: remote default branch).") @@ -1364,6 +1394,18 @@ def main() -> None: return runtime_cfg = find_runtime_config() or {} + cli_limits = {name: value for name, value in { + "max_agent_turns": args.max_agent_turns, + "command_timeout_s": args.command_timeout, + "verify_timeout_s": args.verify_timeout, + }.items() if value is not None} + try: + dispatch_limits = resolve_execution_limits( + runtime=runtime_cfg.get("execution_limits", {}), cli=cli_limits, + ) + except ValueError as exc: + parser.error(str(exc)) + job_name = args.job_name or runtime_cfg.get("job_name") or "agystack-swarm-worker" region = args.region or runtime_cfg.get("region") or "us-central1" project = args.project or runtime_cfg.get("project_id") @@ -1417,7 +1459,15 @@ def main() -> None: print(f"Error: Failed to harvest GCS results for session {session_id}: {exc}", file=sys.stderr) sys.exit(1) - print_candidate_patches_table(harvested_patches, baseline_score=args.baseline_score) + failed_count = failed_task_count(harvested_patches, args.tasks) if args.tasks and args.tasks > 0 else 1 + print_candidate_patches_table( + harvested_patches, baseline_score=args.baseline_score, failed_count=failed_count, + allow_partial=args.ignore_partial_failures, + ) + if not args.tasks: + print("Complete task coverage is unknown. Supply --tasks to verify the expected count.") + if failed_count: + sys.exit(1) return if args.wait_execution: @@ -1477,8 +1527,13 @@ def main() -> None: except Exception as exc: print(f"Warning: Failed to harvest GCS results: {exc}", file=sys.stderr) + expected_count = args.tasks + if isinstance(desc_data, dict): + expected_count = desc_data.get("spec", {}).get("taskCount", desc_data.get("taskCount", expected_count)) + failed_cnt = failed_task_count(harvested_patches, expected_count) if type(expected_count) is int and expected_count > 0 else 1 + if not exec_succeeded: + failed_cnt = max(1, failed_cnt) if harvested_patches: - failed_cnt = 0 if exec_succeeded else 1 print_candidate_patches_table( harvested_patches, baseline_score=args.baseline_score, @@ -1487,7 +1542,8 @@ def main() -> None: allow_partial=getattr(args, "ignore_partial_failures", False), ) - if not exec_succeeded: + if not exec_succeeded or failed_cnt: + print("Execution lacks complete passing task status coverage.", file=sys.stderr) sys.exit(1) return @@ -1583,7 +1639,15 @@ def main() -> None: print(f"Error: Failed to load manifest: {exc}", file=sys.stderr) sys.exit(1) + try: + for task in tasks_list: + if isinstance(task, dict): + resolve_execution_limits(task.get("execution_limits", {}), environ={}) + except ValueError as exc: + parser.error(str(exc)) task_count = args.tasks or (len(tasks_list) if tasks_list else 1) + if tasks_list and task_count != len(tasks_list): + parser.error("--tasks must match the manifest task count") manifest_serialized = json.dumps(tasks_list) if tasks_list else "" env_vars = { @@ -1607,6 +1671,9 @@ def main() -> None: session_id = args.session_id or f"swarm-{int(time.time())}" env_vars["SWARM_SESSION_ID"] = session_id env_vars["SESSION_ID"] = session_id + for name, env_key in LIMIT_ENV.items(): + if name in cli_limits or env_key in os.environ or name in runtime_cfg.get("execution_limits", {}): + env_vars[env_key] = str(getattr(dispatch_limits, name)) if args.orchestrator_timeout: env_vars["ORCHESTRATOR_TIMEOUT"] = str(args.orchestrator_timeout) if args.base_branch: @@ -1881,7 +1948,7 @@ def main() -> None: print(f"Total: {len(report_items)} | PASS: {pass_count} | ISSUES: {issues_count} | BLOCKED: {blocked_count}") print("=" * 80) - failed_tasks_count = issues_count + blocked_count + failed_tasks_count = failed_task_count(report_items, task_count) if not exec_succeeded and failed_tasks_count == 0: failed_tasks_count = 1 @@ -1901,7 +1968,7 @@ def main() -> None: cleanup_session(session_id=session_id, bucket_name=gcs_bucket) - if not exec_succeeded: + if not exec_succeeded or failed_tasks_count > 0: sys.exit(1) except subprocess.CalledProcessError as exc: diff --git a/skills/swarm/scripts/cloud_worker.py b/skills/swarm/scripts/cloud_worker.py index 305d59b..3f61849 100644 --- a/skills/swarm/scripts/cloud_worker.py +++ b/skills/swarm/scripts/cloud_worker.py @@ -10,7 +10,6 @@ import json import os -import re import shutil import subprocess import sys @@ -34,7 +33,7 @@ 1. Implement the requested task directly and surgically in the workspace. 2. Maintain high engineering rigor: run test suites and verification commands before completing. 3. Keep diffs minimal and clean. Avoid narrating comments or speculative code. -4. Conclude with a clear summary and status ([STATUS: PASS] or [STATUS: ISSUES]). +4. End your final response with exactly one standalone unquoted status line outside code fences, [STATUS: PASS], [STATUS: ISSUES], or [STATUS: BLOCKED]. """ @@ -169,20 +168,10 @@ def emit_agent_response( ) -CREDENTIAL_PATTERNS = [ - re.compile(r"ghp_[A-Za-z0-9_]{36,}"), - re.compile(r"AIza[0-9A-Za-z-_]{35}"), - re.compile(r"ya29\.[0-9A-Za-z-_]+"), - re.compile(r"Bearer [A-Za-z0-9_\-\.]+"), -] - - -def scrub_credentials(text: str) -> str: - if not text: - return text - for pattern in CREDENTIAL_PATTERNS: - text = pattern.sub("[REDACTED]", text) - return text +try: + from worker.execution import CREDENTIAL_PATTERNS as CREDENTIAL_PATTERNS, scrub_credentials +except ImportError: + from skills.swarm.scripts.worker.execution import CREDENTIAL_PATTERNS as CREDENTIAL_PATTERNS, scrub_credentials class TokenShield: @@ -701,14 +690,10 @@ def parse_porcelain_status(stdout: str) -> List[str]: return modified_files -def evaluate_agent_status(agent_text: str) -> Tuple[str, str]: - match = re.search(r"\[STATUS:\s*(PASS|ISSUES|BLOCKED|DEV_IMPROVED)\s*\]", agent_text, re.IGNORECASE) - if match: - st = match.group(1).upper() - if st == "DEV_IMPROVED": - return "PASS", agent_text - return st, agent_text - return "ISSUES", agent_text +try: + from worker.execution import ExecutionLimits, evaluate_agent_status, limits_for_task, run_bounded_shell, write_diagnostic +except ImportError: + from skills.swarm.scripts.worker.execution import ExecutionLimits, evaluate_agent_status, limits_for_task, run_bounded_shell, write_diagnostic try: @@ -811,11 +796,15 @@ def execute_task( task_index: int = 0, messenger: Optional[Any] = None, orchestrator_timeout: float = 600.0, + *, + limits: Optional[ExecutionLimits] = None, ) -> Tuple[str, str]: task_type = "agent" if isinstance(task_item, dict): task_type = task_item.get("type", "agent") + limits = limits or limits_for_task(task_item, command_default=900 if task_type == "command" else 300) + # Command execution runner if task_type == "command": cmd_str = "" @@ -828,26 +817,9 @@ def execute_task( emit_tool_invocation(task_index, "run_command", {"command": cmd_str}) emit_milestone(task_index, "RUNNING_COMMAND", cmd_str) print(f"Executing command task: {cmd_str}", flush=True) - timeout = int(os.environ.get("COMMAND_TIMEOUT", "900")) - try: - res = subprocess.run( - cmd_str, - shell=True, - cwd=repo_dir, - capture_output=True, - text=True, - timeout=timeout, - ) - if res.returncode == 0: - emit_tool_result(task_index, "run_command", res.stdout, success=True) - return "PASS", res.stdout - else: - err_msg = res.stderr if res.stderr else res.stdout - emit_tool_result(task_index, "run_command", err_msg, success=False) - return "ISSUES", err_msg - except subprocess.TimeoutExpired: - emit_tool_result(task_index, "run_command", f"Command timed out after {timeout} seconds", success=False) - return "ISSUES", f"Command timed out after {timeout} seconds: {cmd_str}" + res, output = run_bounded_shell(cmd_str, repo_dir, limits.command_timeout_s) + emit_tool_result(task_index, "run_command", output, success=res.returncode == 0) + return ("PASS" if res.returncode == 0 else "ISSUES"), output resolved_model = resolve_worker_model(raw_model=model_override, task_item=task_item) if is_deepseek_model(resolved_model): @@ -862,6 +834,7 @@ def execute_task( project=project, location=location_override, messenger=messenger, + limits=limits, ) if is_partner_model(resolved_model): @@ -875,6 +848,7 @@ def execute_task( project=project, location=location, messenger=messenger, + limits=limits, ) # General Agent Execution using Google Antigravity SDK @@ -1031,6 +1005,8 @@ def main() -> None: ) sys.exit(1) + limits = limits_for_task(task_item, command_default=900 if isinstance(task_item, dict) and task_item.get("type") == "command" else 300) + task_model = resolve_worker_model(raw_model=model_override, task_item=task_item) use_vertex = ( os.environ.get("USE_VERTEX_AI", "").lower() in ("1", "true", "yes") @@ -1144,22 +1120,11 @@ def main() -> None: task_index=task_index, messenger=messenger, orchestrator_timeout=orchestrator_timeout, + limits=limits, ) - if agent_status == "BLOCKED": - emit_milestone(task_index, "COMPLETE", "BLOCKED") - print("=" * 80) - print("[STATUS: BLOCKED]") - print("Evidence:") - print(f"- Task Index: {task_index}") - print(f"- Branch: {branch_name}") - print("- Commit SHA: N/A") - print("- Changes Pushed: False") - print("- Diff Stat:\nExecution blocked before changes.") - print("Summary:") - print(agent_summary.strip()) - print("=" * 80, flush=True) - sys.exit(1) + agent_summary = scrub_credentials(agent_summary) + write_diagnostic(repo_dir, "agent-result", agent_summary) score, accuracy, latency_ms = parse_score_metrics(repo_dir) gcs_bucket = os.environ.get("GCS_RESULTS_BUCKET", "").strip() @@ -1209,53 +1174,38 @@ def main() -> None: agent_summary += f"\n[Steering Instructions Applied]: {steer_summary}" print(f"Notice: Applied {len(steer_messages)} steering instruction(s): {steer_summary}", flush=True) - # Prevent ghost commits: Only commit when agent_status == "PASS" and candidate files exist - if agent_status == "PASS" and candidate_files: + if agent_status == "PASS" and verify_cmd_str: emit_milestone(task_index, "VALIDATING_CANDIDATE") - if verify_cmd_str: - print(f"Running verification command: {verify_cmd_str}", flush=True) - verify_timeout = int(os.environ.get("VERIFY_TIMEOUT", "300")) - try: - test_check = subprocess.run( - verify_cmd_str, - shell=True, - cwd=repo_dir, - capture_output=True, - text=True, - timeout=verify_timeout, - ) - if test_check.returncode != 0: - agent_status = "ISSUES" - err_snippet = (test_check.stderr.strip() or test_check.stdout.strip())[-500:] - diff_stat = f"Commit rejected: Candidate failed verification command (exit code {test_check.returncode}): {err_snippet}" - print(f"Warning: Candidate modifications failed verification command (exit code {test_check.returncode}). Commit blocked.", file=sys.stderr) - except subprocess.TimeoutExpired: - agent_status = "ISSUES" - diff_stat = f"Commit rejected: Candidate verification command timed out after {verify_timeout}s." - print(f"Warning: Candidate verification command timed out after {verify_timeout}s. Commit blocked.", file=sys.stderr) - - if agent_status == "PASS": - try: - for cand_file in candidate_files: - run_command(["git", "add", "--", cand_file], cwd=repo_dir, check=True) - cached_diff = run_command(["git", "diff", "--cached", "--name-only"], cwd=repo_dir, check=False) - if not cached_diff.stdout.strip(): - diff_stat = "Commit skipped: No candidate changes staged." - print("Notice: No candidate changes staged; skipping git commit.", file=sys.stderr) - else: - commit_msg = f"worker-{task_index}: execute swarm brief" - run_command(["git", "commit", "-m", commit_msg], cwd=repo_dir, check=True) - commit_res = run_command(["git", "rev-parse", "HEAD"], cwd=repo_dir, check=True) - commit_sha = commit_res.stdout.strip() - diff_res = run_command(["git", "diff", "--stat", "HEAD~1", "HEAD"], cwd=repo_dir, check=False) - diff_stat = diff_res.stdout.strip() - except subprocess.CalledProcessError as exc: - agent_status = "ISSUES" - commit_sha = None - err_text = (exc.stderr or str(exc)).strip() - diff_stat = f"Commit failed: {err_text}" - err_msg = exc.stderr.replace(gh_token, "REDACTED") if gh_token and gh_token in exc.stderr else err_text - print(f"Warning: Git commit preparation failed: {err_msg}", file=sys.stderr) + test_check, verify_output = run_bounded_shell( + verify_cmd_str, repo_dir, limits.verify_timeout_s, label="verification", + ) + agent_summary += f"\nVerification:\n{verify_output}" + if test_check.returncode != 0: + agent_status = "ISSUES" + diff_stat = f"Commit rejected: Candidate failed verification command (exit code {test_check.returncode})." + + if agent_status == "PASS" and candidate_files: + try: + for cand_file in candidate_files: + run_command(["git", "add", "--", cand_file], cwd=repo_dir, check=True) + cached_diff = run_command(["git", "diff", "--cached", "--name-only"], cwd=repo_dir, check=False) + if not cached_diff.stdout.strip(): + diff_stat = "Commit skipped: No candidate changes staged." + print("Notice: No candidate changes staged; skipping git commit.", file=sys.stderr) + else: + commit_msg = f"worker-{task_index}: execute swarm brief" + run_command(["git", "commit", "-m", commit_msg], cwd=repo_dir, check=True) + commit_res = run_command(["git", "rev-parse", "HEAD"], cwd=repo_dir, check=True) + commit_sha = commit_res.stdout.strip() + diff_res = run_command(["git", "diff", "--stat", "HEAD~1", "HEAD"], cwd=repo_dir, check=False) + diff_stat = diff_res.stdout.strip() + except subprocess.CalledProcessError as exc: + agent_status = "ISSUES" + commit_sha = None + err_text = (exc.stderr or str(exc)).strip() + diff_stat = f"Commit failed: {err_text}" + err_msg = exc.stderr.replace(gh_token, "REDACTED") if gh_token and gh_token in exc.stderr else err_text + print(f"Warning: Git commit preparation failed: {err_msg}", file=sys.stderr) elif agent_status != "PASS": diff_stat = f"Commit skipped: worker status is {agent_status} (PASS required; ghost commits prohibited)." print(f"Notice: Agent status is {agent_status}. Skipping git commit.", file=sys.stderr) @@ -1303,6 +1253,7 @@ def main() -> None: status=final_status, candidate_files=candidate_files, commit_sha=commit_sha, + evidence_files=task_item.get("evidence_files", []) if isinstance(task_item, dict) else [], ) if gcs_uris: emit_milestone(task_index, "ARTIFACTS_UPLOADED") @@ -1320,11 +1271,8 @@ def main() -> None: except Exception: pass - # Fail-closed check: if final_status == "PASS" and candidate_files were modified, - # but upload_run_artifacts() failed to upload patch.diff or status.json to GCS, - # or if upload explicitly failed, override final_status = "ISSUES". - if final_status == "PASS" and candidate_files: - if upload_failed or "patch.diff" not in gcs_uris or "status.json" not in gcs_uris: + if final_status == "PASS": + if upload_failed or "status.json" not in gcs_uris or (candidate_files and "patch.diff" not in gcs_uris): final_status = "ISSUES" fail_evidence = "Artifact upload to GCS failed; candidate patch could not be preserved." agent_summary += f"\n{fail_evidence}" @@ -1339,7 +1287,6 @@ def main() -> None: emit_milestone(task_index, "COMPLETE", final_status) print("=" * 80) - print(f"[STATUS: {final_status}]") print("Evidence:") print(f"- Task Index: {task_index}") print(f"- Branch: {branch_name}") @@ -1368,8 +1315,12 @@ def main() -> None: print(f"- GCS Score: {gcs_uris['score.json']}") print(f"- Diff Stat:\n{diff_stat}") print("Summary:") - print(agent_summary.strip()) - print("=" * 80, flush=True) + for summary_line in agent_summary.strip().splitlines(): + print(f"> {summary_line}") + print("=" * 80) + print(f"[STATUS: {final_status}]", flush=True) + if final_status != "PASS": + sys.exit(1) if __name__ == "__main__": diff --git a/skills/swarm/scripts/result_harvester.py b/skills/swarm/scripts/result_harvester.py index 67aac79..828da52 100644 --- a/skills/swarm/scripts/result_harvester.py +++ b/skills/swarm/scripts/result_harvester.py @@ -159,6 +159,12 @@ def harvest_candidate_patches( target_status = target_task_dir / "status.json" shutil.copy2(status_src, target_status) stf = target_status + for filename in ("partial.patch.diff", "results.tsv"): + source = task_folder / filename + if source.is_file(): + shutil.copy2(source, target_task_dir / filename) + if (task_folder / "evidence").is_dir(): + shutil.copytree(task_folder / "evidence", target_task_dir / "evidence", dirs_exist_ok=True) tasks_found[task_idx] = {"score_file": sf, "patch_file": pf, "status_file": stf} if not tasks_found: @@ -169,18 +175,22 @@ def harvest_candidate_patches( bucket = client.bucket(bucket_name) blobs = bucket.list_blobs(prefix=clean_prefix) for blob in blobs: - m = re.search(r"task-(\d+)/([^/]+)$", blob.name) + m = re.search(r"(?:^|/)task-(\d+)/(.+)$", blob.name) if not m: continue task_idx = int(m.group(1)) filename = m.group(2) - if filename not in ("score.json", "patch.diff", "status.json"): + relative = Path(filename) + if relative.is_absolute() or ".." in relative.parts: + continue + if filename not in ("score.json", "patch.diff", "partial.patch.diff", "status.json", "results.tsv") and not filename.startswith("evidence/"): continue if task_idx not in tasks_found: tasks_found[task_idx] = {} target_task_dir = dest / f"task-{task_idx}" target_task_dir.mkdir(parents=True, exist_ok=True) target_file = target_task_dir / filename + target_file.parent.mkdir(parents=True, exist_ok=True) blob.download_to_filename(str(target_file)) if filename == "score.json": tasks_found[task_idx]["score_file"] = target_file @@ -230,7 +240,6 @@ def harvest_candidate_patches( try: data = json.loads(Path(score_file).read_text(encoding="utf-8")) score = data.get("score") - status = data.get("status", "PASS" if score is not None else "UNKNOWN") except Exception: pass @@ -250,8 +259,13 @@ def harvest_candidate_patches( if isinstance(score, (int, float)) and isinstance(baseline_score, (int, float)): score_delta = round(float(score) - float(baseline_score), 4) + if status != "PASS": + patch_file = None + partial_patch = dest / f"task-{task_idx}" / "partial.patch.diff" results.append({ "task_index": task_idx, + "partial_patch_file": str(partial_patch) if partial_patch.is_file() else None, + "evidence_dir": str(dest / f"task-{task_idx}" / "evidence"), "score": score, "score_delta": score_delta, "status": status, diff --git a/skills/swarm/scripts/storage_uploader.py b/skills/swarm/scripts/storage_uploader.py index 411405e..b0915d6 100644 --- a/skills/swarm/scripts/storage_uploader.py +++ b/skills/swarm/scripts/storage_uploader.py @@ -256,6 +256,8 @@ def upload_run_artifacts( status: str = "UNKNOWN", candidate_files: Optional[List[str]] = None, commit_sha: Optional[str] = None, + *, + evidence_files: Optional[List[str]] = None, ) -> Dict[str, str]: repo = Path(repo_dir) clean_prefix = prefix.strip().strip("/") @@ -264,27 +266,56 @@ def upload_run_artifacts( uploaded: Dict[str, str] = {} - # ATOMIC PUBLICATION: - # Under zero-push architecture, if status == "PASS": - # Generate patch.diff and upload it to GCS FIRST (target_prefix/patch.diff). + delivery_errors = [] + proposed_pass = status == "PASS" + evidence_paths = [] + root = repo.resolve() + for relative in evidence_files or []: + path = (root / relative).resolve() + if root not in path.parents or ".git" in path.relative_to(root).parts: + delivery_errors.append(f"Evidence path outside allowed repository files: {relative}") + continue + if path.is_dir(): + evidence_paths.extend(path.rglob("*")) + elif path.is_file(): + evidence_paths.append(path) + else: + delivery_errors.append(f"Evidence file missing: {relative}") + diagnostics = root / ".agystack/diagnostics" + if diagnostics.is_dir(): + evidence_paths.extend(diagnostics.glob("*.log")) + for path in sorted(set(evidence_paths)): + resolved = path.resolve() + if not resolved.is_file() or root not in resolved.parents or ".git" in resolved.relative_to(root).parts: + continue + name = f"evidence/{path.relative_to(root).as_posix()}" + dest = f"{target_prefix}/{name}" + if upload_to_gcs(bucket_name, dest, path): + uploaded[name] = f"gs://{bucket_name}/{dest}" + else: + delivery_errors.append(f"Evidence upload failed: {name}") + if delivery_errors and proposed_pass: + status = "ISSUES" + patch_uploaded = False - if status == "PASS": - patch_path = repo / "patch.diff" - if not patch_path.is_file(): + if candidate_files or (status == "PASS" and candidate_files is None): + patch_name = "patch.diff" if status == "PASS" else "partial.patch.diff" + patch_path = repo / patch_name + if candidate_files or not patch_path.is_file(): patch_content = _generate_git_patch(repo, candidate_files=candidate_files, commit_sha=commit_sha) if patch_content: patch_path.write_text(patch_content, encoding="utf-8") if patch_path.is_file() and patch_path.stat().st_size > 0: - dest = f"{target_prefix}/patch.diff" + dest = f"{target_prefix}/{patch_name}" if upload_to_gcs(bucket_name, dest, patch_path): - uploaded["patch.diff"] = f"gs://{bucket_name}/{dest}" + uploaded[patch_name] = f"gs://{bucket_name}/{dest}" patch_uploaded = True - # If candidate_files were modified and patch.diff failed to upload or is missing/empty: - # Override status = "ISSUES"! if candidate_files and not patch_uploaded: - status = "ISSUES" + delivery_errors.append("Candidate patch failed to upload or is missing under zero-push delivery.") + if status == "PASS": + status = "ISSUES" # Upload score.json and results.tsv score_path = repo / "score.json" @@ -309,8 +340,9 @@ def upload_run_artifacts( status_data = {} status_data["task_index"] = task_index status_data["status"] = status - if candidate_files and not patch_uploaded and status == "ISSUES": - status_data["evidence"] = "Candidate patch failed to upload or is missing under zero-push delivery." + status_data["artifacts"] = uploaded + if delivery_errors: + status_data["evidence"] = "\n".join(delivery_errors) status_path.write_text(json.dumps(status_data, indent=2), encoding="utf-8") dest = f"{target_prefix}/status.json" diff --git a/skills/swarm/scripts/worker/execution.py b/skills/swarm/scripts/worker/execution.py new file mode 100644 index 0000000..8a1cce9 --- /dev/null +++ b/skills/swarm/scripts/worker/execution.py @@ -0,0 +1,189 @@ +"""Execution boundaries shared by cloud workers and dispatch.""" + +import os +import re +import signal +import subprocess +import threading +import time +import uuid +from collections.abc import Mapping +from dataclasses import dataclass, fields +from pathlib import Path + +LIMIT_ENV = { + "max_agent_turns": "AGYSTACK_MAX_AGENT_TURNS", + "command_timeout_s": "COMMAND_TIMEOUT", + "verify_timeout_s": "VERIFY_TIMEOUT", +} +TERMINAL_STATES = frozenset({"PASS", "ISSUES", "BLOCKED"}) +_UNSET = object() + + +def _positive_int(value, name): + if isinstance(value, str) and re.fullmatch(r"[0-9]+", value): + value = int(value) + if type(value) is not int or value <= 0: + raise ValueError(f"{name} must be a positive integer") + return value + + +@dataclass(frozen=True) +class ExecutionLimits: + max_agent_turns: int = 30 + command_timeout_s: int = 300 + verify_timeout_s: int = 300 + + def __post_init__(self): + for field in fields(self): + if type(getattr(self, field.name)) is not int: + raise ValueError(f"{field.name} must be a positive integer") + _positive_int(getattr(self, field.name), field.name) + + +def _limit_fields(value): + if not isinstance(value, Mapping): + raise ValueError("execution_limits must be an object") + unknown = value.keys() - LIMIT_ENV.keys() + if unknown: + raise ValueError(f"Unknown execution_limits fields: {sorted(unknown)}") + for name, raw in value.items(): + if type(raw) is not int: + raise ValueError(f"{name} must be a positive integer") + return {name: _positive_int(raw, name) for name, raw in value.items()} + + +def resolve_execution_limits(task_limits=_UNSET, *, environ=None, runtime=_UNSET, cli=_UNSET, command_default=300): + env = os.environ if environ is None else environ + values = dict(max_agent_turns=30, command_timeout_s=command_default, verify_timeout_s=300) + if runtime is not _UNSET: + values.update(_limit_fields(runtime)) + values.update({name: _positive_int(env[key], key) for name, key in LIMIT_ENV.items() if key in env}) + if cli is not _UNSET: + values.update(_limit_fields(cli)) + if task_limits is not _UNSET: + values.update(_limit_fields(task_limits)) + return ExecutionLimits(**values) + + +def limits_for_task(task_item, *, command_default=300): + overrides = task_item.get("execution_limits", {}) if isinstance(task_item, dict) else {} + return resolve_execution_limits(overrides, command_default=command_default) + + +def evaluate_agent_status(agent_text): + declarations = [] + lines = agent_text.rstrip().splitlines() + fence = None + for index, line in enumerate(lines): + stripped = line.strip() + marker = re.match(r"^(`{3,}|~{3,})", stripped) + if marker: + token = marker.group(1) + if fence is None: + fence = token + elif token[0] == fence[0] and len(token) >= len(fence) and stripped == token: + fence = None + continue + if fence is not None or line.startswith((" ", "\t")): + continue + match = re.fullmatch(r"\[STATUS: (PASS|ISSUES|BLOCKED)\]", stripped) + if match: + declarations.append((index, match.group(1))) + elif re.fullmatch(r"\[STATUS\b.*\]", stripped, re.IGNORECASE): + declarations.append((index, "ISSUES")) + if len(declarations) == 1 and declarations[0][0] == len(lines) - 1: + return declarations[0][1], agent_text + return "ISSUES", agent_text + + +CREDENTIAL_PATTERNS = [ + re.compile(r"ghp_[A-Za-z0-9_]{36,}"), + re.compile(r"AIza[0-9A-Za-z-_]{35}"), + re.compile(r"github_pat_[0-9A-Za-z_]{20,}"), + re.compile(r"ya29\.[0-9A-Za-z-_]+"), + re.compile(r"Bearer [A-Za-z0-9_\-\.]+"), +] + + +def scrub_credentials(text): + for pattern in CREDENTIAL_PATTERNS: + text = pattern.sub("[REDACTED]", text) + return text + + +def write_diagnostic(repo_dir, name, text): + directory = Path(repo_dir) / ".agystack/diagnostics" + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{name}-{uuid.uuid4().hex}.log" + path.write_text(scrub_credentials(text)[-262144:], encoding="utf-8") + return path + + +def exhaustion_result(repo_dir, limit, consumed, last_tool, last_response): + summary = ( + f"Agent turn limit exhausted: limit={limit}, consumed={consumed}, " + f"last_tool={last_tool or 'none'}, last_response={last_response[-4096:]!r}" + ) + summary = scrub_credentials(summary) + write_diagnostic(repo_dir, "agent-exhaustion", summary) + return "ISSUES", summary + + +def run_bounded_shell(command, repo_dir, timeout_s, *, label="command"): + buffers = [bytearray(), bytearray()] + shell_args = ["bash", "-o", "pipefail", "-c", command] if os.name == "posix" else command + try: + process = subprocess.Popen( + shell_args, + shell=os.name != "posix", + cwd=repo_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=os.name == "posix", + ) + except OSError as exc: + error = scrub_credentials(str(exc)) + log_path = write_diagnostic(repo_dir, label, f"Command could not start: {error}") + return subprocess.CompletedProcess(command, 127, "", error), f"Exit code 127: {error}\nLog: {log_path}" + + def drain(stream, buffer): + with stream: + while chunk := os.read(stream.fileno(), 8192): + buffer.extend(chunk) + del buffer[:-131072] + + readers = [ + threading.Thread(target=drain, args=(stream, buffer), daemon=True) + for stream, buffer in zip((process.stdout, process.stderr), buffers) + ] + for reader in readers: + reader.start() + timed_out = False + deadline = time.monotonic() + timeout_s + try: + process.wait(timeout=timeout_s) + for reader in readers: + reader.join(timeout=max(0, deadline - time.monotonic())) + if any(reader.is_alive() for reader in readers): + raise subprocess.TimeoutExpired(command, timeout_s) + except subprocess.TimeoutExpired: + timed_out = True + if os.name == "posix": + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + else: + process.kill() + process.wait() + for reader in readers: + reader.join(timeout=1) + stdout, stderr = [scrub_credentials(bytes(buffer).decode("utf-8", errors="replace")) for buffer in buffers] + if timed_out: + stderr += f"\nCommand timed out after {timeout_s} seconds." + returncode = 124 if timed_out else process.returncode + output = f"Exit code {returncode}:\n{stdout}\n{stderr}" + log_path = write_diagnostic(repo_dir, label, output) + excerpt = f"Exit code {returncode}:\n{stdout[-8192:]}\n{stderr[-8192:]}\nLog: {log_path}" + return subprocess.CompletedProcess(command, returncode, stdout, stderr), excerpt diff --git a/skills/swarm/scripts/worker/partner_engine.py b/skills/swarm/scripts/worker/partner_engine.py index b1cedd5..35fa420 100644 --- a/skills/swarm/scripts/worker/partner_engine.py +++ b/skills/swarm/scripts/worker/partner_engine.py @@ -6,12 +6,13 @@ """ import os -import subprocess import sys import time from pathlib import Path from typing import Any, Dict, List, Optional, Tuple +from .execution import ExecutionLimits, evaluate_agent_status as _evaluate_agent_status, exhaustion_result, limits_for_task, run_bounded_shell + try: from pre_tool_safety import evaluate_safety except ImportError: @@ -30,7 +31,7 @@ 1. Implement the requested task directly and surgically in the workspace. 2. Maintain high engineering rigor: run test suites and verification commands before completing. 3. Keep diffs minimal and clean. Avoid narrating comments or speculative code. -4. Conclude with a clear summary and status: you MUST include either [STATUS: PASS] or [STATUS: ISSUES] at the end of your final response. +4. End your final response with exactly one standalone unquoted line, outside code fences: [STATUS: PASS], [STATUS: ISSUES], or [STATUS: BLOCKED]. """ @@ -288,7 +289,7 @@ def _safe_resolve_repo_path(repo_dir: Path, rel_path: str) -> Optional[Path]: return None -def execute_workspace_tool(tool_name: str, tool_input: Dict[str, Any], repo_dir: Path) -> str: +def execute_workspace_tool(tool_name: str, tool_input: Dict[str, Any], repo_dir: Path, *, limits: Optional[ExecutionLimits] = None) -> str: try: if tool_name == "read_file": path_str = str(tool_input.get("path", "")).strip() @@ -352,27 +353,9 @@ def execute_workspace_tool(tool_name: str, tool_input: Dict[str, Any], repo_dir: reason = safety_res.get("reason", "Destructive command rejected.") return f"Error: Command blocked by safety policy ({safety_res.get('decision')}): {reason}" - timeout = int(os.environ.get("COMMAND_TIMEOUT", "300")) - try: - res = subprocess.run( - cmd_str, - shell=True, - cwd=repo_dir, - capture_output=True, - text=True, - timeout=timeout, - ) - output = res.stdout - if res.stderr: - if output: - output += "\n" + res.stderr - else: - output = res.stderr - return f"Exit code {res.returncode}:\n{output}" - except subprocess.TimeoutExpired: - return f"Error: command timed out after {timeout} seconds: {cmd_str}" - except Exception as e: - return f"Error executing command: {e}" + limits = limits or limits_for_task(None) + _, output = run_bounded_shell(cmd_str, repo_dir, limits.command_timeout_s) + return output else: return f"Error: unknown tool \x27{tool_name}\x27." @@ -495,7 +478,10 @@ def execute_claude_model_agent( project: Optional[str] = None, location: Optional[str] = None, messenger: Optional[Any] = None, + *, + limits: Optional[ExecutionLimits] = None, ) -> Tuple[str, str]: + limits = limits or limits_for_task(task_item) anthropic = _get_anthropic() if anthropic is None: return "BLOCKED", "Missing dependency: anthropic[vertex] Python package is not installed." @@ -527,10 +513,14 @@ def execute_claude_model_agent( messages: List[Dict[str, Any]] = [ {"role": "user", "content": task_brief} ] - max_turns = 30 + max_turns = limits.max_agent_turns + consumed = 0 + last_tool = "" last_text = "" for turn in range(max_turns): + last_text = "" + consumed += 1 response = client.messages.create( model=model, max_tokens=4096, @@ -584,7 +574,8 @@ def execute_claude_model_agent( tool_results = [] for tool_id, tool_name, tool_input in tool_uses: emit_tool_invocation(task_index, tool_name, tool_input) - result_str = execute_workspace_tool(tool_name, tool_input, repo_dir) + last_tool = tool_name + result_str = execute_workspace_tool(tool_name, tool_input, repo_dir, limits=limits) emit_tool_result(task_index, tool_name, result_str) tool_results.append({ "type": "tool_result", @@ -597,7 +588,7 @@ def execute_claude_model_agent( "content": tool_results, }) - return _evaluate_agent_status(last_text) + return exhaustion_result(repo_dir, max_turns, consumed, last_tool, last_text) except Exception as exc: return "ISSUES", f"Partner model execution exception: {exc}" @@ -611,7 +602,10 @@ def execute_fable_model_agent( project: Optional[str] = None, location: Optional[str] = None, messenger: Optional[Any] = None, + *, + limits: Optional[ExecutionLimits] = None, ) -> Tuple[str, str]: + limits = limits or limits_for_task(task_item) aiplatform = _get_aiplatform() if aiplatform is None: @@ -655,11 +649,15 @@ def execute_fable_model_agent( messages: List[Dict[str, Any]] = [ {"role": "user", "content": task_brief} ] - max_turns = 30 + max_turns = limits.max_agent_turns + consumed = 0 + last_tool = "" last_text = "" for turn in range(max_turns): + last_text = "" response = None + consumed += 1 if endpoint is not None and hasattr(endpoint, "predict"): response = endpoint.predict( instances=[{ @@ -738,7 +736,8 @@ def execute_fable_model_agent( tool_results = [] for tool_id, tool_name, tool_input in tool_uses: emit_tool_invocation(task_index, tool_name, tool_input) - result_str = execute_workspace_tool(tool_name, tool_input, repo_dir) + last_tool = tool_name + result_str = execute_workspace_tool(tool_name, tool_input, repo_dir, limits=limits) emit_tool_result(task_index, tool_name, result_str) tool_results.append({ "type": "tool_result", @@ -751,7 +750,7 @@ def execute_fable_model_agent( "content": tool_results, }) - return _evaluate_agent_status(last_text) + return exhaustion_result(repo_dir, max_turns, consumed, last_tool, last_text) except Exception as exc: return "ISSUES", f"Vertex Model Garden Fable execution exception: {exc}" @@ -765,7 +764,10 @@ def execute_deepseek_model_agent( project: Optional[str] = None, location: Optional[str] = None, messenger: Optional[Any] = None, + *, + limits: Optional[ExecutionLimits] = None, ) -> Tuple[str, str]: + limits = limits or limits_for_task(task_item) genai = _get_genai() if genai is None: return "BLOCKED", "Missing dependency: google-genai Python package is not installed." @@ -821,13 +823,21 @@ def execute_deepseek_model_agent( parts=[genai.types.Part.from_text(text=task_brief)], ) ] - max_turns = 30 + max_turns = limits.max_agent_turns + consumed = 0 + last_tool = "" last_text = "" for turn in range(max_turns): + if consumed >= max_turns: + return exhaustion_result(repo_dir, max_turns, consumed, last_tool, last_text) + last_text = "" response = None for attempt in range(5): + if consumed >= max_turns: + return exhaustion_result(repo_dir, max_turns, consumed, last_tool, last_text) try: + consumed += 1 response = client.models.generate_content( model=model, contents=contents, @@ -843,10 +853,12 @@ def execute_deepseek_model_agent( or "resource_exhausted" in err_str or "deadline" in err_str or "connection" in err_str - ) and attempt < 4: + ) and attempt < 4 and consumed < max_turns: backoff = (2 ** attempt) * 2 emit_milestone(task_index, "RETRY", f"Transient {call_exc}; retrying in {backoff}s (attempt {attempt + 1}/5)...") time.sleep(backoff) + elif consumed >= max_turns: + return exhaustion_result(repo_dir, max_turns, consumed, last_tool, str(call_exc)) else: raise @@ -882,6 +894,12 @@ def execute_deepseek_model_agent( if fc: func_calls.append(fc) + try: + response_text = getattr(response, "text", "") + last_text = response_text if isinstance(response_text, str) else "" + except Exception: + last_text = "" + if not func_calls: resp_text = None try: @@ -903,19 +921,21 @@ def execute_deepseek_model_agent( ] last_text = "\n".join(text_pieces).strip() - if not last_text and turn < max_turns - 1: + if not last_text and consumed >= max_turns: + return exhaustion_result(repo_dir, max_turns, consumed, last_tool, last_text) + if not last_text: contents.append(genai.types.Content( role="model", parts=[genai.types.Part.from_text(text="")] )) contents.append(genai.types.Content( role="user", - parts=[genai.types.Part.from_text(text="Please state your final summary and conclusion status: [STATUS: PASS] or [STATUS: ISSUES].")] + parts=[genai.types.Part.from_text(text="Give your final summary. End with exactly one standalone unquoted status line outside code fences, [STATUS: PASS], [STATUS: ISSUES], or [STATUS: BLOCKED].")] )) continue cleaned_text = re.sub(r".*?", "", last_text, flags=re.DOTALL).strip() - return _evaluate_agent_status(cleaned_text or last_text) + return _evaluate_agent_status(cleaned_text) # Filter out empty text parts so DeepSeek does not echo empty text on next turn clean_parts = [ @@ -935,7 +955,8 @@ def execute_deepseek_model_agent( except Exception: tool_input = {} emit_tool_invocation(task_index, tool_name, tool_input) - result_str = execute_workspace_tool(tool_name, tool_input, repo_dir) + last_tool = tool_name + result_str = execute_workspace_tool(tool_name, tool_input, repo_dir, limits=limits) emit_tool_result(task_index, tool_name, result_str) response_parts.append( genai.types.Part.from_function_response( @@ -946,8 +967,7 @@ def execute_deepseek_model_agent( contents.append(genai.types.Content(role="user", parts=response_parts)) - cleaned_text = re.sub(r".*?", "", last_text, flags=re.DOTALL).strip() - return _evaluate_agent_status(cleaned_text or last_text) + return exhaustion_result(repo_dir, max_turns, consumed, last_tool, last_text) except Exception as exc: return "ISSUES", f"DeepSeek model execution exception: {exc}" @@ -961,7 +981,10 @@ def execute_partner_model_agent( project: Optional[str] = None, location: Optional[str] = None, messenger: Optional[Any] = None, + *, + limits: Optional[ExecutionLimits] = None, ) -> Tuple[str, str]: + limits = limits or limits_for_task(task_item) if is_deepseek_model(model): return execute_deepseek_model_agent( model=model, @@ -972,6 +995,7 @@ def execute_partner_model_agent( project=project, location=location, messenger=messenger, + limits=limits, ) if is_fable_model(model): return execute_fable_model_agent( @@ -983,6 +1007,7 @@ def execute_partner_model_agent( project=project, location=location, messenger=messenger, + limits=limits, ) return execute_claude_model_agent( model=model, @@ -993,15 +1018,5 @@ def execute_partner_model_agent( project=project, location=location, messenger=messenger, + limits=limits, ) - - -def _evaluate_agent_status(agent_text: str) -> Tuple[str, str]: - import re - match = re.search(r"\[STATUS:\s*(PASS|ISSUES|BLOCKED|DEV_IMPROVED)\s*\]", agent_text, re.IGNORECASE) - if match: - st = match.group(1).upper() - if st == "DEV_IMPROVED": - return "PASS", agent_text - return st, agent_text - return "ISSUES", agent_text diff --git a/tests/test_architecture_remediation.py b/tests/test_architecture_remediation.py index 8f38cbe..4e1cf08 100755 --- a/tests/test_architecture_remediation.py +++ b/tests/test_architecture_remediation.py @@ -86,22 +86,22 @@ def test_cli_argument_resolution_parallelism_and_gcs_prefix(self): def test_strict_status_regex_parsing(self): # 1. Exact valid tokens - status, _ = cloud_worker.evaluate_agent_status("Finished work. [STATUS: PASS]") + status, _ = cloud_worker.evaluate_agent_status("Finished work.\n[STATUS: PASS]") self.assertEqual(status, "PASS") status, _ = cloud_worker.evaluate_agent_status("[STATUS: ISSUES]\nFound some edge cases") self.assertEqual(status, "ISSUES") - status, _ = cloud_worker.evaluate_agent_status("[STATUS: BLOCKED ] Out of quota") + status, _ = cloud_worker.evaluate_agent_status("Out of quota\n[STATUS: BLOCKED]") self.assertEqual(status, "BLOCKED") - # Case-insensitive + # Malformed declarations fail closed. status, _ = cloud_worker.evaluate_agent_status("[status: pass]") - self.assertEqual(status, "PASS") + self.assertEqual(status, "ISSUES") - # Support DEV_IMPROVED mapped to PASS + # DEV_IMPROVED does not certify completion status, _ = cloud_worker.evaluate_agent_status("[STATUS: DEV_IMPROVED]\nCandidate improved score") - self.assertEqual(status, "PASS") + self.assertEqual(status, "ISSUES") # 2. False positives without strict token must fall back to ISSUES status, _ = cloud_worker.evaluate_agent_status("I hope this passes and does not fail password test.") @@ -145,9 +145,11 @@ def test_harvest_candidate_patches(self): task1_dir.mkdir(parents=True, exist_ok=True) (task0_dir / "score.json").write_text(json.dumps({"score": 98.2, "status": "PASS"}), encoding="utf-8") + (task0_dir / "status.json").write_text(json.dumps({"status": "PASS"}), encoding="utf-8") (task0_dir / "patch.diff").write_text("--- a/foo\n+++ b/foo\n+pass", encoding="utf-8") (task1_dir / "score.json").write_text(json.dumps({"score": 45.0, "status": "ISSUES"}), encoding="utf-8") + (task1_dir / "status.json").write_text(json.dumps({"status": "ISSUES"}), encoding="utf-8") (task1_dir / "patch.diff").write_text("--- a/bar\n+++ b/bar\n+fail", encoding="utf-8") dest_dir = self.tmp_path / "downloaded_patches" @@ -390,8 +392,11 @@ def test_harvest_candidate_patches_score_delta_ranking(self): t.mkdir(parents=True, exist_ok=True) (t0 / "score.json").write_text(json.dumps({"score": 90.0, "status": "PASS"}), encoding="utf-8") + (t0 / "status.json").write_text(json.dumps({"status": "PASS"}), encoding="utf-8") (t1 / "score.json").write_text(json.dumps({"score": 95.0, "status": "PASS"}), encoding="utf-8") + (t1 / "status.json").write_text(json.dumps({"status": "PASS"}), encoding="utf-8") (t2 / "score.json").write_text(json.dumps({"score": 99.0, "status": "ISSUES"}), encoding="utf-8") + (t2 / "status.json").write_text(json.dumps({"status": "ISSUES"}), encoding="utf-8") (t3 / "status.json").write_text(json.dumps({"status": "PASS"}), encoding="utf-8") dest_dir = self.tmp_path / "harvest_delta_out" @@ -432,12 +437,14 @@ def test_harvest_session_cli_standalone(self): task_dir = mock_storage / bucket / f"swarms/{session_id}" / "task-0" task_dir.mkdir(parents=True, exist_ok=True) (task_dir / "score.json").write_text(json.dumps({"score": 92.5, "status": "PASS"}), encoding="utf-8") + (task_dir / "status.json").write_text(json.dumps({"status": "PASS"}), encoding="utf-8") (task_dir / "patch.diff").write_text("--- a/f\n+++ b/f\n+ok", encoding="utf-8") cmd = [ sys.executable, str(dispatch_script), "--harvest-session", session_id, + "--tasks", "1", "--gcs-bucket", bucket, "--baseline-score", "90.0", ] @@ -543,12 +550,15 @@ def test_result_harvester_sorts_non_numeric_and_nan_metrics(self): # Task 0: string score "N/A" (t0 / "score.json").write_text(json.dumps({"score": "N/A", "status": "PASS"}), encoding="utf-8") + (t0 / "status.json").write_text(json.dumps({"status": "PASS"}), encoding="utf-8") # Task 1: valid float score 95.0 (t1 / "score.json").write_text(json.dumps({"score": 95.0, "status": "PASS"}), encoding="utf-8") + (t1 / "status.json").write_text(json.dumps({"status": "PASS"}), encoding="utf-8") # Task 2: None score (t2 / "status.json").write_text(json.dumps({"status": "PASS"}), encoding="utf-8") # Task 3: valid float score 99.0 (t3 / "score.json").write_text(json.dumps({"score": 99.0, "status": "PASS"}), encoding="utf-8") + (t3 / "status.json").write_text(json.dumps({"status": "PASS"}), encoding="utf-8") dest_dir = self.tmp_path / "harvest_nan_out" with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(mock_storage)}): @@ -685,7 +695,9 @@ def fake_execute(*args, **kwargs): with patch("cloud_worker.execute_task", side_effect=fake_execute): import io with patch("sys.stdout", new_callable=io.StringIO) as mock_out: - cloud_worker.main() + with self.assertRaises(SystemExit) as exit_result: + cloud_worker.main() + self.assertEqual(exit_result.exception.code, 1) out = mock_out.getvalue() self.assertIn("[STATUS: ISSUES]", out) self.assertIn("Fail-closed: No GCS bucket configured", out) diff --git a/tests/test_cloud_preflight_and_worker.py b/tests/test_cloud_preflight_and_worker.py index 93d5df9..2e8fc70 100644 --- a/tests/test_cloud_preflight_and_worker.py +++ b/tests/test_cloud_preflight_and_worker.py @@ -412,7 +412,7 @@ def test_agent_mode_passes_poteto_system_prompt_to_config(self): mock_agent_instance = MagicMock() mock_resp = MagicMock() - mock_resp.text = AsyncMock(return_value="[STATUS: PASS] completed successfully") + mock_resp.text = AsyncMock(return_value="completed successfully\n[STATUS: PASS]") mock_agent_instance.chat = AsyncMock(return_value=mock_resp) mock_agent_cm = MagicMock() @@ -449,7 +449,7 @@ def test_agent_mode_env_var_overrides_poteto_system_prompt(self): mock_agent_instance = MagicMock() mock_resp = MagicMock() - mock_resp.text = AsyncMock(return_value="[STATUS: PASS] done") + mock_resp.text = AsyncMock(return_value="done\n[STATUS: PASS]") mock_agent_instance.chat = AsyncMock(return_value=mock_resp) mock_agent_cm = MagicMock() @@ -488,7 +488,7 @@ def fake_local_agent_config(**kwargs): mock_agent_instance = MagicMock() mock_resp = MagicMock() - mock_resp.text = AsyncMock(return_value="[STATUS: PASS] fallback success") + mock_resp.text = AsyncMock(return_value="fallback success\n[STATUS: PASS]") mock_agent_instance.chat = AsyncMock(return_value=mock_resp) mock_agent_cm = MagicMock() @@ -1817,18 +1817,17 @@ def post_tool_call(self, fn): def test_command_task_timeout_handling(self): with tempfile.TemporaryDirectory() as tmp_dir: repo_dir = Path(tmp_dir) - with patch.dict(os.environ, {"COMMAND_TIMEOUT": "5"}), patch( - "subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="sleep 100", timeout=5) - ): + with patch.dict(os.environ, {"COMMAND_TIMEOUT": "1"}): status, output = execute_task( - task_item={"type": "command", "command": "sleep 100"}, + task_item={"type": "command", "command": "echo started; sleep 2"}, task_brief="Run long command", repo_dir=repo_dir, model_override="flash", api_key="test-key", ) self.assertEqual(status, "ISSUES") - self.assertEqual(output, "Command timed out after 5 seconds: sleep 100") + self.assertIn("timed out after 1 seconds", output) + self.assertIn("started", output) if __name__ == "__main__": diff --git a/tests/test_cloud_worker_vertex_partner.py b/tests/test_cloud_worker_vertex_partner.py index a9b641e..8a93910 100644 --- a/tests/test_cloud_worker_vertex_partner.py +++ b/tests/test_cloud_worker_vertex_partner.py @@ -138,7 +138,7 @@ def test_adc_project_and_region_pass_through(self): mock_client = MagicMock() mock_response = MagicMock() mock_block = MagicMock() - mock_block.text = "[STATUS: PASS] Work verified." + mock_block.text = "Work verified.\n[STATUS: PASS]" mock_response.content = [mock_block] mock_client.messages.create.return_value = mock_response mock_anthropic.AnthropicVertex.return_value = mock_client @@ -164,7 +164,7 @@ def test_adc_env_var_precedence_and_us_central1_fallback(self): mock_client = MagicMock() mock_response = MagicMock() mock_block = MagicMock() - mock_block.text = "[STATUS: PASS] Success." + mock_block.text = "Success.\n[STATUS: PASS]" mock_response.content = [mock_block] mock_client.messages.create.return_value = mock_response mock_anthropic.AnthropicVertex.return_value = mock_client @@ -224,7 +224,7 @@ def test_milestone_emission_partner_model(self): mock_client = MagicMock() mock_response = MagicMock() mock_block = MagicMock() - mock_block.text = "[STATUS: PASS] Done." + mock_block.text = "Done.\n[STATUS: PASS]" mock_response.content = [mock_block] mock_client.messages.create.return_value = mock_response mock_anthropic.AnthropicVertex.return_value = mock_client @@ -249,7 +249,7 @@ def test_milestone_emission_fable_model(self): mock_aiplatform = MagicMock() mock_endpoint = MagicMock() mock_resp = MagicMock() - mock_resp.predictions = ["[STATUS: PASS] Done."] + mock_resp.predictions = ["Done.\n[STATUS: PASS]"] mock_endpoint.predict.return_value = mock_resp mock_aiplatform.Endpoint.return_value = mock_endpoint @@ -273,14 +273,14 @@ def test_milestone_emission_deepseek_model(self): mock_genai = MagicMock() mock_client = MagicMock() mock_resp = MagicMock() - mock_resp.text = "[STATUS: PASS] DeepSeek done." + mock_resp.text = "DeepSeek done.\n[STATUS: PASS]" mock_resp.function_calls = [] mock_resp.candidates = [ MagicMock( content=MagicMock( parts=[ MagicMock( - text="[STATUS: PASS] DeepSeek done.", + text="DeepSeek done.\n[STATUS: PASS]", thought=None, function_call=None, ) @@ -336,7 +336,7 @@ def test_autonomous_tool_calling_loop_writes_files_and_passes(self): resp1.content = [block_tool2] block_final = MagicMock() - block_final.text = "[STATUS: PASS] Solution implemented and tests pass." + block_final.text = "Solution implemented and tests pass.\n[STATUS: PASS]" resp2 = MagicMock() resp2.content = [block_final] @@ -414,7 +414,7 @@ def fake_content(role=None, parts=None): # Turn 1: DeepSeek finishes with PASS part_final = MagicMock() part_final.thought = None - part_final.text = "[STATUS: PASS] DeepSeek solution implemented." + part_final.text = "DeepSeek solution implemented.\n[STATUS: PASS]" part_final.function_call = None resp1 = MagicMock() @@ -422,7 +422,7 @@ def fake_content(role=None, parts=None): cand1 = MagicMock() cand1.content = MagicMock(parts=[part_final]) resp1.candidates = [cand1] - resp1.text = "[STATUS: PASS] DeepSeek solution implemented." + resp1.text = "DeepSeek solution implemented.\n[STATUS: PASS]" mock_client.models.generate_content.side_effect = [resp0, resp1] mock_genai.Client.return_value = mock_client @@ -485,7 +485,7 @@ def test_autonomous_tool_calling_loop_edit_and_read_file(self): resp1.content = [block_edit] block_final = MagicMock() - block_final.text = "[STATUS: PASS] Readme edited." + block_final.text = "Readme edited.\n[STATUS: PASS]" resp2 = MagicMock() resp2.content = [block_final] @@ -532,7 +532,7 @@ def test_mixed_manifest_picks_correct_client(self): mock_ag.policy.allow_all = MagicMock() mock_agent_instance = MagicMock() mock_ag_resp = MagicMock() - mock_ag_resp.text = AsyncMock(return_value="[STATUS: PASS] Gemini agent done") + mock_ag_resp.text = AsyncMock(return_value="Gemini agent done\n[STATUS: PASS]") mock_agent_instance.chat = AsyncMock(return_value=mock_ag_resp) mock_agent_cm = MagicMock() mock_agent_cm.__aenter__ = AsyncMock(return_value=mock_agent_instance) @@ -543,7 +543,7 @@ def test_mixed_manifest_picks_correct_client(self): mock_claude_client = MagicMock() mock_claude_resp = MagicMock() mock_claude_block = MagicMock() - mock_claude_block.text = "[STATUS: PASS] Claude agent done" + mock_claude_block.text = "Claude agent done\n[STATUS: PASS]" mock_claude_resp.content = [mock_claude_block] mock_claude_client.messages.create.return_value = mock_claude_resp mock_anthropic.AnthropicVertex.return_value = mock_claude_client @@ -551,7 +551,7 @@ def test_mixed_manifest_picks_correct_client(self): mock_aiplatform = MagicMock() mock_fable_endpoint = MagicMock() mock_fable_resp = MagicMock() - mock_fable_resp.predictions = ["[STATUS: PASS] Fable agent done"] + mock_fable_resp.predictions = ["Fable agent done\n[STATUS: PASS]"] mock_fable_endpoint.predict.return_value = mock_fable_resp mock_aiplatform.Endpoint.return_value = mock_fable_endpoint @@ -641,7 +641,7 @@ def test_worker_task_manifest_deepseek_enables_vertex_and_global_location(self): mock_genai = MagicMock() mock_client = MagicMock() mock_resp = MagicMock() - mock_resp.text = "[STATUS: PASS] DeepSeek task done" + mock_resp.text = "DeepSeek task done\n[STATUS: PASS]" mock_resp.function_calls = [] mock_resp.candidates = [] mock_client.models.generate_content.return_value = mock_resp diff --git a/tests/test_worker_execution_lifecycle.py b/tests/test_worker_execution_lifecycle.py index 94ceeac..5eda680 100644 --- a/tests/test_worker_execution_lifecycle.py +++ b/tests/test_worker_execution_lifecycle.py @@ -19,22 +19,28 @@ from worker import partner_engine -@pytest.mark.parametrize("text, expected", [ - ("Reporting `[STATUS: PASS]` now would be false. The correct status is [STATUS: BLOCKED]", "ISSUES"), - ("Deprecated `[STATUS: PASS]`\n[STATUS: BLOCKED]", "BLOCKED"), - ("Finished.\n[STATUS: PASS]", "PASS"), - ("[STATUS: PASS]\n[STATUS: BLOCKED]", "ISSUES"), - ("[STATUS: PASS]\n[STATUS: PASS]", "ISSUES"), - ("```text\n[STATUS: PASS]\n```", "ISSUES"), - ("~~~\n[STATUS: PASS]\n~~~\n[STATUS: BLOCKED]", "BLOCKED"), - ("> [STATUS: PASS]", "ISSUES"), - ('"[STATUS: PASS]"', "ISSUES"), - ("`[STATUS: PASS]`", "ISSUES"), - ("[STATUS: PASS] trailing prose", "ISSUES"), - ("[STATUS: DEV_IMPROVED]", "ISSUES"), - ("[STATUS: PASS]\nMore prose", "ISSUES"), - ("[STATUS: MAYBE]", "ISSUES"), -]) +@pytest.mark.parametrize( + "text, expected", + [ + ("Reporting `[STATUS: PASS]` now would be false. The correct status is [STATUS: BLOCKED]", "ISSUES"), + ("Deprecated `[STATUS: PASS]`\n[STATUS: BLOCKED]", "BLOCKED"), + ("Finished.\n[STATUS: PASS]", "PASS"), + ("[STATUS: PASS]\n[STATUS: BLOCKED]", "ISSUES"), + ("[STATUS: PASS]\n[STATUS: PASS]", "ISSUES"), + ("```text\n[STATUS: PASS]\n```", "ISSUES"), + ("~~~\n[STATUS: PASS]\n~~~\n[STATUS: BLOCKED]", "BLOCKED"), + ("> [STATUS: PASS]", "ISSUES"), + ('"[STATUS: PASS]"', "ISSUES"), + ("`[STATUS: PASS]`", "ISSUES"), + ("[STATUS: PASS] trailing prose", "ISSUES"), + ("[STATUS: DEV_IMPROVED]", "ISSUES"), + ("[STATUS: PASS]\nMore prose", "ISSUES"), + ("[STATUS: MAYBE]", "ISSUES"), + ("[status: issues]\n[STATUS: PASS]", "ISSUES"), + ("[STATUS PASS]\n[STATUS: PASS]", "ISSUES"), + (" [STATUS: PASS]", "ISSUES"), + ], +) def test_strict_terminal_status(text, expected): for evaluate in (cloud_worker.evaluate_agent_status, partner_engine._evaluate_agent_status): assert evaluate(text)[0] == expected @@ -43,6 +49,7 @@ def test_strict_terminal_status(text, expected): @pytest.mark.parametrize("bad", [0, -1, True, 1.5, "1.5", "oops", None]) def test_invalid_execution_limits(bad): from worker.execution import resolve_execution_limits + with pytest.raises(ValueError): resolve_execution_limits({"max_agent_turns": bad}, environ={}) @@ -50,6 +57,7 @@ def test_invalid_execution_limits(bad): def test_limits_precedence_and_immutability(): from dataclasses import FrozenInstanceError from worker.execution import resolve_execution_limits + limits = resolve_execution_limits( {"verify_timeout_s": 41}, runtime={"max_agent_turns": 31, "command_timeout_s": 500}, @@ -63,13 +71,16 @@ def test_limits_precedence_and_immutability(): resolve_execution_limits({"unexpected": 5}, environ={}) with pytest.raises(ValueError): resolve_execution_limits([], environ={}) + with pytest.raises(ValueError): + resolve_execution_limits(runtime=None, environ={}) assert resolve_execution_limits(environ={}).command_timeout_s == 300 assert resolve_execution_limits(environ={}, command_default=900).command_timeout_s == 900 @pytest.mark.parametrize("provider", ["claude", "fable", "deepseek"]) @pytest.mark.parametrize("finish", [True, False]) -def test_provider_turn_limits_and_final_response(tmp_path, provider, finish): +@pytest.mark.parametrize("source", ["task", "keyword"]) +def test_provider_turn_limits_and_final_response(tmp_path, provider, finish, source): (tmp_path / "note.txt").write_text("still working") tool = {"type": "tool_use", "id": "read", "name": "read_file", "input": {"path": "note.txt"}} responses = [] @@ -96,7 +107,20 @@ def test_provider_turn_limits_and_final_response(tmp_path, provider, finish): getter = "_get_genai" request.side_effect = responses with patch.object(partner_engine, getter, return_value=sdk): - status, summary = entry(provider, "work", tmp_path, task_item={"execution_limits": {"max_agent_turns": 33}}) + if source == "task": + status, summary = entry(provider, "work", tmp_path, task_item={"execution_limits": {"max_agent_turns": 33}}) + else: + from worker.execution import ExecutionLimits + + model = {"claude": "claude-3-opus@20240229", "fable": "fable-5-1", "deepseek": "deepseek"}[provider] + status, summary = cloud_worker.execute_task( + {}, + "work", + tmp_path, + model, + "", + limits=ExecutionLimits(max_agent_turns=33), + ) assert request.call_count == 33 assert status == ("PASS" if finish else "ISSUES") if not finish: @@ -106,12 +130,18 @@ def test_provider_turn_limits_and_final_response(tmp_path, provider, finish): assert "last_response" in summary -def test_timeout_keeps_output_and_kills_descendants(tmp_path): +@pytest.mark.parametrize("suffix", ["& wait", "| cat", "&"]) +def test_timeout_keeps_output_and_kills_descendants(tmp_path, suffix): from worker.execution import ExecutionLimits - child = "import time; time.sleep(2); open('survived', 'w').write('bad')" - command = f"echo partial-output; echo partial-error >&2; {shlex.quote(sys.executable)} -c {shlex.quote(child)} & wait" + + child = "import os,time; open('child.pid','w').write(str(os.getpid())); time.sleep(2); open('survived', 'w').write('bad')" + command = ( + f"echo partial-output; echo partial-error >&2; {shlex.quote(sys.executable)} -c {shlex.quote(child)} {suffix}" + ) output = partner_engine.execute_workspace_tool( - "run_command", {"command": command}, tmp_path, + "run_command", + {"command": command}, + tmp_path, limits=ExecutionLimits(command_timeout_s=1), ) assert "timed out" in output.lower() @@ -119,14 +149,15 @@ def test_timeout_keeps_output_and_kills_descendants(tmp_path): assert "partial-error" in output logs = list((tmp_path / ".agystack/diagnostics").glob("*.log")) assert logs and "partial-output" in logs[0].read_text() + child_pid = int((tmp_path / "child.pid").read_text()) time.sleep(1.5) assert not (tmp_path / "survived").exists() + state = subprocess.run(["ps", "-o", "stat=", "-p", str(child_pid)], capture_output=True, text=True) + assert not state.stdout.strip() or state.stdout.strip().startswith("Z") def test_command_pipeline_failure_is_not_hidden(tmp_path): - status, _ = cloud_worker.execute_task( - {"type": "command", "command": "false | tail -n 1"}, "", tmp_path, "", "" - ) + status, _ = cloud_worker.execute_task({"type": "command", "command": "false | tail -n 1"}, "", tmp_path, "", "") assert status == "ISSUES" @@ -142,11 +173,26 @@ def worker_repo(tmp_path): return repo -@pytest.mark.parametrize("status, candidate", [("BLOCKED", True), ("ISSUES", True), ("PASS", False)]) -def test_worker_finalization_preserves_evidence(worker_repo, tmp_path, monkeypatch, status, candidate): +@pytest.mark.parametrize( + "status, candidate, verify", + [ + ("BLOCKED", True, "exit 1"), + ("ISSUES", True, "exit 1"), + ("PASS", False, "exit 1"), + ("PASS", False, "exit 0"), + ("PASS", True, "exit 1"), + ("PASS", False, "sleep 2"), + ], +) +def test_worker_finalization_preserves_evidence(worker_repo, tmp_path, monkeypatch, status, candidate, verify, capsys): storage = tmp_path / "storage" - task = {"brief": "work", "evidence_files": ["results/proof.txt"], - "verify_command": "echo verifier-ran > verification.txt; exit 1"} + task = { + "brief": "work", + "candidate_files": ["service.py"], + "evidence_files": ["results/proof.txt"], + "verify_command": f"echo verifier-ran > verification.txt; {verify}", + "execution_limits": {"verify_timeout_s": 1}, + } monkeypatch.setenv("REPO_URL", "https://example.com/repo") monkeypatch.setenv("TASK_MANIFEST", json.dumps([task])) monkeypatch.setenv("USE_VERTEX_AI", "1") @@ -155,6 +201,7 @@ def test_worker_finalization_preserves_evidence(worker_repo, tmp_path, monkeypat monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(storage)) monkeypatch.chdir(tmp_path) monkeypatch.setattr(cloud_worker.tempfile, "gettempdir", lambda: str(tmp_path)) + def clone(**kwargs): repo = kwargs["repo_dir"] subprocess.run(["git", "init"], cwd=repo, check=True, capture_output=True) @@ -163,6 +210,7 @@ def clone(**kwargs): (repo / "service.py").write_text("value = 1\n") subprocess.run(["git", "add", "."], cwd=repo, check=True) subprocess.run(["git", "commit", "-m", "base"], cwd=repo, check=True, capture_output=True) + def execute(**kwargs): repo = kwargs["repo_dir"] if candidate: @@ -170,6 +218,7 @@ def execute(**kwargs): (repo / "results").mkdir() (repo / "results/proof.txt").write_text("partial proof") return status, "work summary" + monkeypatch.setattr(cloud_worker, "clone_and_checkout_task", clone) monkeypatch.setattr(cloud_worker, "execute_task", execute) monkeypatch.setattr(cloud_worker, "StorageMessenger", None) @@ -179,13 +228,17 @@ def execute(**kwargs): assert exc.code == 1 target = storage / "test-bucket/run/task-0" published = json.loads((target / "status.json").read_text()) - assert published["status"] == ("ISSUES" if status == "PASS" else status) + expected = "PASS" if status == "PASS" and verify == "exit 0" else "ISSUES" if status == "PASS" else status + assert published["status"] == expected + report = cloud_dispatch.parse_worker_logs(capsys.readouterr().out) + assert len(report) == 1 and report[0]["status"] == expected and report[0]["task_index"] == 0 assert (target / "evidence/results/proof.txt").read_text() == "partial proof" assert not (target / "patch.diff").exists() if candidate: assert "+value = 2" in (target / "partial.patch.diff").read_text() - else: + if status == "PASS": assert (worker_repo / "verification.txt").is_file() + assert list((target / "evidence/.agystack/diagnostics").glob("verification-*.log")) harvested = result_harvester.harvest_candidate_patches("test-bucket", "run", tmp_path / "harvest") assert len(harvested) == 1 assert harvested[0]["status"] == published["status"] @@ -193,14 +246,241 @@ def execute(**kwargs): assert (tmp_path / "harvest/task-0/evidence/results/proof.txt").is_file() -@pytest.mark.parametrize("reports", [[], [{"task_index": 0, "status": "PASS"}], - [{"task_index": 0, "status": "PASS"}, {"task_index": 1, "status": "UNKNOWN"}], - [{"task_index": 0, "status": "PASS"}, {"task_index": 0, "status": "PASS"}]]) +@pytest.mark.parametrize( + "reports", + [ + [], + [{"task_index": 0, "status": "PASS"}], + [{"task_index": 0, "status": "PASS"}, {"task_index": 1, "status": "UNKNOWN"}], + [{"task_index": 0, "status": "PASS"}, {"task_index": 0, "status": "PASS"}], + ], +) def test_dispatch_requires_complete_terminal_coverage(reports): assert cloud_dispatch.failed_task_count(reports, 2) > 0 def test_dispatch_complete_pass(): - assert cloud_dispatch.failed_task_count([ - {"task_index": 0, "status": "PASS"}, {"task_index": 1, "status": "PASS"} - ], 2) == 0 + assert ( + cloud_dispatch.failed_task_count([{"task_index": 0, "status": "PASS"}, {"task_index": 1, "status": "PASS"}], 2) + == 0 + ) + + +@pytest.mark.parametrize("limits", [None, [], {"extra": 3}, {"max_agent_turns": "40"}, {"max_agent_turns": True}]) +def test_dispatch_rejects_invalid_task_limits_before_launch(tmp_path, limits): + command = [ + sys.executable, + str(SCRIPTS / "cloud_dispatch.py"), + "--dry-run", + "--repo", + "https://example.com/repo", + "--vertex", + "--project", + "test", + "--manifest", + json.dumps([{"brief": "work", "execution_limits": limits}]), + ] + result = subprocess.run(command, cwd=tmp_path, capture_output=True, text=True) + assert result.returncode != 0 + assert "execution_limits" in result.stderr or "positive integer" in result.stderr + + +def test_dispatch_cli_env_runtime_precedence(tmp_path): + (tmp_path / "agystack-runtime.json").write_text( + json.dumps({"execution_limits": {"max_agent_turns": 40, "command_timeout_s": 500, "verify_timeout_s": 501}}) + ) + env = dict(os.environ, AGYSTACK_MAX_AGENT_TURNS="41", COMMAND_TIMEOUT="600") + env.pop("VERIFY_TIMEOUT", None) + result = subprocess.run( + [ + sys.executable, + str(SCRIPTS / "cloud_dispatch.py"), + "--dry-run", + "--repo", + "https://example.com/repo", + "--vertex", + "--project", + "test", + "--max-agent-turns", + "42", + "--verify-timeout", + "601", + "--manifest", + json.dumps([{"brief": "work", "execution_limits": {"max_agent_turns": 43}}]), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(result.stdout) + assert payload["env_vars"]["AGYSTACK_MAX_AGENT_TURNS"] == "42" + assert payload["env_vars"]["COMMAND_TIMEOUT"] == "600" + assert payload["env_vars"]["VERIFY_TIMEOUT"] == "601" + + +def test_deepseek_retries_consume_request_limit(tmp_path): + sdk = MagicMock() + request = sdk.Client.return_value.models.generate_content + request.side_effect = RuntimeError("503 unavailable") + with patch.object(partner_engine, "_get_genai", return_value=sdk), patch.object(partner_engine.time, "sleep"): + status, summary = partner_engine.execute_deepseek_model_agent( + "deepseek", + "work", + tmp_path, + task_item={"execution_limits": {"max_agent_turns": 2}}, + ) + assert request.call_count == 2 + assert status == "ISSUES" + assert "limit=2" in summary and "consumed=2" in summary + + +@pytest.mark.parametrize("provider", ["claude", "fable"]) +def test_empty_final_response_cannot_reuse_previous_pass(tmp_path, provider): + sdk = MagicMock() + tool = {"type": "tool_use", "id": "read", "name": "read_file", "input": {"path": "absent.txt"}} + responses = [ + SimpleNamespace(content=[{"type": "text", "text": "[STATUS: PASS]"}, tool]), + SimpleNamespace(content=[]), + ] + if provider == "claude": + sdk.AnthropicVertex.return_value.messages.create.side_effect = responses + entry, getter = partner_engine.execute_claude_model_agent, "_get_anthropic" + else: + sdk.Endpoint.return_value.predict.side_effect = responses + entry, getter = partner_engine.execute_fable_model_agent, "_get_aiplatform" + with patch.object(partner_engine, getter, return_value=sdk): + assert entry(provider, "work", tmp_path)[0] == "ISSUES" + + +def test_evidence_failure_downgrades_before_status_publication(worker_repo): + (worker_repo / "service.py").write_text("value = 2\n") + (worker_repo / "proof.txt").write_text("proof") + published = [] + + def upload(bucket, name, path): + published.append((name, path.read_text())) + return not name.endswith("proof.txt") + + with patch.object(storage_uploader, "upload_to_gcs", side_effect=upload): + artifacts = storage_uploader.upload_run_artifacts( + "bucket", + "run", + worker_repo, + 0, + "PASS", + ["service.py"], + evidence_files=["proof.txt"], + ) + assert published[-1][0].endswith("/status.json") + assert json.loads(published[-1][1])["status"] == "ISSUES" + assert "patch.diff" not in artifacts + assert "partial.patch.diff" in artifacts + + +def test_missing_status_file_never_promotes_score_or_patch(tmp_path, monkeypatch): + source = tmp_path / "storage/bucket/run/task-0" + source.mkdir(parents=True) + (source / "score.json").write_text(json.dumps({"score": 999, "status": "PASS"})) + (source / "patch.diff").write_text("candidate") + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(tmp_path / "storage")) + results = result_harvester.harvest_candidate_patches("bucket", "run", tmp_path / "harvest") + assert results[0]["status"] == "UNKNOWN" + assert results[0]["patch_file"] is None + + +def test_shell_output_is_bounded_and_credentials_scrubbed(tmp_path): + from worker.execution import run_bounded_shell + + secret = "AIza" + "a" * 35 + script = f"print('x' * 400000); print('{secret}'); print('last-line')" + result, output = run_bounded_shell(f"{shlex.quote(sys.executable)} -c {shlex.quote(script)}", tmp_path, 5) + assert result.returncode == 0 + assert len(output) < 17000 + log = next((tmp_path / ".agystack/diagnostics").glob("*.log")) + assert log.stat().st_size < 263000 + assert secret not in output and secret not in log.read_text() + assert "[REDACTED]" in output and "last-line" in output + + +@pytest.mark.parametrize("mode", ["complete", "missing", "unknown"]) +def test_attach_requires_all_worker_statuses(tmp_path, monkeypatch, mode, capsys): + storage = tmp_path / "storage" + for index in range(2 if mode != "missing" else 1): + directory = storage / f"bucket/swarms/run/task-{index}" + directory.mkdir(parents=True) + status = "UNKNOWN" if mode == "unknown" and index == 1 else "PASS" + (directory / "status.json").write_text(json.dumps({"task_index": index, "status": status})) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(storage)) + monkeypatch.setattr( + sys, + "argv", + ["cloud_dispatch.py", "--wait-execution", "execution", "--session", "run", "--gcs-bucket", "bucket"], + ) + monkeypatch.setattr(cloud_dispatch, "wait_for_execution", lambda **kwargs: ({"spec": {"taskCount": 2}}, True, None)) + monkeypatch.setenv("ANTIGRAVITY_APP_DATA_DIR", str(tmp_path / "app-data")) + if mode == "complete": + cloud_dispatch.main() + else: + with pytest.raises(SystemExit) as exc: + cloud_dispatch.main() + assert exc.value.code == 1 + + +def test_sdk_harvest_downloads_nested_partial_evidence(tmp_path, monkeypatch): + payloads = { + "run/task-0/status.json": json.dumps({"status": "BLOCKED"}), + "run/task-0/partial.patch.diff": "partial diff", + "run/task-0/evidence/results/proof.txt": "proof", + "run/task-0/evidence/.agystack/diagnostics/timeout.log": "partial output", + } + + class Blob: + def __init__(self, name, payload): + self.name, self.payload = name, payload + + def download_to_filename(self, filename): + Path(filename).write_text(self.payload) + + storage = MagicMock() + storage.Client.return_value.bucket.return_value.list_blobs.return_value = [ + Blob(name, value) for name, value in payloads.items() + ] + monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) + with ( + patch.dict(sys.modules, {"google.cloud.storage": storage}), + patch("google.cloud.storage", storage, create=True), + ): + results = result_harvester.harvest_candidate_patches("bucket", "run", tmp_path / "harvest") + assert results[0]["status"] == "BLOCKED" + assert results[0]["patch_file"] is None + assert Path(results[0]["partial_patch_file"]).read_text() == "partial diff" + assert (tmp_path / "harvest/task-0/evidence/results/proof.txt").read_text() == "proof" + assert (tmp_path / "harvest/task-0/evidence/.agystack/diagnostics/timeout.log").read_text() == "partial output" + + +@pytest.mark.parametrize("expected_count", [None, 2]) +def test_standalone_harvest_quarantines_incomplete_coverage(tmp_path, monkeypatch, expected_count): + source = tmp_path / "storage/bucket/run/task-0" + source.mkdir(parents=True) + (source / "status.json").write_text(json.dumps({"status": "PASS"})) + (source / "patch.diff").write_text("candidate") + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(tmp_path / "storage")) + command = [ + sys.executable, + str(SCRIPTS / "cloud_dispatch.py"), + "--harvest-session", + "run", + "--gcs-prefix", + "run", + "--gcs-bucket", + "bucket", + ] + if expected_count: + command += ["--tasks", str(expected_count)] + result = subprocess.run(command, cwd=tmp_path, capture_output=True, text=True) + assert result.returncode == 1 + assert "Quarantined Candidate" in result.stdout + assert "Winning Candidate" not in result.stdout From bd521a33e75e564f39c024996713e4b05b6e2cda Mon Sep 17 00:00:00 2001 From: jtaroreh Date: Sat, 24 Oct 2026 16:01:19 -0700 Subject: [PATCH 03/11] Remove narrative comments from worker repair --- skills/swarm/scripts/worker/execution.py | 1 - tests/test_architecture_remediation.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/skills/swarm/scripts/worker/execution.py b/skills/swarm/scripts/worker/execution.py index 8a1cce9..1011f7e 100644 --- a/skills/swarm/scripts/worker/execution.py +++ b/skills/swarm/scripts/worker/execution.py @@ -1,4 +1,3 @@ -"""Execution boundaries shared by cloud workers and dispatch.""" import os import re diff --git a/tests/test_architecture_remediation.py b/tests/test_architecture_remediation.py index 4e1cf08..9a8eee9 100755 --- a/tests/test_architecture_remediation.py +++ b/tests/test_architecture_remediation.py @@ -95,11 +95,9 @@ def test_strict_status_regex_parsing(self): status, _ = cloud_worker.evaluate_agent_status("Out of quota\n[STATUS: BLOCKED]") self.assertEqual(status, "BLOCKED") - # Malformed declarations fail closed. status, _ = cloud_worker.evaluate_agent_status("[status: pass]") self.assertEqual(status, "ISSUES") - # DEV_IMPROVED does not certify completion status, _ = cloud_worker.evaluate_agent_status("[STATUS: DEV_IMPROVED]\nCandidate improved score") self.assertEqual(status, "ISSUES") From 9d873ca0861943efc618da123e508d052076c47a Mon Sep 17 00:00:00 2001 From: jtaroreh Date: Sat, 24 Oct 2026 22:45:11 -0700 Subject: [PATCH 04/11] Test: Reproduce evidence credential leaks, result upload drops, and harvester prefix collisions --- tests/test_storage_uploader.py | 145 +++++++++++++++++++++++ tests/test_worker_execution_lifecycle.py | 87 +++++++++++++- 2 files changed, 229 insertions(+), 3 deletions(-) diff --git a/tests/test_storage_uploader.py b/tests/test_storage_uploader.py index 09ca080..7dfcf50 100644 --- a/tests/test_storage_uploader.py +++ b/tests/test_storage_uploader.py @@ -198,5 +198,150 @@ def test_parse_score_metrics_missing(self): self.assertIsNone(latency_ms) +class TestEvidenceCredentialSafeguards(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.repo_dir = Path(self.temp_dir.name) + subprocess.run(["git", "init"], cwd=self.repo_dir, capture_output=True, check=True) + subprocess.run(["git", "config", "user.name", "Tester"], cwd=self.repo_dir, capture_output=True, check=True) + subprocess.run(["git", "config", "user.email", "test@test.local"], cwd=self.repo_dir, capture_output=True, check=True) + (self.repo_dir / "app.py").write_text("print('hello')\n", encoding="utf-8") + subprocess.run(["git", "add", "app.py"], cwd=self.repo_dir, capture_output=True, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=self.repo_dir, capture_output=True, check=True) + + def tearDown(self): + self.temp_dir.cleanup() + + @patch("storage_uploader.upload_to_gcs") + def test_evidence_with_credentials_rejected_and_degrades_pass(self, mock_upload): + mock_upload.return_value = True + + evidence_file = self.repo_dir / "proof.txt" + secret_token = "ghp_" + "a" * 36 + evidence_file.write_text(f"run log with sensitive token: {secret_token}\n", encoding="utf-8") + + results = storage_uploader.upload_run_artifacts( + bucket_name="test-bucket", + prefix="runs/exp1", + repo_dir=self.repo_dir, + task_index=0, + status="PASS", + evidence_files=["proof.txt"], + ) + + # File content must NOT have been modified + self.assertIn(secret_token, evidence_file.read_text(encoding="utf-8")) + # Evidence file must not be in uploaded artifacts + self.assertNotIn("evidence/proof.txt", results) + + # status.json locally must degrade from PASS to ISSUES + status_file = self.repo_dir / "status.json" + self.assertTrue(status_file.is_file()) + status_data = json.loads(status_file.read_text(encoding="utf-8")) + self.assertEqual(status_data["status"], "ISSUES") + self.assertIn("credentials", status_data.get("evidence", "")) + + @patch("storage_uploader.upload_to_gcs") + def test_sensitive_filenames_rejected(self, mock_upload): + mock_upload.return_value = True + + sensitive_names = [ + ".env", + "server.pem", + "private.key", + "id_rsa", + "id_ed25519", + "credentials.json", + "token.json", + ] + for name in sensitive_names: + file_path = self.repo_dir / name + file_path.write_text("dummy sensitive data\n", encoding="utf-8") + self.assertTrue( + storage_uploader.is_sensitive_path(file_path, self.repo_dir), + f"{name} should be recognized as sensitive path", + ) + + # Also verify symlink is recognized as sensitive path + target_file = self.repo_dir / "target.txt" + target_file.write_text("target", encoding="utf-8") + symlink_file = self.repo_dir / "symlink_file.txt" + symlink_file.symlink_to(target_file) + self.assertTrue( + storage_uploader.is_sensitive_path(symlink_file, self.repo_dir), + "Symlink should be recognized as sensitive path", + ) + + # When passed as evidence_files, all sensitive files must be rejected and status degraded + results = storage_uploader.upload_run_artifacts( + bucket_name="test-bucket", + prefix="runs/exp1", + repo_dir=self.repo_dir, + task_index=0, + status="PASS", + evidence_files=sensitive_names + ["symlink_file.txt"], + ) + + for name in sensitive_names: + self.assertNotIn(f"evidence/{name}", results) + self.assertNotIn("evidence/symlink_file.txt", results) + + status_file = self.repo_dir / "status.json" + status_data = json.loads(status_file.read_text(encoding="utf-8")) + self.assertEqual(status_data["status"], "ISSUES") + + @patch("storage_uploader.upload_to_gcs") + def test_failed_score_or_results_upload_degrades_pass_to_issues(self, mock_upload): + score_file = self.repo_dir / "score.json" + score_file.write_text(json.dumps({"score": 0.95}), encoding="utf-8") + + tsv_file = self.repo_dir / "results.tsv" + tsv_file.write_text("metric\tvalue\nscore\t0.95\n", encoding="utf-8") + + # 1. Test score.json upload failure + def fail_score_upload(bkt, dest, path): + if "score.json" in str(dest): + return False + return True + + mock_upload.side_effect = fail_score_upload + + results = storage_uploader.upload_run_artifacts( + bucket_name="test-bucket", + prefix="runs/exp1", + repo_dir=self.repo_dir, + task_index=0, + status="PASS", + ) + + self.assertNotIn("score.json", results) + status_file = self.repo_dir / "status.json" + status_data = json.loads(status_file.read_text(encoding="utf-8")) + self.assertEqual(status_data["status"], "ISSUES") + self.assertIn("score.json upload failed", status_data.get("evidence", "")) + + # 2. Test results.tsv upload failure + def fail_results_upload(bkt, dest, path): + if "results.tsv" in str(dest): + return False + return True + + mock_upload.side_effect = fail_results_upload + + results = storage_uploader.upload_run_artifacts( + bucket_name="test-bucket", + prefix="runs/exp1", + repo_dir=self.repo_dir, + task_index=0, + status="PASS", + ) + + self.assertNotIn("results.tsv", results) + status_data = json.loads(status_file.read_text(encoding="utf-8")) + self.assertEqual(status_data["status"], "ISSUES") + self.assertIn("results.tsv upload failed", status_data.get("evidence", "")) + + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_worker_execution_lifecycle.py b/tests/test_worker_execution_lifecycle.py index 5eda680..0186e01 100644 --- a/tests/test_worker_execution_lifecycle.py +++ b/tests/test_worker_execution_lifecycle.py @@ -4,6 +4,7 @@ import subprocess import sys import time +import types from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -449,9 +450,17 @@ def download_to_filename(self, filename): Blob(name, value) for name, value in payloads.items() ] monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) - with ( - patch.dict(sys.modules, {"google.cloud.storage": storage}), - patch("google.cloud.storage", storage, create=True), + google_mod = types.ModuleType("google") + cloud_mod = types.ModuleType("google.cloud") + google_mod.cloud = cloud_mod + cloud_mod.storage = storage + with patch.dict( + sys.modules, + { + "google": google_mod, + "google.cloud": cloud_mod, + "google.cloud.storage": storage, + }, ): results = result_harvester.harvest_candidate_patches("bucket", "run", tmp_path / "harvest") assert results[0]["status"] == "BLOCKED" @@ -484,3 +493,75 @@ def test_standalone_harvest_quarantines_incomplete_coverage(tmp_path, monkeypatc assert result.returncode == 1 assert "Quarantined Candidate" in result.stdout assert "Winning Candidate" not in result.stdout + + +def test_storage_mock_hierarchy_without_google_module(tmp_path, monkeypatch): + storage = MagicMock() + google_mod = types.ModuleType("google") + cloud_mod = types.ModuleType("google.cloud") + google_mod.cloud = cloud_mod + cloud_mod.storage = storage + + monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) + clean_modules = {k: v for k, v in sys.modules.items() if not k.startswith("google")} + clean_modules["google"] = google_mod + clean_modules["google.cloud"] = cloud_mod + clean_modules["google.cloud.storage"] = storage + + with patch.dict(sys.modules, clean_modules, clear=True): + from google.cloud import storage as resolved_storage + + assert resolved_storage is storage + storage.Client.return_value.bucket.return_value.list_blobs.return_value = [] + res = result_harvester.harvest_candidate_patches("bkt", "prefix", tmp_path / "dest") + assert res == [] + + +def test_harvest_candidate_patches_strict_prefix_matching(tmp_path, monkeypatch): + payloads = { + "run/task-0/status.json": json.dumps({"status": "PASS"}), + "run/task-0/score.json": json.dumps({"score": 0.95}), + "run/task-0/patch.diff": "valid patch", + # Sibling prefix that starts with 'run' but is not 'run/' + "run-other/task-1/status.json": json.dumps({"status": "PASS"}), + "run-other/task-1/score.json": json.dumps({"score": 0.99}), + "run-other/task-1/patch.diff": "invalid patch", + # Invalid task naming inside 'run/' + "run/mytask-2/status.json": json.dumps({"status": "PASS"}), + "run/task-3-suffix/status.json": json.dumps({"status": "PASS"}), + } + + class Blob: + def __init__(self, name, payload): + self.name, self.payload = name, payload + + def download_to_filename(self, filename): + Path(filename).write_text(self.payload) + + storage = MagicMock() + + def fake_list_blobs(prefix=""): + return [Blob(name, val) for name, val in payloads.items() if name.startswith(prefix)] + + storage.Client.return_value.bucket.return_value.list_blobs.side_effect = fake_list_blobs + + google_mod = types.ModuleType("google") + cloud_mod = types.ModuleType("google.cloud") + google_mod.cloud = cloud_mod + cloud_mod.storage = storage + + monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) + with patch.dict( + sys.modules, + { + "google": google_mod, + "google.cloud": cloud_mod, + "google.cloud.storage": storage, + }, + ): + results = result_harvester.harvest_candidate_patches("bucket", "run", tmp_path / "harvest") + + assert len(results) == 1 + assert results[0]["task_index"] == 0 + assert results[0]["status"] == "PASS" + assert results[0]["score"] == 0.95 From ec5dbe76f2d0d85a4473b2c570ea03ffdd912b94 Mon Sep 17 00:00:00 2001 From: jtaroreh Date: Sat, 24 Oct 2026 22:45:17 -0700 Subject: [PATCH 05/11] Fix: Reject credential evidence, record result upload errors, and anchor harvester prefix --- skills/swarm/scripts/result_harvester.py | 17 ++- skills/swarm/scripts/storage_uploader.py | 171 ++++++++++++++++++++--- 2 files changed, 164 insertions(+), 24 deletions(-) diff --git a/skills/swarm/scripts/result_harvester.py b/skills/swarm/scripts/result_harvester.py index 828da52..48eb40a 100644 --- a/skills/swarm/scripts/result_harvester.py +++ b/skills/swarm/scripts/result_harvester.py @@ -78,14 +78,17 @@ def harvest_gcs_results( target_filename: str = "score.json", ) -> List[Dict[str, Any]]: clean_prefix = prefix.strip("/") + storage_prefix = f"{clean_prefix}/" if clean_prefix else "" try: from google.cloud import storage client = storage.Client() bucket = client.bucket(bucket_name) - blobs = bucket.list_blobs(prefix=clean_prefix) + blobs = bucket.list_blobs(prefix=storage_prefix) results: List[Dict[str, Any]] = [] for blob in blobs: + if not blob.name.startswith(storage_prefix): + continue if blob.name.endswith(target_filename): payload = json.loads(blob.download_as_text(encoding="utf-8")) results.append(payload) @@ -126,6 +129,7 @@ def harvest_candidate_patches( dest = Path(dest_dir) dest.mkdir(parents=True, exist_ok=True) clean_prefix = prefix.strip("/") + storage_prefix = f"{clean_prefix}/" if clean_prefix else "" tasks_found: Dict[int, Dict[str, Any]] = {} local_dir = os.environ.get("STORAGE_MESSENGER_LOCAL_DIR") @@ -135,7 +139,7 @@ def harvest_candidate_patches( for task_folder in sorted(src_base.iterdir()): if not task_folder.is_dir(): continue - m = re.search(r"task-(\d+)", task_folder.name) + m = re.fullmatch(r"task-(\d+)", task_folder.name) if not m: continue task_idx = int(m.group(1)) @@ -173,9 +177,12 @@ def harvest_candidate_patches( client = storage.Client() bucket = client.bucket(bucket_name) - blobs = bucket.list_blobs(prefix=clean_prefix) + blobs = bucket.list_blobs(prefix=storage_prefix) for blob in blobs: - m = re.search(r"(?:^|/)task-(\d+)/(.+)$", blob.name) + if not blob.name.startswith(storage_prefix): + continue + rel_name = blob.name[len(storage_prefix):] + m = re.match(r"^task-(\d+)/(.+)$", rel_name) if not m: continue task_idx = int(m.group(1)) @@ -211,7 +218,7 @@ def harvest_candidate_patches( ) for item in dest.glob("task-*"): if item.is_dir(): - m = re.search(r"task-(\d+)", item.name) + m = re.fullmatch(r"task-(\d+)", item.name) if m: task_idx = int(m.group(1)) sf = item / "score.json" diff --git a/skills/swarm/scripts/storage_uploader.py b/skills/swarm/scripts/storage_uploader.py index b0915d6..05af3ac 100644 --- a/skills/swarm/scripts/storage_uploader.py +++ b/skills/swarm/scripts/storage_uploader.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 +import fnmatch import json import os +import re import subprocess import sys import urllib.error @@ -9,6 +11,81 @@ from pathlib import Path from typing import Dict, List, Optional, Union +try: + from worker.execution import CREDENTIAL_PATTERNS +except ImportError: + try: + from skills.swarm.scripts.worker.execution import CREDENTIAL_PATTERNS + except ImportError: + CREDENTIAL_PATTERNS = [ + re.compile(r"ghp_[A-Za-z0-9_]{36,}"), + re.compile(r"AIza[0-9A-Za-z-_]{35}"), + re.compile(r"github_pat_[0-9A-Za-z_]{20,}"), + re.compile(r"ya29\.[0-9A-Za-z-_]+"), + re.compile(r"Bearer [A-Za-z0-9_\-\.]+"), + ] + + +def is_sensitive_path(path: Path, root: Path) -> bool: + try: + p = Path(path) + root_path = Path(root) + full_path = p if p.is_absolute() else (root_path / p) + if os.path.islink(full_path) or full_path.is_symlink(): + return True + r = root_path.resolve() + resolved = full_path.resolve() + curr = full_path.parent + while curr != root_path and curr.resolve() != r and curr != curr.parent: + if os.path.islink(curr) or curr.is_symlink(): + return True + curr = curr.parent + if r != resolved and r not in resolved.parents: + return True + parts = resolved.relative_to(r).parts + if ".git" in parts or ".git" in p.parts or ".git" in full_path.parts: + return True + sensitive_patterns = [ + ".env", + ".env.*", + "*.pem", + "*.key", + "id_rsa*", + "id_ed25519*", + "credentials.json", + "token.json", + ] + for pattern in sensitive_patterns: + if fnmatch.fnmatch(full_path.name.lower(), pattern.lower()): + return True + if fnmatch.fnmatch(resolved.name.lower(), pattern.lower()): + return True + for part in parts: + for pattern in sensitive_patterns: + if fnmatch.fnmatch(part.lower(), pattern.lower()): + return True + return False + except Exception: + return True + + +def file_contains_credentials(path: Path) -> bool: + p = Path(path) + if not p.is_file(): + return False + try: + content = p.read_text(encoding="utf-8", errors="ignore") + except Exception: + try: + content = p.read_bytes().decode("utf-8", errors="ignore") + except Exception: + return False + for pat in CREDENTIAL_PATTERNS: + if pat.search(content): + return True + return False + + def get_oauth_token() -> Optional[str]: env_token = os.environ.get("GCS_OAUTH_TOKEN") or os.environ.get("GOOGLE_OAUTH_ACCESS_TOKEN") @@ -266,34 +343,60 @@ def upload_run_artifacts( uploaded: Dict[str, str] = {} - delivery_errors = [] + delivery_errors: List[str] = [] proposed_pass = status == "PASS" - evidence_paths = [] + evidence_paths: List[Path] = [] root = repo.resolve() + for relative in evidence_files or []: - path = (root / relative).resolve() - if root not in path.parents or ".git" in path.relative_to(root).parts: - delivery_errors.append(f"Evidence path outside allowed repository files: {relative}") + raw_path = root / relative + if is_sensitive_path(raw_path, root): + delivery_errors.append(f"Evidence path is sensitive or outside repository: {relative}") + continue + path = raw_path.resolve() + if is_sensitive_path(path, root): + delivery_errors.append(f"Evidence path is sensitive or outside repository: {relative}") continue if path.is_dir(): - evidence_paths.extend(path.rglob("*")) + for child in path.rglob("*"): + if is_sensitive_path(child, root): + delivery_errors.append( + f"Evidence path is sensitive or outside repository: {child.relative_to(root).as_posix() if root in child.resolve().parents else child}" + ) + elif child.is_file(): + evidence_paths.append(child) elif path.is_file(): evidence_paths.append(path) else: delivery_errors.append(f"Evidence file missing: {relative}") + diagnostics = root / ".agystack/diagnostics" if diagnostics.is_dir(): - evidence_paths.extend(diagnostics.glob("*.log")) + for diag in diagnostics.glob("*.log"): + if is_sensitive_path(diag, root): + delivery_errors.append(f"Diagnostic path is sensitive: {diag.name}") + elif diag.is_file(): + evidence_paths.append(diag) + for path in sorted(set(evidence_paths)): resolved = path.resolve() - if not resolved.is_file() or root not in resolved.parents or ".git" in resolved.relative_to(root).parts: + if not resolved.is_file(): continue - name = f"evidence/{path.relative_to(root).as_posix()}" + rel_name = path.relative_to(root).as_posix() if root in resolved.parents else path.name + if is_sensitive_path(path, root): + delivery_errors.append(f"Evidence path is sensitive: {rel_name}") + continue + if file_contains_credentials(path): + delivery_errors.append(f"Evidence file contains credentials: {rel_name}") + continue + + name = f"evidence/{rel_name}" dest = f"{target_prefix}/{name}" if upload_to_gcs(bucket_name, dest, path): uploaded[name] = f"gs://{bucket_name}/{dest}" else: delivery_errors.append(f"Evidence upload failed: {name}") + if delivery_errors and proposed_pass: status = "ISSUES" @@ -307,10 +410,17 @@ def upload_run_artifacts( patch_path.write_text(patch_content, encoding="utf-8") if patch_path.is_file() and patch_path.stat().st_size > 0: - dest = f"{target_prefix}/{patch_name}" - if upload_to_gcs(bucket_name, dest, patch_path): - uploaded[patch_name] = f"gs://{bucket_name}/{dest}" - patch_uploaded = True + if is_sensitive_path(patch_path, root): + delivery_errors.append(f"Candidate patch path is sensitive: {patch_name}") + elif file_contains_credentials(patch_path): + delivery_errors.append(f"Candidate patch contains credentials: {patch_name}") + else: + dest = f"{target_prefix}/{patch_name}" + if upload_to_gcs(bucket_name, dest, patch_path): + uploaded[patch_name] = f"gs://{bucket_name}/{dest}" + patch_uploaded = True + else: + delivery_errors.append(f"Candidate patch upload failed: {patch_name}") if candidate_files and not patch_uploaded: delivery_errors.append("Candidate patch failed to upload or is missing under zero-push delivery.") @@ -320,15 +430,33 @@ def upload_run_artifacts( # Upload score.json and results.tsv score_path = repo / "score.json" if score_path.is_file(): - dest = f"{target_prefix}/score.json" - if upload_to_gcs(bucket_name, dest, score_path): - uploaded["score.json"] = f"gs://{bucket_name}/{dest}" + if is_sensitive_path(score_path, root): + delivery_errors.append("score.json is a sensitive path") + elif file_contains_credentials(score_path): + delivery_errors.append("score.json contains credentials") + else: + dest = f"{target_prefix}/score.json" + if upload_to_gcs(bucket_name, dest, score_path): + uploaded["score.json"] = f"gs://{bucket_name}/{dest}" + else: + delivery_errors.append("score.json upload failed") results_path = repo / "results.tsv" if results_path.is_file(): - dest = f"{target_prefix}/results.tsv" - if upload_to_gcs(bucket_name, dest, results_path): - uploaded["results.tsv"] = f"gs://{bucket_name}/{dest}" + if is_sensitive_path(results_path, root): + delivery_errors.append("results.tsv is a sensitive path") + elif file_contains_credentials(results_path): + delivery_errors.append("results.tsv contains credentials") + else: + dest = f"{target_prefix}/results.tsv" + if upload_to_gcs(bucket_name, dest, results_path): + uploaded["results.tsv"] = f"gs://{bucket_name}/{dest}" + else: + delivery_errors.append("results.tsv upload failed") + + # Finalize delivery_errors and degrade status from PASS to ISSUES if delivery_errors is non-empty + if delivery_errors and status == "PASS": + status = "ISSUES" # Write and upload status.json LAST with the finalized status. status_path = repo / "status.json" @@ -343,10 +471,15 @@ def upload_run_artifacts( status_data["artifacts"] = uploaded if delivery_errors: status_data["evidence"] = "\n".join(delivery_errors) + status_data["delivery_errors"] = delivery_errors + else: + status_data.pop("delivery_errors", None) status_path.write_text(json.dumps(status_data, indent=2), encoding="utf-8") dest = f"{target_prefix}/status.json" if upload_to_gcs(bucket_name, dest, status_path): uploaded["status.json"] = f"gs://{bucket_name}/{dest}" + else: + delivery_errors.append("status.json upload failed") return uploaded From 83ca58522dc5f25c7c45e4a7638a87196f8dbc14 Mon Sep 17 00:00:00 2001 From: jtaroreh Date: Sun, 25 Oct 2026 00:04:54 -0700 Subject: [PATCH 06/11] Test: Add cloud swarm canary integration and transport verification suite --- tests/test_cloud_swarm_canary_integration.py | 799 +++++++++++++++++++ 1 file changed, 799 insertions(+) create mode 100644 tests/test_cloud_swarm_canary_integration.py diff --git a/tests/test_cloud_swarm_canary_integration.py b/tests/test_cloud_swarm_canary_integration.py new file mode 100644 index 0000000..57c7ca8 --- /dev/null +++ b/tests/test_cloud_swarm_canary_integration.py @@ -0,0 +1,799 @@ +import copy +from dataclasses import FrozenInstanceError +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import shutil +import subprocess +import sys +import time +import types +from unittest.mock import MagicMock, patch + +import pytest + +SCRIPTS = Path(__file__).resolve().parents[1] / "skills/swarm/scripts" +sys.path.insert(0, str(SCRIPTS)) + +import cloud_dispatch +import cloud_worker +import result_harvester +import storage_uploader +from worker.execution import ExecutionLimits, resolve_execution_limits, run_bounded_shell + +CANARY_PACKAGE_INVENTORY = Path( + "/Users/joeltaroreh/projects/phasepoly-swarm-prep/.audit/cloud-swarm-20261024/revision3/package-inventory.json" +) +CANARY_REVISION3_DIR = CANARY_PACKAGE_INVENTORY.parent + + +def _init_git_repo(path: Path) -> Path: + subprocess.run(["git", "init"], cwd=path, check=True, capture_output=True) + subprocess.run(["git", "config", "user.name", "Canary Tester"], cwd=path, check=True, capture_output=True) + subprocess.run(["git", "config", "user.email", "canary@test.local"], cwd=path, check=True, capture_output=True) + (path / "base.txt").write_text("initial base\n", encoding="utf-8") + subprocess.run(["git", "add", "base.txt"], cwd=path, check=True, capture_output=True) + subprocess.run(["git", "commit", "-m", "init base"], cwd=path, check=True, capture_output=True) + return path + + +EXPERIMENT_SCRIPT = """import json, sys +from pathlib import Path + +# Jordan-Wigner Majorana representation for 2 modes (4 operators) +# gamma_0 = X_0, gamma_1 = Y_0, gamma_2 = Z_0 X_1, gamma_3 = Z_0 Y_1 +majoranas = [ + {"x": 1, "z": 0}, + {"x": 1, "z": 1}, + {"x": 2, "z": 1}, + {"x": 2, "z": 3}, +] + +# Check anticommutation violations: (x1*z2 + z1*x2)%2 must be 1 for distinct pairs +violations = sum( + (bin(a["x"] & b["z"]).count("1") + bin(a["z"] & b["x"]).count("1")) % 2 != 1 + for i, a in enumerate(majoranas) + for b in majoranas[i + 1:] +) + +candidate_weight = sum(bin(m["x"] | m["z"]).count("1") for m in majoranas) +baseline_weight = 2 * (2 + 1) + +output_data = { + "size": 2, + "seed": 42, + "candidate_weight": candidate_weight, + "baseline_weight": baseline_weight, + "violations": violations, + "majoranas": majoranas, +} + +out_path = Path("screen.raw.json") +out_path.write_text(json.dumps(output_data, indent=2), encoding="utf-8") +print(f"Computed Majorana algebra: violations={violations}, weight={candidate_weight}") +""" + +DOMAIN_ORACLE_SCRIPT = """import json, sys +from pathlib import Path + +raw_file = Path("screen.raw.json") +if not raw_file.is_file(): + print("Error: screen.raw.json missing", file=sys.stderr) + sys.exit(1) + +data = json.loads(raw_file.read_text(encoding="utf-8")) +if data.get("violations", -1) != 0: + print(f"Oracle verification failed: violations={data.get('violations')}", file=sys.stderr) + sys.exit(1) + +# Exact matrix verification of 4x4 Majorana anticommutators +I = [[1, 0], [0, 1]] +X = [[0, 1], [1, 0]] +Y = [[0, -1j], [1j, 0]] +Z = [[1, 0], [0, -1]] + +def tensor(a, b): + return [[a[i // len(b)][j // len(b)] * b[i % len(b)][j % len(b)] + for j in range(len(a) * len(b))] for i in range(len(a) * len(b))] + +def mat_mul(a, b): + n = len(a) + return [[sum(a[i][k] * b[k][j] for k in range(n)) for j in range(n)] for i in range(n)] + +def matrix(mask): + out = [[1]] + for k in range(2): + x, z = (mask["x"] >> k) & 1, (mask["z"] >> k) & 1 + out = tensor(out, Y if x and z else X if x else Z if z else I) + return out + +matrices = [matrix(m) for m in data["majoranas"]] +sample_count = 0 +for i, a in enumerate(matrices): + for j, b in enumerate(matrices): + ab, ba = mat_mul(a, b), mat_mul(b, a) + expected = 2 if i == j else 0 + for r in range(4): + for c in range(4): + val = ab[r][c] + ba[r][c] + target = expected if r == c else 0 + if abs(val - target) > 1e-10: + print(f"Algebra violation at ({i},{j}) element ({r},{c}): {val} != {target}", file=sys.stderr) + sys.exit(1) + sample_count += 1 + +oracle_res = { + "checks": [ + {"name": "majorana_anticommutators_4x4", "passed": True, "sample_count": sample_count} + ] +} +Path("oracle_result.json").write_text(json.dumps(oracle_res, indent=2), encoding="utf-8") +print(f"Oracle verified {sample_count} matrix checks cleanly.") +""" + + +def test_case_1_real_scientific_computation_harvests_exact_bytes(tmp_path, monkeypatch): + """Case 1: A tiny real scientific computation produces raw numeric output and a sound domain oracle. + + Safe evidence is delivered and independently harvested under the exact canary session. + Assert harvested status is PASS, raw outputs and evidence files are preserved with exact bytes. + """ + repo = tmp_path / "canary_worker_repo" + repo.mkdir() + _init_git_repo(repo) + + task_dir = repo / "research/canary-task-0" + task_dir.mkdir(parents=True) + (task_dir / "experiment.py").write_text(EXPERIMENT_SCRIPT, encoding="utf-8") + (task_dir / "oracle.py").write_text(DOMAIN_ORACLE_SCRIPT, encoding="utf-8") + + # Run the real scientific computation + calc_res = subprocess.run([sys.executable, "experiment.py"], cwd=task_dir, capture_output=True, text=True, check=True) + assert "Computed Majorana algebra: violations=0" in calc_res.stdout + raw_path = task_dir / "screen.raw.json" + assert raw_path.is_file() + expected_raw_bytes = raw_path.read_bytes() + expected_raw_sha = hashlib.sha256(expected_raw_bytes).hexdigest() + + # Run the sound domain oracle + oracle_res = subprocess.run([sys.executable, "oracle.py"], cwd=task_dir, capture_output=True, text=True, check=True) + assert "Oracle verified 16 matrix checks cleanly." in oracle_res.stdout + oracle_path = task_dir / "oracle_result.json" + assert oracle_path.is_file() + expected_oracle_bytes = oracle_path.read_bytes() + expected_oracle_sha = hashlib.sha256(expected_oracle_bytes).hexdigest() + + # Track modified candidate file + subprocess.run(["git", "add", "research/canary-task-0/experiment.py"], cwd=repo, check=True) + + fake_storage = tmp_path / "fake_storage" + session_id = "canary-session" + bucket_name = "canary-bucket" + gcs_prefix = f"swarms/{session_id}" + + task_item = { + "task_index": 0, + "brief": "Run minimal Majorana algebra screen", + "candidate_files": ["research/canary-task-0/experiment.py"], + "evidence_files": [ + "research/canary-task-0/screen.raw.json", + "research/canary-task-0/oracle_result.json", + ], + "verify_command": f"{shlex.quote(sys.executable)} oracle.py", + "execution_limits": {"verify_timeout_s": 30, "command_timeout_s": 30}, + } + + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) + + # Perform artifact upload under the exact canary session layout + uploaded = storage_uploader.upload_run_artifacts( + bucket_name=bucket_name, + prefix=gcs_prefix, + repo_dir=repo, + task_index=0, + status="PASS", + candidate_files=task_item["candidate_files"], + evidence_files=task_item["evidence_files"], + ) + + # Verify object layout in fake storage: swarms//task-/ + task_storage = fake_storage / bucket_name / gcs_prefix / "task-0" + assert task_storage.is_dir() + assert (task_storage / "status.json").is_file() + assert (task_storage / "patch.diff").is_file() + storage_raw = task_storage / "evidence/research/canary-task-0/screen.raw.json" + storage_oracle = task_storage / "evidence/research/canary-task-0/oracle_result.json" + assert storage_raw.is_file() + assert storage_oracle.is_file() + + # Byte-exact preservation in storage + assert storage_raw.read_bytes() == expected_raw_bytes + assert hashlib.sha256(storage_raw.read_bytes()).hexdigest() == expected_raw_sha + assert storage_oracle.read_bytes() == expected_oracle_bytes + assert hashlib.sha256(storage_oracle.read_bytes()).hexdigest() == expected_oracle_sha + + # Harvest independently under the exact canary session + harvest_dest = tmp_path / "harvested" + harvested = result_harvester.harvest_candidate_patches( + bucket_name=bucket_name, + prefix=gcs_prefix, + dest_dir=harvest_dest, + ) + + assert len(harvested) == 1 + item = harvested[0] + assert item["task_index"] == 0 + assert item["status"] == "PASS" + assert item["patch_file"] is not None and Path(item["patch_file"]).is_file() + + harvested_raw = harvest_dest / "task-0/evidence/research/canary-task-0/screen.raw.json" + harvested_oracle = harvest_dest / "task-0/evidence/research/canary-task-0/oracle_result.json" + assert harvested_raw.is_file() + assert harvested_oracle.is_file() + + # Assert exact byte preservation after transport and harvesting + assert harvested_raw.read_bytes() == expected_raw_bytes + assert hashlib.sha256(harvested_raw.read_bytes()).hexdigest() == expected_raw_sha + assert harvested_oracle.read_bytes() == expected_oracle_bytes + assert hashlib.sha256(harvested_oracle.read_bytes()).hexdigest() == expected_oracle_sha + + +def test_case_2_wrong_scientific_result_fails_verification_and_degrades_to_issues(tmp_path, monkeypatch): + """Case 2: An intentionally wrong scientific result fails verification even when the agent + + requests PASS and makes no source change. Status is degraded to ISSUES and harvested status is ISSUES. + """ + repo = tmp_path / "canary_worker_repo_c2" + repo.mkdir() + _init_git_repo(repo) + + task_dir = repo / "research/canary-task-0" + task_dir.mkdir(parents=True) + (task_dir / "oracle.py").write_text(DOMAIN_ORACLE_SCRIPT, encoding="utf-8") + + # Intentionally corrupt the raw scientific output (violations = 2) + corrupted_data = { + "size": 2, + "seed": 42, + "candidate_weight": 6, + "baseline_weight": 6, + "violations": 2, + "majoranas": [{"x": 1, "z": 0}, {"x": 1, "z": 0}, {"x": 2, "z": 1}, {"x": 2, "z": 3}], + } + (task_dir / "screen.raw.json").write_text(json.dumps(corrupted_data, indent=2), encoding="utf-8") + + # No source changes made to candidate files + fake_storage = tmp_path / "fake_storage" + session_id = "canary-session" + bucket_name = "canary-bucket" + gcs_prefix = f"swarms/{session_id}" + + task_item = { + "task_index": 0, + "brief": "Run screen with corrupted algebra", + "candidate_files": ["research/canary-task-0/calc.py"], + "evidence_files": ["research/canary-task-0/screen.raw.json"], + "verify_command": f"{shlex.quote(sys.executable)} research/canary-task-0/oracle.py", + "execution_limits": {"verify_timeout_s": 10}, + } + + monkeypatch.setenv("REPO_URL", "https://example.com/repo") + monkeypatch.setenv("TASK_MANIFEST", json.dumps([task_item])) + monkeypatch.setenv("USE_VERTEX_AI", "1") + monkeypatch.setenv("CLOUD_RUN_TASK_INDEX", "0") + monkeypatch.setenv("SWARM_SESSION_ID", session_id) + monkeypatch.setenv("GCS_RESULTS_BUCKET", bucket_name) + monkeypatch.setenv("GCS_BUCKET", bucket_name) + monkeypatch.setenv("GCS_PREFIX", gcs_prefix) + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) + monkeypatch.setattr(cloud_worker.tempfile, "gettempdir", lambda: str(tmp_path)) + + def fake_clone(**kwargs): + dest_repo = kwargs["repo_dir"] + _init_git_repo(dest_repo) + t_dir = dest_repo / "research/canary-task-0" + t_dir.mkdir(parents=True, exist_ok=True) + (t_dir / "oracle.py").write_text(DOMAIN_ORACLE_SCRIPT, encoding="utf-8") + corrupted_data = { + "size": 2, + "seed": 42, + "candidate_weight": 6, + "baseline_weight": 6, + "violations": 2, + "majoranas": [{"x": 1, "z": 0}, {"x": 1, "z": 0}, {"x": 2, "z": 1}, {"x": 2, "z": 3}], + } + (t_dir / "screen.raw.json").write_text(json.dumps(corrupted_data, indent=2), encoding="utf-8") + + # Mock execute_task so agent requests PASS without modifying candidate files + def fake_execute(**kwargs): + return "PASS", "Agent claimed pass with no source changes." + + monkeypatch.setattr(cloud_worker, "clone_and_checkout_task", fake_clone) + monkeypatch.setattr(cloud_worker, "execute_task", fake_execute) + monkeypatch.setattr(cloud_worker, "StorageMessenger", None) + + orig_cwd = os.getcwd() + try: + # cloud_worker.main() should run, run verify_command which fails, degrade to ISSUES, and exit(1) + with pytest.raises(SystemExit) as exc_info: + cloud_worker.main() + assert exc_info.value.code == 1 + finally: + os.chdir(orig_cwd) + + # Check status.json locally in the task repo and in storage + active_repo = tmp_path / "repo-task-0" + status_file = active_repo / "status.json" + assert status_file.is_file() + status_data = json.loads(status_file.read_text(encoding="utf-8")) + assert status_data["status"] == "ISSUES" + assert "Verification:" in status_data["summary"] + + # Harvest and verify that harvested status is ISSUES and patch is quarantined + harvest_dest = tmp_path / "harvested_c2" + harvested = result_harvester.harvest_candidate_patches( + bucket_name=bucket_name, + prefix=gcs_prefix, + dest_dir=harvest_dest, + ) + assert len(harvested) == 1 + assert harvested[0]["task_index"] == 0 + assert harvested[0]["status"] == "ISSUES" + assert harvested[0]["patch_file"] is None + + +def test_case_3_command_timeout_diagnostics_retained_no_orphans_issues(tmp_path): + """Case 3: A bounded command times out (e.g. 1s timeout on a 3s sleep). + + Diagnostics retain partial output, child processes terminate without orphans, status is not PASS. + """ + repo = tmp_path / "canary_worker_repo_c3" + repo.mkdir() + _init_git_repo(repo) + + child_script = ( + "import os, time; " + "open('child.pid', 'w').write(str(os.getpid())); " + "time.sleep(4); " + "open('survived.txt', 'w').write('orphaned');" + ) + command = ( + f"echo 'partial-stdout-line-1'; echo 'partial-stderr-line-1' >&2; " + f"{shlex.quote(sys.executable)} -c {shlex.quote(child_script)}" + ) + + res, output = run_bounded_shell(command, repo, timeout_s=1, label="canary-timeout") + + # Command timed out -> exit code 124 + assert res.returncode == 124 + assert "Command timed out after 1 seconds" in res.stderr + assert "partial-stdout-line-1" in output + assert "partial-stderr-line-1" in output + + # Diagnostics retained in log file + diag_logs = list((repo / ".agystack/diagnostics").glob("canary-timeout-*.log")) + assert len(diag_logs) == 1 + log_content = diag_logs[0].read_text(encoding="utf-8") + assert "partial-stdout-line-1" in log_content + assert "partial-stderr-line-1" in log_content + + # Child process was terminated without orphans + pid_file = repo / "child.pid" + assert pid_file.is_file() + child_pid = int(pid_file.read_text(encoding="utf-8").strip()) + + time.sleep(1.0) + assert not (repo / "survived.txt").exists() + + # Check process table: process should not be actively running + proc_check = subprocess.run(["ps", "-o", "stat=", "-p", str(child_pid)], capture_output=True, text=True) + stat = proc_check.stdout.strip() + assert not stat or stat.startswith("Z") # Dead or zombie + + +def test_case_4_synthetic_credentials_in_evidence_rejected_intact_on_disk_issues(tmp_path): + """Case 4: A synthetic credential (e.g. AIza... or ghp_... token pattern) in declared evidence + + is rejected. The safe artifact's bytes remain intact on disk, no credential appears in published objects + or diagnostics, delivery_errors records the error, and status is degraded to ISSUES. + """ + repo = tmp_path / "canary_worker_repo_c4" + repo.mkdir() + _init_git_repo(repo) + + evidence_dir = repo / "research/canary-task-0" + evidence_dir.mkdir(parents=True) + + # 1. Safe artifact + safe_file = evidence_dir / "safe_computation.json" + safe_content = json.dumps({"metric": "majorana_parity", "value": 1.0, "clean": True}, indent=2) + safe_file.write_text(safe_content, encoding="utf-8") + safe_bytes_before = safe_file.read_bytes() + safe_sha_before = hashlib.sha256(safe_bytes_before).hexdigest() + + # 2. File with synthetic credentials + secret_ghp = "ghp_" + "C" * 36 + secret_aiza = "AIza" + "D" * 35 + cred_file = evidence_dir / "leaked_evidence.txt" + cred_content = f"Debug log containing sensitive tokens: {secret_ghp} and {secret_aiza}\n" + cred_file.write_text(cred_content, encoding="utf-8") + + fake_storage = tmp_path / "fake_storage" + session_id = "canary-session" + bucket_name = "canary-bucket" + gcs_prefix = f"swarms/{session_id}" + + with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage)}): + results = storage_uploader.upload_run_artifacts( + bucket_name=bucket_name, + prefix=gcs_prefix, + repo_dir=repo, + task_index=0, + status="PASS", + evidence_files=[ + "research/canary-task-0/safe_computation.json", + "research/canary-task-0/leaked_evidence.txt", + ], + ) + + # Safe artifact bytes remain intact on disk + assert safe_file.read_bytes() == safe_bytes_before + assert hashlib.sha256(safe_file.read_bytes()).hexdigest() == safe_sha_before + + # Credential file on disk is not silently modified or wiped + assert cred_file.read_text(encoding="utf-8") == cred_content + + # Credential file was REJECTED from uploaded artifacts + assert "evidence/research/canary-task-0/leaked_evidence.txt" not in results + assert "evidence/research/canary-task-0/safe_computation.json" in results + + # No credential appears anywhere in published objects in storage + task_storage = fake_storage / bucket_name / gcs_prefix / "task-0" + assert task_storage.is_dir() + for pub_file in task_storage.rglob("*"): + if pub_file.is_file(): + content = pub_file.read_text(encoding="utf-8", errors="ignore") + assert secret_ghp not in content + assert secret_aiza not in content + + # Diagnostics do not leak credentials + diag_dir = repo / ".agystack/diagnostics" + if diag_dir.is_dir(): + for dfile in diag_dir.glob("*.log"): + dcontent = dfile.read_text(encoding="utf-8", errors="ignore") + assert secret_ghp not in dcontent + assert secret_aiza not in dcontent + + # delivery_errors in status.json records the error and status is degraded to ISSUES + status_file = repo / "status.json" + assert status_file.is_file() + status_data = json.loads(status_file.read_text(encoding="utf-8")) + assert status_data["status"] == "ISSUES" + assert "delivery_errors" in status_data + assert any("contains credentials" in err for err in status_data["delivery_errors"]) + + # Harvested status is ISSUES + harvested = result_harvester.harvest_candidate_patches( + bucket_name=bucket_name, + prefix=gcs_prefix, + dest_dir=tmp_path / "harvest_c4", + ) + assert len(harvested) == 1 + assert harvested[0]["status"] == "ISSUES" + + +def test_case_5_failed_score_or_results_upload_produces_delivery_errors_and_issues(tmp_path): + """Case 5: A failed score or results upload produces delivery errors and ISSUES. + + Test both candidate files and no-change tasks. + """ + fake_storage = tmp_path / "fake_storage_c5" + bucket_name = "canary-bucket" + + # 5a: With candidate files + repo_cand = tmp_path / "canary_worker_repo_c5_cand" + repo_cand.mkdir() + _init_git_repo(repo_cand) + + (repo_cand / "experiment.py").write_text("updated = True\n", encoding="utf-8") + (repo_cand / "score.json").write_text(json.dumps({"score": 0.95}), encoding="utf-8") + (repo_cand / "results.tsv").write_text("metric\tvalue\nscore\t0.95\n", encoding="utf-8") + + def fail_score_upload(bucket, obj_name, file_path): + if "score.json" in str(obj_name): + return False + # Save to local storage for realistic harvesting + dest = fake_storage / bucket / obj_name + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(file_path, dest) + return True + + prefix_cand = "swarms/canary-session-cand" + with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage)}): + with patch.object(storage_uploader, "upload_to_gcs", side_effect=fail_score_upload): + uploaded_cand = storage_uploader.upload_run_artifacts( + bucket_name=bucket_name, + prefix=prefix_cand, + repo_dir=repo_cand, + task_index=0, + status="PASS", + candidate_files=["experiment.py"], + ) + + assert "score.json" not in uploaded_cand + status_file_cand = repo_cand / "status.json" + status_data_cand = json.loads(status_file_cand.read_text(encoding="utf-8")) + assert status_data_cand["status"] == "ISSUES" + assert "delivery_errors" in status_data_cand + assert any("score.json upload failed" in err for err in status_data_cand["delivery_errors"]) + + harvested_cand = result_harvester.harvest_candidate_patches( + bucket_name=bucket_name, + prefix=prefix_cand, + dest_dir=tmp_path / "harvest_c5_cand", + ) + assert len(harvested_cand) == 1 + assert harvested_cand[0]["status"] == "ISSUES" + # Non-PASS candidate patch is quarantined and not promoted + assert harvested_cand[0]["patch_file"] is None + + # 5b: No-change tasks (clean working directory, no candidate files modified) + repo_clean = tmp_path / "canary_worker_repo_c5_clean" + repo_clean.mkdir() + _init_git_repo(repo_clean) + + (repo_clean / "score.json").write_text(json.dumps({"score": 0.90}), encoding="utf-8") + (repo_clean / "results.tsv").write_text("metric\tvalue\nscore\t0.90\n", encoding="utf-8") + + def fail_results_upload(bucket, obj_name, file_path): + if "results.tsv" in str(obj_name): + return False + dest = fake_storage / bucket / obj_name + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(file_path, dest) + return True + + prefix_clean = "swarms/canary-session-clean" + with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage)}): + with patch.object(storage_uploader, "upload_to_gcs", side_effect=fail_results_upload): + uploaded_clean = storage_uploader.upload_run_artifacts( + bucket_name=bucket_name, + prefix=prefix_clean, + repo_dir=repo_clean, + task_index=0, + status="PASS", + candidate_files=[], + ) + + assert "results.tsv" not in uploaded_clean + status_file_clean = repo_clean / "status.json" + status_data_clean = json.loads(status_file_clean.read_text(encoding="utf-8")) + assert status_data_clean["status"] == "ISSUES" + assert "delivery_errors" in status_data_clean + assert any("results.tsv upload failed" in err for err in status_data_clean["delivery_errors"]) + + harvested_clean = result_harvester.harvest_candidate_patches( + bucket_name=bucket_name, + prefix=prefix_clean, + dest_dir=tmp_path / "harvest_c5_clean", + ) + assert len(harvested_clean) == 1 + assert harvested_clean[0]["status"] == "ISSUES" + assert harvested_clean[0]["patch_file"] is None + + + + +def test_case_6_sibling_prefix_isolation(tmp_path, monkeypatch): + """Case 6: Sibling prefix isolation: swarms/canary-other/ cannot satisfy canary coverage + + or contribute a promotable patch in result_harvester. + """ + fake_storage = tmp_path / "storage" + canary_dir = fake_storage / "bucket/swarms/canary/task-0" + canary_dir.mkdir(parents=True) + (canary_dir / "status.json").write_text(json.dumps({"task_index": 0, "status": "PASS", "score": 0.80})) + (canary_dir / "score.json").write_text(json.dumps({"score": 0.80})) + (canary_dir / "patch.diff").write_text("diff --git a/canary b/canary\n+canary") + + sibling_dir = fake_storage / "bucket/swarms/canary-other/task-1" + sibling_dir.mkdir(parents=True) + (sibling_dir / "status.json").write_text(json.dumps({"task_index": 1, "status": "PASS", "score": 0.99})) + (sibling_dir / "score.json").write_text(json.dumps({"score": 0.99})) + (sibling_dir / "patch.diff").write_text("diff --git a/sibling b/sibling\n+sibling") + + sibling_task0 = fake_storage / "bucket/swarms/canary-other/task-0" + sibling_task0.mkdir(parents=True) + (sibling_task0 / "status.json").write_text(json.dumps({"task_index": 0, "status": "PASS", "score": 0.99})) + + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) + + # Harvest for swarms/canary + harvest_dest = tmp_path / "harvest" + harvested = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", harvest_dest) + + # Must contain ONLY task-0 from canary, score 0.80 + assert len(harvested) == 1 + assert harvested[0]["task_index"] == 0 + assert harvested[0]["score"] == 0.80 + assert Path(harvested[0]["patch_file"]).read_text() == "diff --git a/canary b/canary\n+canary" + + # Sibling prefix canary-other cannot satisfy canary coverage if 2 tasks are expected + assert cloud_dispatch.failed_task_count(harvested, 2) == 1 + + # Also test GCS mock with trailing-slash prefix isolation + monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) + blobs = { + "swarms/canary/task-0/status.json": json.dumps({"task_index": 0, "status": "PASS", "score": 0.80}), + "swarms/canary/task-0/patch.diff": "canary patch", + "swarms/canary-other/task-1/status.json": json.dumps({"task_index": 1, "status": "PASS", "score": 0.99}), + "swarms/canary-other/task-1/patch.diff": "sibling patch", + } + + class Blob: + def __init__(self, name, payload): + self.name, self.payload = name, payload + + def download_to_filename(self, filename): + Path(filename).parent.mkdir(parents=True, exist_ok=True) + Path(filename).write_text(self.payload, encoding="utf-8") + + mock_storage = MagicMock() + mock_storage.Client.return_value.bucket.return_value.list_blobs.side_effect = ( + lambda prefix="": [Blob(k, v) for k, v in blobs.items() if k.startswith(prefix)] + ) + + google_mod = types.ModuleType("google") + cloud_mod = types.ModuleType("google.cloud") + google_mod.cloud = cloud_mod + cloud_mod.storage = mock_storage + + with patch.dict(sys.modules, {"google": google_mod, "google.cloud": cloud_mod, "google.cloud.storage": mock_storage}): + harvest_gcs = tmp_path / "harvest_gcs" + gcs_results = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", harvest_gcs) + + assert len(gcs_results) == 1 + assert gcs_results[0]["task_index"] == 0 + assert gcs_results[0]["score"] == 0.80 + assert Path(gcs_results[0]["patch_file"]).read_text() == "canary patch" + + +def test_task_level_execution_limits_override_cli_env_runtime_defaults(): + """Verify that task-level execution limits override CLI, environment, runtime config, and defaults. + + Also verify immutability and rejection of invalid values. + """ + # 1. Defaults + default_limits = resolve_execution_limits(environ={}) + assert (default_limits.max_agent_turns, default_limits.command_timeout_s, default_limits.verify_timeout_s) == (30, 300, 300) + + # 2. Runtime overrides defaults + runtime_limits = resolve_execution_limits( + environ={}, + runtime={"max_agent_turns": 40, "command_timeout_s": 350}, + ) + assert (runtime_limits.max_agent_turns, runtime_limits.command_timeout_s, runtime_limits.verify_timeout_s) == (40, 350, 300) + + # 3. Environment overrides runtime + env_limits = resolve_execution_limits( + environ={"AGYSTACK_MAX_AGENT_TURNS": "50", "COMMAND_TIMEOUT": "450"}, + runtime={"max_agent_turns": 40, "command_timeout_s": 350}, + ) + assert (env_limits.max_agent_turns, env_limits.command_timeout_s, env_limits.verify_timeout_s) == (50, 450, 300) + + # 4. CLI overrides environment + cli_limits = resolve_execution_limits( + environ={"AGYSTACK_MAX_AGENT_TURNS": "50", "COMMAND_TIMEOUT": "450"}, + runtime={"max_agent_turns": 40, "command_timeout_s": 350}, + cli={"max_agent_turns": 60}, + ) + assert (cli_limits.max_agent_turns, cli_limits.command_timeout_s, cli_limits.verify_timeout_s) == (60, 450, 300) + + # 5. Task-level limits override CLI, environment, runtime, and defaults + task_limits = resolve_execution_limits( + {"max_agent_turns": 80, "command_timeout_s": 700, "verify_timeout_s": 900}, + environ={"AGYSTACK_MAX_AGENT_TURNS": "50", "COMMAND_TIMEOUT": "450", "VERIFY_TIMEOUT": "550"}, + runtime={"max_agent_turns": 40, "command_timeout_s": 350, "verify_timeout_s": 400}, + cli={"max_agent_turns": 60, "command_timeout_s": 500}, + ) + assert (task_limits.max_agent_turns, task_limits.command_timeout_s, task_limits.verify_timeout_s) == (80, 700, 900) + + # Immutability + with pytest.raises(FrozenInstanceError): + task_limits.max_agent_turns = 100 + + # Invalid values rejected + for invalid in [0, -1, True, False, 2.5, "ten", None]: + with pytest.raises(ValueError): + resolve_execution_limits({"max_agent_turns": invalid}, environ={}) + + +def test_fake_storage_reproduces_layout_trailing_slash_and_nested_paths(tmp_path, monkeypatch): + """Verify fake storage reproduces swarms//task-/ layout, trailing-slash list prefixes, + + and nested task paths with exact byte preservation. + """ + repo = tmp_path / "layout_repo" + repo.mkdir() + _init_git_repo(repo) + + # Create nested evidence directories and files + deep_evidence = repo / "nested/domain/oracle/checks.json" + deep_evidence.parent.mkdir(parents=True) + deep_content = json.dumps({"depth": 3, "oracle_verified": True}, indent=2) + deep_evidence.write_text(deep_content, encoding="utf-8") + + fake_storage = tmp_path / "fake_storage" + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) + + session_id = "canary-layout-session" + bucket_name = "test-layout-bucket" + prefix = f"swarms/{session_id}" + + storage_uploader.upload_run_artifacts( + bucket_name=bucket_name, + prefix=prefix, + repo_dir=repo, + task_index=0, + status="PASS", + evidence_files=["nested/domain/oracle/checks.json"], + ) + + # Verify exact layout in fake storage + expected_storage_path = ( + fake_storage + / bucket_name + / prefix + / "task-0" + / "evidence/nested/domain/oracle/checks.json" + ) + assert expected_storage_path.is_file() + assert expected_storage_path.read_text(encoding="utf-8") == deep_content + + # Harvest nested paths + harvest_dest = tmp_path / "harvest_layout" + harvested = result_harvester.harvest_candidate_patches(bucket_name, prefix, harvest_dest) + assert len(harvested) == 1 + harvested_nested = harvest_dest / "task-0/evidence/nested/domain/oracle/checks.json" + assert harvested_nested.is_file() + assert harvested_nested.read_text(encoding="utf-8") == deep_content + + +def test_verify_upload_mapping_against_package_inventory(): + """Verify proposed upload mapping locally against the package inventory (package-inventory.json). + + Checks that all listed files exist, sha256 hashes match on disk, and map cleanly to preparation prefixes. + """ + assert CANARY_PACKAGE_INVENTORY.is_file(), f"Package inventory missing at {CANARY_PACKAGE_INVENTORY}" + inventory = json.loads(CANARY_PACKAGE_INVENTORY.read_text(encoding="utf-8")) + assert len(inventory) >= 27, f"Expected at least 27 inventory files, found {len(inventory)}" + + # Required key files + assert "bootstrap.py" in inventory + assert "verify_research.py" in inventory + assert "EVIDENCE_SCHEMA.md" in inventory + assert any(re.match(r"^packets/task-\d{2}\.md$", k) for k in inventory) + + canary_session = "phasepoly24-canary-20261025" + preparation_prefix = f"swarms/{canary_session}/preparation" + + upload_mapping = {} + for rel_path, expected_sha in inventory.items(): + file_path = CANARY_REVISION3_DIR / rel_path + assert file_path.is_file(), f"Inventory item {rel_path} does not exist at {file_path}" + actual_sha = hashlib.sha256(file_path.read_bytes()).hexdigest() + assert actual_sha == expected_sha, ( + f"Hash mismatch for {rel_path}: expected {expected_sha}, got {actual_sha}" + ) + # Proposed destination in bucket + dest_blob = f"{preparation_prefix}/{rel_path}" + upload_mapping[rel_path] = dest_blob + + # Verify every destination blob has proper prefix and valid layout + for rel_path, dest_blob in upload_mapping.items(): + assert dest_blob.startswith(f"{preparation_prefix}/") + assert ".." not in dest_blob + assert not dest_blob.endswith("/") From 8fb4a8790bc93a05f2478dce403dd0eb79780035 Mon Sep 17 00:00:00 2001 From: jtaroreh Date: Sun, 25 Oct 2026 05:09:27 -0700 Subject: [PATCH 07/11] Fix: Decouple dry-run auth, reorder manifest validation, and make canary tests portable --- skills/swarm/scripts/cloud_dispatch.py | 54 ++++++++++---------- tests/test_cloud_swarm_canary_integration.py | 45 ++++++++++++++-- 2 files changed, 68 insertions(+), 31 deletions(-) diff --git a/skills/swarm/scripts/cloud_dispatch.py b/skills/swarm/scripts/cloud_dispatch.py index 3674cf9..bab8f60 100644 --- a/skills/swarm/scripts/cloud_dispatch.py +++ b/skills/swarm/scripts/cloud_dispatch.py @@ -1547,6 +1547,30 @@ def main() -> None: sys.exit(1) return + tasks_list: List[Any] = [] + sys.path.insert(0, str(Path(__file__).parent)) + + if args.seed_lottery: + from manifest_generator import generate_seed_lottery + tasks_list = generate_seed_lottery(seeds_count=args.seed_lottery) + elif args.manifest: + try: + tasks_list = load_manifest(args.manifest) + except Exception as exc: + print(f"Error: Failed to load manifest: {exc}", file=sys.stderr) + sys.exit(1) + + try: + for task in tasks_list: + if isinstance(task, dict): + resolve_execution_limits(task.get("execution_limits", {}), environ={}) + except ValueError as exc: + parser.error(str(exc)) + task_count = args.tasks or (len(tasks_list) if tasks_list else 1) + if tasks_list and task_count != len(tasks_list): + parser.error("--tasks must match the manifest task count") + manifest_serialized = json.dumps(tasks_list) if tasks_list else "" + repo_url = args.repo or get_git_remote_url() gh_token = get_gh_token() gemini_api_key = os.environ.get("GEMINI_API_KEY", "").strip() @@ -1619,37 +1643,15 @@ def main() -> None: sys.exit(1) if not gh_token and "GH_TOKEN" not in active_secret_keys and "gh_token" not in active_secret_keys: - print("Error: GH_TOKEN could not be resolved from environment or `gh auth token`", file=sys.stderr) - sys.exit(1) + if not args.dry_run: + print("Error: GH_TOKEN could not be resolved from environment or `gh auth token`", file=sys.stderr) + sys.exit(1) + gh_token = "DRY_RUN_GH_TOKEN" if not use_vertex and not gemini_api_key and not args.dry_run and "GEMINI_API_KEY" not in active_secret_keys and "gemini_api_key" not in active_secret_keys: print("Error: GEMINI_API_KEY environment variable is required when not in Vertex AI mode.", file=sys.stderr) sys.exit(1) - tasks_list: List[Any] = [] - sys.path.insert(0, str(Path(__file__).parent)) - - if args.seed_lottery: - from manifest_generator import generate_seed_lottery - tasks_list = generate_seed_lottery(seeds_count=args.seed_lottery) - elif args.manifest: - try: - tasks_list = load_manifest(args.manifest) - except Exception as exc: - print(f"Error: Failed to load manifest: {exc}", file=sys.stderr) - sys.exit(1) - - try: - for task in tasks_list: - if isinstance(task, dict): - resolve_execution_limits(task.get("execution_limits", {}), environ={}) - except ValueError as exc: - parser.error(str(exc)) - task_count = args.tasks or (len(tasks_list) if tasks_list else 1) - if tasks_list and task_count != len(tasks_list): - parser.error("--tasks must match the manifest task count") - manifest_serialized = json.dumps(tasks_list) if tasks_list else "" - env_vars = { "REPO_URL": repo_url, } diff --git a/tests/test_cloud_swarm_canary_integration.py b/tests/test_cloud_swarm_canary_integration.py index 57c7ca8..c854fc4 100644 --- a/tests/test_cloud_swarm_canary_integration.py +++ b/tests/test_cloud_swarm_canary_integration.py @@ -25,7 +25,10 @@ from worker.execution import ExecutionLimits, resolve_execution_limits, run_bounded_shell CANARY_PACKAGE_INVENTORY = Path( - "/Users/joeltaroreh/projects/phasepoly-swarm-prep/.audit/cloud-swarm-20261024/revision3/package-inventory.json" + os.environ.get( + "AGYSTACK_CANARY_PACKAGE_INVENTORY", + "/Users/joeltaroreh/projects/phasepoly-swarm-prep/.audit/cloud-swarm-20261024/revision3/package-inventory.json", + ) ) CANARY_REVISION3_DIR = CANARY_PACKAGE_INVENTORY.parent @@ -762,13 +765,45 @@ def test_fake_storage_reproduces_layout_trailing_slash_and_nested_paths(tmp_path assert harvested_nested.read_text(encoding="utf-8") == deep_content -def test_verify_upload_mapping_against_package_inventory(): +def test_verify_upload_mapping_against_package_inventory(tmp_path): """Verify proposed upload mapping locally against the package inventory (package-inventory.json). Checks that all listed files exist, sha256 hashes match on disk, and map cleanly to preparation prefixes. """ - assert CANARY_PACKAGE_INVENTORY.is_file(), f"Package inventory missing at {CANARY_PACKAGE_INVENTORY}" - inventory = json.loads(CANARY_PACKAGE_INVENTORY.read_text(encoding="utf-8")) + inventory_path = Path( + os.environ.get("AGYSTACK_CANARY_PACKAGE_INVENTORY") or CANARY_PACKAGE_INVENTORY + ) + if not inventory_path.is_file(): + fixture_dir = tmp_path / "canary_fixture" + fixture_dir.mkdir(parents=True, exist_ok=True) + packets_dir = fixture_dir / "packets" + packets_dir.mkdir(parents=True, exist_ok=True) + + inventory_data = {} + key_files = { + "bootstrap.py": "#!/usr/bin/env python3\nprint('bootstrapping')\n", + "verify_research.py": "#!/usr/bin/env python3\nprint('verifying')\n", + "EVIDENCE_SCHEMA.md": "# Evidence Schema\n", + } + for filename, content in key_files.items(): + file_path = fixture_dir / filename + file_path.write_text(content, encoding="utf-8") + inventory_data[filename] = hashlib.sha256(file_path.read_bytes()).hexdigest() + + for i in range(1, 25): + packet_name = f"packets/task-{i:02d}.md" + packet_path = fixture_dir / packet_name + packet_path.write_text(f"# Task {i:02d}\n", encoding="utf-8") + inventory_data[packet_name] = hashlib.sha256(packet_path.read_bytes()).hexdigest() + + inventory_path = fixture_dir / "package-inventory.json" + inventory_path.write_text(json.dumps(inventory_data, indent=2), encoding="utf-8") + revision_dir = fixture_dir + else: + revision_dir = inventory_path.parent + + assert inventory_path.is_file(), f"Package inventory missing at {inventory_path}" + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) assert len(inventory) >= 27, f"Expected at least 27 inventory files, found {len(inventory)}" # Required key files @@ -782,7 +817,7 @@ def test_verify_upload_mapping_against_package_inventory(): upload_mapping = {} for rel_path, expected_sha in inventory.items(): - file_path = CANARY_REVISION3_DIR / rel_path + file_path = revision_dir / rel_path assert file_path.is_file(), f"Inventory item {rel_path} does not exist at {file_path}" actual_sha = hashlib.sha256(file_path.read_bytes()).hexdigest() assert actual_sha == expected_sha, ( From 5e96cd4ff02057a874f1ad93c58f75ac6841e6ac Mon Sep 17 00:00:00 2001 From: jtaroreh Date: Sun, 25 Oct 2026 05:27:57 -0700 Subject: [PATCH 08/11] test: consolidate canary suite, strengthen credential safeguard, and isolate dispatch env --- tests/test_cloud_swarm_canary_integration.py | 830 +++---------------- tests/test_storage_uploader.py | 36 +- tests/test_worker_execution_lifecycle.py | 5 +- 3 files changed, 152 insertions(+), 719 deletions(-) diff --git a/tests/test_cloud_swarm_canary_integration.py b/tests/test_cloud_swarm_canary_integration.py index c854fc4..466abb9 100644 --- a/tests/test_cloud_swarm_canary_integration.py +++ b/tests/test_cloud_swarm_canary_integration.py @@ -1,15 +1,11 @@ -import copy -from dataclasses import FrozenInstanceError import hashlib import json import os from pathlib import Path -import re import shlex import shutil import subprocess import sys -import time import types from unittest.mock import MagicMock, patch @@ -22,325 +18,121 @@ import cloud_worker import result_harvester import storage_uploader -from worker.execution import ExecutionLimits, resolve_execution_limits, run_bounded_shell - -CANARY_PACKAGE_INVENTORY = Path( - os.environ.get( - "AGYSTACK_CANARY_PACKAGE_INVENTORY", - "/Users/joeltaroreh/projects/phasepoly-swarm-prep/.audit/cloud-swarm-20261024/revision3/package-inventory.json", - ) -) -CANARY_REVISION3_DIR = CANARY_PACKAGE_INVENTORY.parent def _init_git_repo(path: Path) -> Path: subprocess.run(["git", "init"], cwd=path, check=True, capture_output=True) - subprocess.run(["git", "config", "user.name", "Canary Tester"], cwd=path, check=True, capture_output=True) - subprocess.run(["git", "config", "user.email", "canary@test.local"], cwd=path, check=True, capture_output=True) - (path / "base.txt").write_text("initial base\n", encoding="utf-8") + subprocess.run(["git", "config", "user.name", "Tester"], cwd=path, check=True, capture_output=True) + subprocess.run(["git", "config", "user.email", "t@test.local"], cwd=path, check=True, capture_output=True) + (path / "base.txt").write_text("base\n", encoding="utf-8") subprocess.run(["git", "add", "base.txt"], cwd=path, check=True, capture_output=True) - subprocess.run(["git", "commit", "-m", "init base"], cwd=path, check=True, capture_output=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=path, check=True, capture_output=True) return path -EXPERIMENT_SCRIPT = """import json, sys -from pathlib import Path - -# Jordan-Wigner Majorana representation for 2 modes (4 operators) -# gamma_0 = X_0, gamma_1 = Y_0, gamma_2 = Z_0 X_1, gamma_3 = Z_0 Y_1 -majoranas = [ - {"x": 1, "z": 0}, - {"x": 1, "z": 1}, - {"x": 2, "z": 1}, - {"x": 2, "z": 3}, -] - -# Check anticommutation violations: (x1*z2 + z1*x2)%2 must be 1 for distinct pairs -violations = sum( - (bin(a["x"] & b["z"]).count("1") + bin(a["z"] & b["x"]).count("1")) % 2 != 1 - for i, a in enumerate(majoranas) - for b in majoranas[i + 1:] -) - -candidate_weight = sum(bin(m["x"] | m["z"]).count("1") for m in majoranas) -baseline_weight = 2 * (2 + 1) - -output_data = { - "size": 2, - "seed": 42, - "candidate_weight": candidate_weight, - "baseline_weight": baseline_weight, - "violations": violations, - "majoranas": majoranas, -} - -out_path = Path("screen.raw.json") -out_path.write_text(json.dumps(output_data, indent=2), encoding="utf-8") -print(f"Computed Majorana algebra: violations={violations}, weight={candidate_weight}") -""" - -DOMAIN_ORACLE_SCRIPT = """import json, sys -from pathlib import Path - -raw_file = Path("screen.raw.json") -if not raw_file.is_file(): - print("Error: screen.raw.json missing", file=sys.stderr) - sys.exit(1) - -data = json.loads(raw_file.read_text(encoding="utf-8")) -if data.get("violations", -1) != 0: - print(f"Oracle verification failed: violations={data.get('violations')}", file=sys.stderr) - sys.exit(1) - -# Exact matrix verification of 4x4 Majorana anticommutators -I = [[1, 0], [0, 1]] -X = [[0, 1], [1, 0]] -Y = [[0, -1j], [1j, 0]] -Z = [[1, 0], [0, -1]] - -def tensor(a, b): - return [[a[i // len(b)][j // len(b)] * b[i % len(b)][j % len(b)] - for j in range(len(a) * len(b))] for i in range(len(a) * len(b))] - -def mat_mul(a, b): - n = len(a) - return [[sum(a[i][k] * b[k][j] for k in range(n)) for j in range(n)] for i in range(n)] - -def matrix(mask): - out = [[1]] - for k in range(2): - x, z = (mask["x"] >> k) & 1, (mask["z"] >> k) & 1 - out = tensor(out, Y if x and z else X if x else Z if z else I) - return out - -matrices = [matrix(m) for m in data["majoranas"]] -sample_count = 0 -for i, a in enumerate(matrices): - for j, b in enumerate(matrices): - ab, ba = mat_mul(a, b), mat_mul(b, a) - expected = 2 if i == j else 0 - for r in range(4): - for c in range(4): - val = ab[r][c] + ba[r][c] - target = expected if r == c else 0 - if abs(val - target) > 1e-10: - print(f"Algebra violation at ({i},{j}) element ({r},{c}): {val} != {target}", file=sys.stderr) - sys.exit(1) - sample_count += 1 - -oracle_res = { - "checks": [ - {"name": "majorana_anticommutators_4x4", "passed": True, "sample_count": sample_count} - ] -} -Path("oracle_result.json").write_text(json.dumps(oracle_res, indent=2), encoding="utf-8") -print(f"Oracle verified {sample_count} matrix checks cleanly.") -""" - - -def test_case_1_real_scientific_computation_harvests_exact_bytes(tmp_path, monkeypatch): - """Case 1: A tiny real scientific computation produces raw numeric output and a sound domain oracle. - - Safe evidence is delivered and independently harvested under the exact canary session. - Assert harvested status is PASS, raw outputs and evidence files are preserved with exact bytes. - """ - repo = tmp_path / "canary_worker_repo" +def test_end_to_end_safe_evidence_transport_and_nested_layout(tmp_path, monkeypatch): + repo = tmp_path / "repo" repo.mkdir() _init_git_repo(repo) - task_dir = repo / "research/canary-task-0" - task_dir.mkdir(parents=True) - (task_dir / "experiment.py").write_text(EXPERIMENT_SCRIPT, encoding="utf-8") - (task_dir / "oracle.py").write_text(DOMAIN_ORACLE_SCRIPT, encoding="utf-8") - - # Run the real scientific computation - calc_res = subprocess.run([sys.executable, "experiment.py"], cwd=task_dir, capture_output=True, text=True, check=True) - assert "Computed Majorana algebra: violations=0" in calc_res.stdout - raw_path = task_dir / "screen.raw.json" - assert raw_path.is_file() - expected_raw_bytes = raw_path.read_bytes() - expected_raw_sha = hashlib.sha256(expected_raw_bytes).hexdigest() - - # Run the sound domain oracle - oracle_res = subprocess.run([sys.executable, "oracle.py"], cwd=task_dir, capture_output=True, text=True, check=True) - assert "Oracle verified 16 matrix checks cleanly." in oracle_res.stdout - oracle_path = task_dir / "oracle_result.json" - assert oracle_path.is_file() - expected_oracle_bytes = oracle_path.read_bytes() - expected_oracle_sha = hashlib.sha256(expected_oracle_bytes).hexdigest() - - # Track modified candidate file - subprocess.run(["git", "add", "research/canary-task-0/experiment.py"], cwd=repo, check=True) - - fake_storage = tmp_path / "fake_storage" - session_id = "canary-session" - bucket_name = "canary-bucket" - gcs_prefix = f"swarms/{session_id}" + deep_evidence = repo / "nested/domain/oracle/checks.json" + deep_evidence.parent.mkdir(parents=True) + raw_payload = json.dumps({"status": "verified", "checks": [1, 2, 3], "score": 0.98}, indent=2).encode("utf-8") + deep_evidence.write_bytes(raw_payload) + raw_sha = hashlib.sha256(raw_payload).hexdigest() - task_item = { - "task_index": 0, - "brief": "Run minimal Majorana algebra screen", - "candidate_files": ["research/canary-task-0/experiment.py"], - "evidence_files": [ - "research/canary-task-0/screen.raw.json", - "research/canary-task-0/oracle_result.json", - ], - "verify_command": f"{shlex.quote(sys.executable)} oracle.py", - "execution_limits": {"verify_timeout_s": 30, "command_timeout_s": 30}, - } + (repo / "solution.py").write_text("result = 42\n", encoding="utf-8") + subprocess.run(["git", "add", "solution.py"], cwd=repo, check=True) + fake_storage = tmp_path / "storage" + session_id, bucket = "canary-session", "canary-bucket" + prefix = f"swarms/{session_id}" monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) - # Perform artifact upload under the exact canary session layout uploaded = storage_uploader.upload_run_artifacts( - bucket_name=bucket_name, - prefix=gcs_prefix, + bucket_name=bucket, + prefix=prefix, repo_dir=repo, task_index=0, status="PASS", - candidate_files=task_item["candidate_files"], - evidence_files=task_item["evidence_files"], + candidate_files=["solution.py"], + evidence_files=["nested/domain/oracle/checks.json"], ) - # Verify object layout in fake storage: swarms//task-/ - task_storage = fake_storage / bucket_name / gcs_prefix / "task-0" - assert task_storage.is_dir() - assert (task_storage / "status.json").is_file() - assert (task_storage / "patch.diff").is_file() - storage_raw = task_storage / "evidence/research/canary-task-0/screen.raw.json" - storage_oracle = task_storage / "evidence/research/canary-task-0/oracle_result.json" - assert storage_raw.is_file() - assert storage_oracle.is_file() - - # Byte-exact preservation in storage - assert storage_raw.read_bytes() == expected_raw_bytes - assert hashlib.sha256(storage_raw.read_bytes()).hexdigest() == expected_raw_sha - assert storage_oracle.read_bytes() == expected_oracle_bytes - assert hashlib.sha256(storage_oracle.read_bytes()).hexdigest() == expected_oracle_sha - - # Harvest independently under the exact canary session - harvest_dest = tmp_path / "harvested" - harvested = result_harvester.harvest_candidate_patches( - bucket_name=bucket_name, - prefix=gcs_prefix, - dest_dir=harvest_dest, - ) + storage_evidence = fake_storage / bucket / prefix / "task-0/evidence/nested/domain/oracle/checks.json" + assert storage_evidence.is_file() + assert storage_evidence.read_bytes() == raw_payload + assert hashlib.sha256(storage_evidence.read_bytes()).hexdigest() == raw_sha + assert (fake_storage / bucket / prefix / "task-0/status.json").is_file() + assert (fake_storage / bucket / prefix / "task-0/patch.diff").is_file() + harvest_dest = tmp_path / "harvest" + harvested = result_harvester.harvest_candidate_patches(bucket_name=bucket, prefix=prefix, dest_dir=harvest_dest) assert len(harvested) == 1 - item = harvested[0] - assert item["task_index"] == 0 - assert item["status"] == "PASS" - assert item["patch_file"] is not None and Path(item["patch_file"]).is_file() - - harvested_raw = harvest_dest / "task-0/evidence/research/canary-task-0/screen.raw.json" - harvested_oracle = harvest_dest / "task-0/evidence/research/canary-task-0/oracle_result.json" - assert harvested_raw.is_file() - assert harvested_oracle.is_file() - - # Assert exact byte preservation after transport and harvesting - assert harvested_raw.read_bytes() == expected_raw_bytes - assert hashlib.sha256(harvested_raw.read_bytes()).hexdigest() == expected_raw_sha - assert harvested_oracle.read_bytes() == expected_oracle_bytes - assert hashlib.sha256(harvested_oracle.read_bytes()).hexdigest() == expected_oracle_sha - + assert harvested[0]["task_index"] == 0 + assert harvested[0]["status"] == "PASS" + assert harvested[0]["patch_file"] is not None and Path(harvested[0]["patch_file"]).is_file() -def test_case_2_wrong_scientific_result_fails_verification_and_degrades_to_issues(tmp_path, monkeypatch): - """Case 2: An intentionally wrong scientific result fails verification even when the agent + harvested_evidence = harvest_dest / "task-0/evidence/nested/domain/oracle/checks.json" + assert harvested_evidence.is_file() + assert harvested_evidence.read_bytes() == raw_payload + assert hashlib.sha256(harvested_evidence.read_bytes()).hexdigest() == raw_sha - requests PASS and makes no source change. Status is degraded to ISSUES and harvested status is ISSUES. - """ - repo = tmp_path / "canary_worker_repo_c2" - repo.mkdir() - _init_git_repo(repo) - task_dir = repo / "research/canary-task-0" - task_dir.mkdir(parents=True) - (task_dir / "oracle.py").write_text(DOMAIN_ORACLE_SCRIPT, encoding="utf-8") - - # Intentionally corrupt the raw scientific output (violations = 2) - corrupted_data = { - "size": 2, - "seed": 42, - "candidate_weight": 6, - "baseline_weight": 6, - "violations": 2, - "majoranas": [{"x": 1, "z": 0}, {"x": 1, "z": 0}, {"x": 2, "z": 1}, {"x": 2, "z": 3}], - } - (task_dir / "screen.raw.json").write_text(json.dumps(corrupted_data, indent=2), encoding="utf-8") - - # No source changes made to candidate files - fake_storage = tmp_path / "fake_storage" - session_id = "canary-session" - bucket_name = "canary-bucket" - gcs_prefix = f"swarms/{session_id}" +def test_failed_verification_degrades_to_issues_and_quarantines_patch(tmp_path, monkeypatch): + fake_storage = tmp_path / "storage_c2" + session_id, bucket = "canary-session-c2", "canary-bucket" + prefix = f"swarms/{session_id}" task_item = { "task_index": 0, - "brief": "Run screen with corrupted algebra", - "candidate_files": ["research/canary-task-0/calc.py"], - "evidence_files": ["research/canary-task-0/screen.raw.json"], - "verify_command": f"{shlex.quote(sys.executable)} research/canary-task-0/oracle.py", + "brief": "Run task with failing verification", + "candidate_files": ["research/task-0/calc.py"], + "evidence_files": [], + "verify_command": f"{shlex.quote(sys.executable)} research/task-0/oracle.py", "execution_limits": {"verify_timeout_s": 10}, } - monkeypatch.setenv("REPO_URL", "https://example.com/repo") - monkeypatch.setenv("TASK_MANIFEST", json.dumps([task_item])) - monkeypatch.setenv("USE_VERTEX_AI", "1") - monkeypatch.setenv("CLOUD_RUN_TASK_INDEX", "0") - monkeypatch.setenv("SWARM_SESSION_ID", session_id) - monkeypatch.setenv("GCS_RESULTS_BUCKET", bucket_name) - monkeypatch.setenv("GCS_BUCKET", bucket_name) - monkeypatch.setenv("GCS_PREFIX", gcs_prefix) - monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) + env_vars = { + "REPO_URL": "https://example.com/repo", + "TASK_MANIFEST": json.dumps([task_item]), + "USE_VERTEX_AI": "1", + "CLOUD_RUN_TASK_INDEX": "0", + "SWARM_SESSION_ID": session_id, + "GCS_RESULTS_BUCKET": bucket, + "GCS_BUCKET": bucket, + "GCS_PREFIX": prefix, + "STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage), + } + for k, v in env_vars.items(): + monkeypatch.setenv(k, v) monkeypatch.setattr(cloud_worker.tempfile, "gettempdir", lambda: str(tmp_path)) def fake_clone(**kwargs): dest_repo = kwargs["repo_dir"] _init_git_repo(dest_repo) - t_dir = dest_repo / "research/canary-task-0" + t_dir = dest_repo / "research/task-0" t_dir.mkdir(parents=True, exist_ok=True) - (t_dir / "oracle.py").write_text(DOMAIN_ORACLE_SCRIPT, encoding="utf-8") - corrupted_data = { - "size": 2, - "seed": 42, - "candidate_weight": 6, - "baseline_weight": 6, - "violations": 2, - "majoranas": [{"x": 1, "z": 0}, {"x": 1, "z": 0}, {"x": 2, "z": 1}, {"x": 2, "z": 3}], - } - (t_dir / "screen.raw.json").write_text(json.dumps(corrupted_data, indent=2), encoding="utf-8") - - # Mock execute_task so agent requests PASS without modifying candidate files - def fake_execute(**kwargs): - return "PASS", "Agent claimed pass with no source changes." + (t_dir / "oracle.py").write_text("import sys; sys.exit(1)\n", encoding="utf-8") monkeypatch.setattr(cloud_worker, "clone_and_checkout_task", fake_clone) - monkeypatch.setattr(cloud_worker, "execute_task", fake_execute) + monkeypatch.setattr(cloud_worker, "execute_task", lambda **kw: ("PASS", "Agent claimed pass.")) monkeypatch.setattr(cloud_worker, "StorageMessenger", None) orig_cwd = os.getcwd() try: - # cloud_worker.main() should run, run verify_command which fails, degrade to ISSUES, and exit(1) with pytest.raises(SystemExit) as exc_info: cloud_worker.main() assert exc_info.value.code == 1 finally: os.chdir(orig_cwd) - # Check status.json locally in the task repo and in storage - active_repo = tmp_path / "repo-task-0" - status_file = active_repo / "status.json" - assert status_file.is_file() - status_data = json.loads(status_file.read_text(encoding="utf-8")) + status_data = json.loads((tmp_path / "repo-task-0/status.json").read_text(encoding="utf-8")) assert status_data["status"] == "ISSUES" assert "Verification:" in status_data["summary"] - # Harvest and verify that harvested status is ISSUES and patch is quarantined - harvest_dest = tmp_path / "harvested_c2" harvested = result_harvester.harvest_candidate_patches( - bucket_name=bucket_name, - prefix=gcs_prefix, - dest_dir=harvest_dest, + bucket_name=bucket, prefix=prefix, dest_dir=tmp_path / "harvest_c2" ) assert len(harvested) == 1 assert harvested[0]["task_index"] == 0 @@ -348,287 +140,76 @@ def fake_execute(**kwargs): assert harvested[0]["patch_file"] is None -def test_case_3_command_timeout_diagnostics_retained_no_orphans_issues(tmp_path): - """Case 3: A bounded command times out (e.g. 1s timeout on a 3s sleep). - - Diagnostics retain partial output, child processes terminate without orphans, status is not PASS. - """ - repo = tmp_path / "canary_worker_repo_c3" - repo.mkdir() - _init_git_repo(repo) - - child_script = ( - "import os, time; " - "open('child.pid', 'w').write(str(os.getpid())); " - "time.sleep(4); " - "open('survived.txt', 'w').write('orphaned');" - ) - command = ( - f"echo 'partial-stdout-line-1'; echo 'partial-stderr-line-1' >&2; " - f"{shlex.quote(sys.executable)} -c {shlex.quote(child_script)}" - ) - - res, output = run_bounded_shell(command, repo, timeout_s=1, label="canary-timeout") - - # Command timed out -> exit code 124 - assert res.returncode == 124 - assert "Command timed out after 1 seconds" in res.stderr - assert "partial-stdout-line-1" in output - assert "partial-stderr-line-1" in output - - # Diagnostics retained in log file - diag_logs = list((repo / ".agystack/diagnostics").glob("canary-timeout-*.log")) - assert len(diag_logs) == 1 - log_content = diag_logs[0].read_text(encoding="utf-8") - assert "partial-stdout-line-1" in log_content - assert "partial-stderr-line-1" in log_content - - # Child process was terminated without orphans - pid_file = repo / "child.pid" - assert pid_file.is_file() - child_pid = int(pid_file.read_text(encoding="utf-8").strip()) - - time.sleep(1.0) - assert not (repo / "survived.txt").exists() - - # Check process table: process should not be actively running - proc_check = subprocess.run(["ps", "-o", "stat=", "-p", str(child_pid)], capture_output=True, text=True) - stat = proc_check.stdout.strip() - assert not stat or stat.startswith("Z") # Dead or zombie - - -def test_case_4_synthetic_credentials_in_evidence_rejected_intact_on_disk_issues(tmp_path): - """Case 4: A synthetic credential (e.g. AIza... or ghp_... token pattern) in declared evidence - - is rejected. The safe artifact's bytes remain intact on disk, no credential appears in published objects - or diagnostics, delivery_errors records the error, and status is degraded to ISSUES. - """ - repo = tmp_path / "canary_worker_repo_c4" +@pytest.mark.parametrize( + "cand_files,fail_file,files,err_msg", + [ + ( + ["cand.py"], + "score.json", + {"cand.py": "x = 1\n", "score.json": json.dumps({"score": 0.95})}, + "score.json upload failed", + ), + ([], "results.tsv", {"results.tsv": "metric\tval\nscore\t0.9\n"}, "results.tsv upload failed"), + ], +) +def test_upload_failures_degrade_to_issues(tmp_path, cand_files, fail_file, files, err_msg): + fake_storage = tmp_path / "storage" + repo = tmp_path / "repo" repo.mkdir() _init_git_repo(repo) + for name, content in files.items(): + (repo / name).write_text(content, encoding="utf-8") - evidence_dir = repo / "research/canary-task-0" - evidence_dir.mkdir(parents=True) - - # 1. Safe artifact - safe_file = evidence_dir / "safe_computation.json" - safe_content = json.dumps({"metric": "majorana_parity", "value": 1.0, "clean": True}, indent=2) - safe_file.write_text(safe_content, encoding="utf-8") - safe_bytes_before = safe_file.read_bytes() - safe_sha_before = hashlib.sha256(safe_bytes_before).hexdigest() - - # 2. File with synthetic credentials - secret_ghp = "ghp_" + "C" * 36 - secret_aiza = "AIza" + "D" * 35 - cred_file = evidence_dir / "leaked_evidence.txt" - cred_content = f"Debug log containing sensitive tokens: {secret_ghp} and {secret_aiza}\n" - cred_file.write_text(cred_content, encoding="utf-8") - - fake_storage = tmp_path / "fake_storage" - session_id = "canary-session" - bucket_name = "canary-bucket" - gcs_prefix = f"swarms/{session_id}" - - with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage)}): - results = storage_uploader.upload_run_artifacts( - bucket_name=bucket_name, - prefix=gcs_prefix, - repo_dir=repo, - task_index=0, - status="PASS", - evidence_files=[ - "research/canary-task-0/safe_computation.json", - "research/canary-task-0/leaked_evidence.txt", - ], - ) - - # Safe artifact bytes remain intact on disk - assert safe_file.read_bytes() == safe_bytes_before - assert hashlib.sha256(safe_file.read_bytes()).hexdigest() == safe_sha_before - - # Credential file on disk is not silently modified or wiped - assert cred_file.read_text(encoding="utf-8") == cred_content - - # Credential file was REJECTED from uploaded artifacts - assert "evidence/research/canary-task-0/leaked_evidence.txt" not in results - assert "evidence/research/canary-task-0/safe_computation.json" in results - - # No credential appears anywhere in published objects in storage - task_storage = fake_storage / bucket_name / gcs_prefix / "task-0" - assert task_storage.is_dir() - for pub_file in task_storage.rglob("*"): - if pub_file.is_file(): - content = pub_file.read_text(encoding="utf-8", errors="ignore") - assert secret_ghp not in content - assert secret_aiza not in content - - # Diagnostics do not leak credentials - diag_dir = repo / ".agystack/diagnostics" - if diag_dir.is_dir(): - for dfile in diag_dir.glob("*.log"): - dcontent = dfile.read_text(encoding="utf-8", errors="ignore") - assert secret_ghp not in dcontent - assert secret_aiza not in dcontent - - # delivery_errors in status.json records the error and status is degraded to ISSUES - status_file = repo / "status.json" - assert status_file.is_file() - status_data = json.loads(status_file.read_text(encoding="utf-8")) - assert status_data["status"] == "ISSUES" - assert "delivery_errors" in status_data - assert any("contains credentials" in err for err in status_data["delivery_errors"]) - - # Harvested status is ISSUES - harvested = result_harvester.harvest_candidate_patches( - bucket_name=bucket_name, - prefix=gcs_prefix, - dest_dir=tmp_path / "harvest_c4", - ) - assert len(harvested) == 1 - assert harvested[0]["status"] == "ISSUES" - - -def test_case_5_failed_score_or_results_upload_produces_delivery_errors_and_issues(tmp_path): - """Case 5: A failed score or results upload produces delivery errors and ISSUES. - - Test both candidate files and no-change tasks. - """ - fake_storage = tmp_path / "fake_storage_c5" - bucket_name = "canary-bucket" - - # 5a: With candidate files - repo_cand = tmp_path / "canary_worker_repo_c5_cand" - repo_cand.mkdir() - _init_git_repo(repo_cand) - - (repo_cand / "experiment.py").write_text("updated = True\n", encoding="utf-8") - (repo_cand / "score.json").write_text(json.dumps({"score": 0.95}), encoding="utf-8") - (repo_cand / "results.tsv").write_text("metric\tvalue\nscore\t0.95\n", encoding="utf-8") - - def fail_score_upload(bucket, obj_name, file_path): - if "score.json" in str(obj_name): + def mock_upload(b, obj_name, file_path): + if fail_file in str(obj_name): return False - # Save to local storage for realistic harvesting - dest = fake_storage / bucket / obj_name + dest = fake_storage / b / obj_name dest.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(file_path, dest) return True - prefix_cand = "swarms/canary-session-cand" + slug = fail_file.replace(".", "-") + prefix = f"swarms/canary-{slug}" with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage)}): - with patch.object(storage_uploader, "upload_to_gcs", side_effect=fail_score_upload): - uploaded_cand = storage_uploader.upload_run_artifacts( - bucket_name=bucket_name, - prefix=prefix_cand, - repo_dir=repo_cand, + with patch.object(storage_uploader, "upload_to_gcs", side_effect=mock_upload): + uploaded = storage_uploader.upload_run_artifacts( + bucket_name="canary-bucket", + prefix=prefix, + repo_dir=repo, task_index=0, status="PASS", - candidate_files=["experiment.py"], + candidate_files=cand_files, ) + assert fail_file not in uploaded + status_data = json.loads((repo / "status.json").read_text(encoding="utf-8")) + assert status_data["status"] == "ISSUES" + assert any(err_msg in err for err in status_data["delivery_errors"]) - assert "score.json" not in uploaded_cand - status_file_cand = repo_cand / "status.json" - status_data_cand = json.loads(status_file_cand.read_text(encoding="utf-8")) - assert status_data_cand["status"] == "ISSUES" - assert "delivery_errors" in status_data_cand - assert any("score.json upload failed" in err for err in status_data_cand["delivery_errors"]) - - harvested_cand = result_harvester.harvest_candidate_patches( - bucket_name=bucket_name, - prefix=prefix_cand, - dest_dir=tmp_path / "harvest_c5_cand", - ) - assert len(harvested_cand) == 1 - assert harvested_cand[0]["status"] == "ISSUES" - # Non-PASS candidate patch is quarantined and not promoted - assert harvested_cand[0]["patch_file"] is None - - # 5b: No-change tasks (clean working directory, no candidate files modified) - repo_clean = tmp_path / "canary_worker_repo_c5_clean" - repo_clean.mkdir() - _init_git_repo(repo_clean) - - (repo_clean / "score.json").write_text(json.dumps({"score": 0.90}), encoding="utf-8") - (repo_clean / "results.tsv").write_text("metric\tvalue\nscore\t0.90\n", encoding="utf-8") - - def fail_results_upload(bucket, obj_name, file_path): - if "results.tsv" in str(obj_name): - return False - dest = fake_storage / bucket / obj_name - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(file_path, dest) - return True - - prefix_clean = "swarms/canary-session-clean" - with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage)}): - with patch.object(storage_uploader, "upload_to_gcs", side_effect=fail_results_upload): - uploaded_clean = storage_uploader.upload_run_artifacts( - bucket_name=bucket_name, - prefix=prefix_clean, - repo_dir=repo_clean, - task_index=0, - status="PASS", - candidate_files=[], - ) - - assert "results.tsv" not in uploaded_clean - status_file_clean = repo_clean / "status.json" - status_data_clean = json.loads(status_file_clean.read_text(encoding="utf-8")) - assert status_data_clean["status"] == "ISSUES" - assert "delivery_errors" in status_data_clean - assert any("results.tsv upload failed" in err for err in status_data_clean["delivery_errors"]) - - harvested_clean = result_harvester.harvest_candidate_patches( - bucket_name=bucket_name, - prefix=prefix_clean, - dest_dir=tmp_path / "harvest_c5_clean", - ) - assert len(harvested_clean) == 1 - assert harvested_clean[0]["status"] == "ISSUES" - assert harvested_clean[0]["patch_file"] is None - - - + harvested = result_harvester.harvest_candidate_patches("canary-bucket", prefix, tmp_path / "harvest") + assert len(harvested) == 1 + assert harvested[0]["status"] == "ISSUES" + assert harvested[0]["patch_file"] is None -def test_case_6_sibling_prefix_isolation(tmp_path, monkeypatch): - """Case 6: Sibling prefix isolation: swarms/canary-other/ cannot satisfy canary coverage - or contribute a promotable patch in result_harvester. - """ +def test_session_prefix_isolation_and_coverage(tmp_path, monkeypatch): fake_storage = tmp_path / "storage" - canary_dir = fake_storage / "bucket/swarms/canary/task-0" - canary_dir.mkdir(parents=True) - (canary_dir / "status.json").write_text(json.dumps({"task_index": 0, "status": "PASS", "score": 0.80})) - (canary_dir / "score.json").write_text(json.dumps({"score": 0.80})) - (canary_dir / "patch.diff").write_text("diff --git a/canary b/canary\n+canary") - - sibling_dir = fake_storage / "bucket/swarms/canary-other/task-1" - sibling_dir.mkdir(parents=True) - (sibling_dir / "status.json").write_text(json.dumps({"task_index": 1, "status": "PASS", "score": 0.99})) - (sibling_dir / "score.json").write_text(json.dumps({"score": 0.99})) - (sibling_dir / "patch.diff").write_text("diff --git a/sibling b/sibling\n+sibling") - - sibling_task0 = fake_storage / "bucket/swarms/canary-other/task-0" - sibling_task0.mkdir(parents=True) - (sibling_task0 / "status.json").write_text(json.dumps({"task_index": 0, "status": "PASS", "score": 0.99})) - monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) - # Harvest for swarms/canary - harvest_dest = tmp_path / "harvest" - harvested = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", harvest_dest) + for sub, idx, score, p_text in [ + ("canary/task-0", 0, 0.80, "diff --git a/canary b/canary\n+canary"), + ("canary-other/task-1", 1, 0.99, "diff --git a/sibling b/sibling\n+sibling"), + ]: + td = fake_storage / "bucket/swarms" / sub + td.mkdir(parents=True) + (td / "status.json").write_text(json.dumps({"task_index": idx, "status": "PASS", "score": score})) + (td / "patch.diff").write_text(p_text) - # Must contain ONLY task-0 from canary, score 0.80 + harvested = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", tmp_path / "harvest") assert len(harvested) == 1 - assert harvested[0]["task_index"] == 0 - assert harvested[0]["score"] == 0.80 + assert (harvested[0]["task_index"], harvested[0]["score"]) == (0, 0.80) assert Path(harvested[0]["patch_file"]).read_text() == "diff --git a/canary b/canary\n+canary" - - # Sibling prefix canary-other cannot satisfy canary coverage if 2 tasks are expected assert cloud_dispatch.failed_task_count(harvested, 2) == 1 - # Also test GCS mock with trailing-slash prefix isolation - monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) blobs = { "swarms/canary/task-0/status.json": json.dumps({"task_index": 0, "status": "PASS", "score": 0.80}), "swarms/canary/task-0/patch.diff": "canary patch", @@ -645,190 +226,19 @@ def download_to_filename(self, filename): Path(filename).write_text(self.payload, encoding="utf-8") mock_storage = MagicMock() - mock_storage.Client.return_value.bucket.return_value.list_blobs.side_effect = ( - lambda prefix="": [Blob(k, v) for k, v in blobs.items() if k.startswith(prefix)] - ) - - google_mod = types.ModuleType("google") - cloud_mod = types.ModuleType("google.cloud") - google_mod.cloud = cloud_mod - cloud_mod.storage = mock_storage + mock_storage.Client.return_value.bucket.return_value.list_blobs.side_effect = lambda prefix="": [ + Blob(k, v) for k, v in blobs.items() if k.startswith(prefix) + ] - with patch.dict(sys.modules, {"google": google_mod, "google.cloud": cloud_mod, "google.cloud.storage": mock_storage}): - harvest_gcs = tmp_path / "harvest_gcs" - gcs_results = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", harvest_gcs) + modules = { + "google": types.SimpleNamespace(cloud=types.SimpleNamespace(storage=mock_storage)), + "google.cloud": types.SimpleNamespace(storage=mock_storage), + "google.cloud.storage": mock_storage, + } + with patch.dict(sys.modules, modules): + monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) + gcs_results = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", tmp_path / "harvest_gcs") assert len(gcs_results) == 1 - assert gcs_results[0]["task_index"] == 0 - assert gcs_results[0]["score"] == 0.80 + assert (gcs_results[0]["task_index"], gcs_results[0]["score"]) == (0, 0.80) assert Path(gcs_results[0]["patch_file"]).read_text() == "canary patch" - - -def test_task_level_execution_limits_override_cli_env_runtime_defaults(): - """Verify that task-level execution limits override CLI, environment, runtime config, and defaults. - - Also verify immutability and rejection of invalid values. - """ - # 1. Defaults - default_limits = resolve_execution_limits(environ={}) - assert (default_limits.max_agent_turns, default_limits.command_timeout_s, default_limits.verify_timeout_s) == (30, 300, 300) - - # 2. Runtime overrides defaults - runtime_limits = resolve_execution_limits( - environ={}, - runtime={"max_agent_turns": 40, "command_timeout_s": 350}, - ) - assert (runtime_limits.max_agent_turns, runtime_limits.command_timeout_s, runtime_limits.verify_timeout_s) == (40, 350, 300) - - # 3. Environment overrides runtime - env_limits = resolve_execution_limits( - environ={"AGYSTACK_MAX_AGENT_TURNS": "50", "COMMAND_TIMEOUT": "450"}, - runtime={"max_agent_turns": 40, "command_timeout_s": 350}, - ) - assert (env_limits.max_agent_turns, env_limits.command_timeout_s, env_limits.verify_timeout_s) == (50, 450, 300) - - # 4. CLI overrides environment - cli_limits = resolve_execution_limits( - environ={"AGYSTACK_MAX_AGENT_TURNS": "50", "COMMAND_TIMEOUT": "450"}, - runtime={"max_agent_turns": 40, "command_timeout_s": 350}, - cli={"max_agent_turns": 60}, - ) - assert (cli_limits.max_agent_turns, cli_limits.command_timeout_s, cli_limits.verify_timeout_s) == (60, 450, 300) - - # 5. Task-level limits override CLI, environment, runtime, and defaults - task_limits = resolve_execution_limits( - {"max_agent_turns": 80, "command_timeout_s": 700, "verify_timeout_s": 900}, - environ={"AGYSTACK_MAX_AGENT_TURNS": "50", "COMMAND_TIMEOUT": "450", "VERIFY_TIMEOUT": "550"}, - runtime={"max_agent_turns": 40, "command_timeout_s": 350, "verify_timeout_s": 400}, - cli={"max_agent_turns": 60, "command_timeout_s": 500}, - ) - assert (task_limits.max_agent_turns, task_limits.command_timeout_s, task_limits.verify_timeout_s) == (80, 700, 900) - - # Immutability - with pytest.raises(FrozenInstanceError): - task_limits.max_agent_turns = 100 - - # Invalid values rejected - for invalid in [0, -1, True, False, 2.5, "ten", None]: - with pytest.raises(ValueError): - resolve_execution_limits({"max_agent_turns": invalid}, environ={}) - - -def test_fake_storage_reproduces_layout_trailing_slash_and_nested_paths(tmp_path, monkeypatch): - """Verify fake storage reproduces swarms//task-/ layout, trailing-slash list prefixes, - - and nested task paths with exact byte preservation. - """ - repo = tmp_path / "layout_repo" - repo.mkdir() - _init_git_repo(repo) - - # Create nested evidence directories and files - deep_evidence = repo / "nested/domain/oracle/checks.json" - deep_evidence.parent.mkdir(parents=True) - deep_content = json.dumps({"depth": 3, "oracle_verified": True}, indent=2) - deep_evidence.write_text(deep_content, encoding="utf-8") - - fake_storage = tmp_path / "fake_storage" - monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) - - session_id = "canary-layout-session" - bucket_name = "test-layout-bucket" - prefix = f"swarms/{session_id}" - - storage_uploader.upload_run_artifacts( - bucket_name=bucket_name, - prefix=prefix, - repo_dir=repo, - task_index=0, - status="PASS", - evidence_files=["nested/domain/oracle/checks.json"], - ) - - # Verify exact layout in fake storage - expected_storage_path = ( - fake_storage - / bucket_name - / prefix - / "task-0" - / "evidence/nested/domain/oracle/checks.json" - ) - assert expected_storage_path.is_file() - assert expected_storage_path.read_text(encoding="utf-8") == deep_content - - # Harvest nested paths - harvest_dest = tmp_path / "harvest_layout" - harvested = result_harvester.harvest_candidate_patches(bucket_name, prefix, harvest_dest) - assert len(harvested) == 1 - harvested_nested = harvest_dest / "task-0/evidence/nested/domain/oracle/checks.json" - assert harvested_nested.is_file() - assert harvested_nested.read_text(encoding="utf-8") == deep_content - - -def test_verify_upload_mapping_against_package_inventory(tmp_path): - """Verify proposed upload mapping locally against the package inventory (package-inventory.json). - - Checks that all listed files exist, sha256 hashes match on disk, and map cleanly to preparation prefixes. - """ - inventory_path = Path( - os.environ.get("AGYSTACK_CANARY_PACKAGE_INVENTORY") or CANARY_PACKAGE_INVENTORY - ) - if not inventory_path.is_file(): - fixture_dir = tmp_path / "canary_fixture" - fixture_dir.mkdir(parents=True, exist_ok=True) - packets_dir = fixture_dir / "packets" - packets_dir.mkdir(parents=True, exist_ok=True) - - inventory_data = {} - key_files = { - "bootstrap.py": "#!/usr/bin/env python3\nprint('bootstrapping')\n", - "verify_research.py": "#!/usr/bin/env python3\nprint('verifying')\n", - "EVIDENCE_SCHEMA.md": "# Evidence Schema\n", - } - for filename, content in key_files.items(): - file_path = fixture_dir / filename - file_path.write_text(content, encoding="utf-8") - inventory_data[filename] = hashlib.sha256(file_path.read_bytes()).hexdigest() - - for i in range(1, 25): - packet_name = f"packets/task-{i:02d}.md" - packet_path = fixture_dir / packet_name - packet_path.write_text(f"# Task {i:02d}\n", encoding="utf-8") - inventory_data[packet_name] = hashlib.sha256(packet_path.read_bytes()).hexdigest() - - inventory_path = fixture_dir / "package-inventory.json" - inventory_path.write_text(json.dumps(inventory_data, indent=2), encoding="utf-8") - revision_dir = fixture_dir - else: - revision_dir = inventory_path.parent - - assert inventory_path.is_file(), f"Package inventory missing at {inventory_path}" - inventory = json.loads(inventory_path.read_text(encoding="utf-8")) - assert len(inventory) >= 27, f"Expected at least 27 inventory files, found {len(inventory)}" - - # Required key files - assert "bootstrap.py" in inventory - assert "verify_research.py" in inventory - assert "EVIDENCE_SCHEMA.md" in inventory - assert any(re.match(r"^packets/task-\d{2}\.md$", k) for k in inventory) - - canary_session = "phasepoly24-canary-20261025" - preparation_prefix = f"swarms/{canary_session}/preparation" - - upload_mapping = {} - for rel_path, expected_sha in inventory.items(): - file_path = revision_dir / rel_path - assert file_path.is_file(), f"Inventory item {rel_path} does not exist at {file_path}" - actual_sha = hashlib.sha256(file_path.read_bytes()).hexdigest() - assert actual_sha == expected_sha, ( - f"Hash mismatch for {rel_path}: expected {expected_sha}, got {actual_sha}" - ) - # Proposed destination in bucket - dest_blob = f"{preparation_prefix}/{rel_path}" - upload_mapping[rel_path] = dest_blob - - # Verify every destination blob has proper prefix and valid layout - for rel_path, dest_blob in upload_mapping.items(): - assert dest_blob.startswith(f"{preparation_prefix}/") - assert ".." not in dest_blob - assert not dest_blob.endswith("/") diff --git a/tests/test_storage_uploader.py b/tests/test_storage_uploader.py index 7dfcf50..6b596eb 100644 --- a/tests/test_storage_uploader.py +++ b/tests/test_storage_uploader.py @@ -214,11 +214,22 @@ def tearDown(self): @patch("storage_uploader.upload_to_gcs") def test_evidence_with_credentials_rejected_and_degrades_pass(self, mock_upload): - mock_upload.return_value = True + uploaded_payloads = {} + + def spy_upload(bucket, obj, path): + uploaded_payloads[obj] = Path(path).read_bytes() + return True + + mock_upload.side_effect = spy_upload + + safe_file = self.repo_dir / "safe.txt" + safe_bytes = b"safe measurement: 42\n" + safe_file.write_bytes(safe_bytes) evidence_file = self.repo_dir / "proof.txt" secret_token = "ghp_" + "a" * 36 - evidence_file.write_text(f"run log with sensitive token: {secret_token}\n", encoding="utf-8") + cred_bytes = f"run log with sensitive token: {secret_token}\n".encode("utf-8") + evidence_file.write_bytes(cred_bytes) results = storage_uploader.upload_run_artifacts( bucket_name="test-bucket", @@ -226,15 +237,26 @@ def test_evidence_with_credentials_rejected_and_degrades_pass(self, mock_upload) repo_dir=self.repo_dir, task_index=0, status="PASS", - evidence_files=["proof.txt"], + evidence_files=["safe.txt", "proof.txt"], ) - # File content must NOT have been modified - self.assertIn(secret_token, evidence_file.read_text(encoding="utf-8")) - # Evidence file must not be in uploaded artifacts + self.assertEqual(safe_file.read_bytes(), safe_bytes) + self.assertEqual(evidence_file.read_bytes(), cred_bytes) self.assertNotIn("evidence/proof.txt", results) + self.assertIn("evidence/safe.txt", results) + self.assertEqual( + uploaded_payloads.get("runs/exp1/task-0/evidence/safe.txt"), + safe_bytes, + ) + + secret_bytes = secret_token.encode("utf-8") + for obj, payload in uploaded_payloads.items(): + self.assertNotIn( + secret_bytes, + payload, + f"Synthetic secret leaked in outbound upload for {obj}", + ) - # status.json locally must degrade from PASS to ISSUES status_file = self.repo_dir / "status.json" self.assertTrue(status_file.is_file()) status_data = json.loads(status_file.read_text(encoding="utf-8")) diff --git a/tests/test_worker_execution_lifecycle.py b/tests/test_worker_execution_lifecycle.py index 0186e01..6a5cf81 100644 --- a/tests/test_worker_execution_lifecycle.py +++ b/tests/test_worker_execution_lifecycle.py @@ -281,7 +281,8 @@ def test_dispatch_rejects_invalid_task_limits_before_launch(tmp_path, limits): "--manifest", json.dumps([{"brief": "work", "execution_limits": limits}]), ] - result = subprocess.run(command, cwd=tmp_path, capture_output=True, text=True) + env = dict(os.environ, GH_TOKEN="fake-test-token") + result = subprocess.run(command, cwd=tmp_path, env=env, capture_output=True, text=True) assert result.returncode != 0 assert "execution_limits" in result.stderr or "positive integer" in result.stderr @@ -290,7 +291,7 @@ def test_dispatch_cli_env_runtime_precedence(tmp_path): (tmp_path / "agystack-runtime.json").write_text( json.dumps({"execution_limits": {"max_agent_turns": 40, "command_timeout_s": 500, "verify_timeout_s": 501}}) ) - env = dict(os.environ, AGYSTACK_MAX_AGENT_TURNS="41", COMMAND_TIMEOUT="600") + env = dict(os.environ, AGYSTACK_MAX_AGENT_TURNS="41", COMMAND_TIMEOUT="600", GH_TOKEN="fake-test-token") env.pop("VERIFY_TIMEOUT", None) result = subprocess.run( [ From 1bd5adc987df02fcb2939788d7f661f1656146ff Mon Sep 17 00:00:00 2001 From: jtaroreh Date: Sun, 25 Oct 2026 05:45:58 -0700 Subject: [PATCH 09/11] Test: Add end-to-end generated canary manifest lifecycle and negative boundary tests --- tests/test_cloud_swarm_canary_integration.py | 1504 ++++++++++++++++-- 1 file changed, 1384 insertions(+), 120 deletions(-) diff --git a/tests/test_cloud_swarm_canary_integration.py b/tests/test_cloud_swarm_canary_integration.py index 466abb9..33ef1f7 100644 --- a/tests/test_cloud_swarm_canary_integration.py +++ b/tests/test_cloud_swarm_canary_integration.py @@ -1,11 +1,19 @@ +import ast +import copy +from dataclasses import FrozenInstanceError import hashlib +import importlib.util +import io import json import os from pathlib import Path +import re import shlex import shutil import subprocess import sys +import tarfile +import time import types from unittest.mock import MagicMock, patch @@ -18,121 +26,325 @@ import cloud_worker import result_harvester import storage_uploader +from worker.execution import ExecutionLimits, resolve_execution_limits, run_bounded_shell + +CANARY_PACKAGE_INVENTORY = Path( + os.environ.get( + "AGYSTACK_CANARY_PACKAGE_INVENTORY", + "/Users/joeltaroreh/projects/phasepoly-swarm-prep/.audit/cloud-swarm-20261024/revision3/package-inventory.json", + ) +) +CANARY_REVISION3_DIR = CANARY_PACKAGE_INVENTORY.parent def _init_git_repo(path: Path) -> Path: subprocess.run(["git", "init"], cwd=path, check=True, capture_output=True) - subprocess.run(["git", "config", "user.name", "Tester"], cwd=path, check=True, capture_output=True) - subprocess.run(["git", "config", "user.email", "t@test.local"], cwd=path, check=True, capture_output=True) - (path / "base.txt").write_text("base\n", encoding="utf-8") + subprocess.run(["git", "config", "user.name", "Canary Tester"], cwd=path, check=True, capture_output=True) + subprocess.run(["git", "config", "user.email", "canary@test.local"], cwd=path, check=True, capture_output=True) + (path / "base.txt").write_text("initial base\n", encoding="utf-8") subprocess.run(["git", "add", "base.txt"], cwd=path, check=True, capture_output=True) - subprocess.run(["git", "commit", "-m", "init"], cwd=path, check=True, capture_output=True) + subprocess.run(["git", "commit", "-m", "init base"], cwd=path, check=True, capture_output=True) return path -def test_end_to_end_safe_evidence_transport_and_nested_layout(tmp_path, monkeypatch): - repo = tmp_path / "repo" +EXPERIMENT_SCRIPT = """import json, sys +from pathlib import Path + +# Jordan-Wigner Majorana representation for 2 modes (4 operators) +# gamma_0 = X_0, gamma_1 = Y_0, gamma_2 = Z_0 X_1, gamma_3 = Z_0 Y_1 +majoranas = [ + {"x": 1, "z": 0}, + {"x": 1, "z": 1}, + {"x": 2, "z": 1}, + {"x": 2, "z": 3}, +] + +# Check anticommutation violations: (x1*z2 + z1*x2)%2 must be 1 for distinct pairs +violations = sum( + (bin(a["x"] & b["z"]).count("1") + bin(a["z"] & b["x"]).count("1")) % 2 != 1 + for i, a in enumerate(majoranas) + for b in majoranas[i + 1:] +) + +candidate_weight = sum(bin(m["x"] | m["z"]).count("1") for m in majoranas) +baseline_weight = 2 * (2 + 1) + +output_data = { + "size": 2, + "seed": 42, + "candidate_weight": candidate_weight, + "baseline_weight": baseline_weight, + "violations": violations, + "majoranas": majoranas, +} + +out_path = Path("screen.raw.json") +out_path.write_text(json.dumps(output_data, indent=2), encoding="utf-8") +print(f"Computed Majorana algebra: violations={violations}, weight={candidate_weight}") +""" + +DOMAIN_ORACLE_SCRIPT = """import json, sys +from pathlib import Path + +raw_file = Path("screen.raw.json") +if not raw_file.is_file(): + print("Error: screen.raw.json missing", file=sys.stderr) + sys.exit(1) + +data = json.loads(raw_file.read_text(encoding="utf-8")) +if data.get("violations", -1) != 0: + print(f"Oracle verification failed: violations={data.get('violations')}", file=sys.stderr) + sys.exit(1) + +# Exact matrix verification of 4x4 Majorana anticommutators +I = [[1, 0], [0, 1]] +X = [[0, 1], [1, 0]] +Y = [[0, -1j], [1j, 0]] +Z = [[1, 0], [0, -1]] + +def tensor(a, b): + return [[a[i // len(b)][j // len(b)] * b[i % len(b)][j % len(b)] + for j in range(len(a) * len(b))] for i in range(len(a) * len(b))] + +def mat_mul(a, b): + n = len(a) + return [[sum(a[i][k] * b[k][j] for k in range(n)) for j in range(n)] for i in range(n)] + +def matrix(mask): + out = [[1]] + for k in range(2): + x, z = (mask["x"] >> k) & 1, (mask["z"] >> k) & 1 + out = tensor(out, Y if x and z else X if x else Z if z else I) + return out + +matrices = [matrix(m) for m in data["majoranas"]] +sample_count = 0 +for i, a in enumerate(matrices): + for j, b in enumerate(matrices): + ab, ba = mat_mul(a, b), mat_mul(b, a) + expected = 2 if i == j else 0 + for r in range(4): + for c in range(4): + val = ab[r][c] + ba[r][c] + target = expected if r == c else 0 + if abs(val - target) > 1e-10: + print(f"Algebra violation at ({i},{j}) element ({r},{c}): {val} != {target}", file=sys.stderr) + sys.exit(1) + sample_count += 1 + +oracle_res = { + "checks": [ + {"name": "majorana_anticommutators_4x4", "passed": True, "sample_count": sample_count} + ] +} +Path("oracle_result.json").write_text(json.dumps(oracle_res, indent=2), encoding="utf-8") +print(f"Oracle verified {sample_count} matrix checks cleanly.") +""" + + +def test_case_1_real_scientific_computation_harvests_exact_bytes(tmp_path, monkeypatch): + """Case 1: A tiny real scientific computation produces raw numeric output and a sound domain oracle. + + Safe evidence is delivered and independently harvested under the exact canary session. + Assert harvested status is PASS, raw outputs and evidence files are preserved with exact bytes. + """ + repo = tmp_path / "canary_worker_repo" repo.mkdir() _init_git_repo(repo) - deep_evidence = repo / "nested/domain/oracle/checks.json" - deep_evidence.parent.mkdir(parents=True) - raw_payload = json.dumps({"status": "verified", "checks": [1, 2, 3], "score": 0.98}, indent=2).encode("utf-8") - deep_evidence.write_bytes(raw_payload) - raw_sha = hashlib.sha256(raw_payload).hexdigest() + task_dir = repo / "research/canary-task-0" + task_dir.mkdir(parents=True) + (task_dir / "experiment.py").write_text(EXPERIMENT_SCRIPT, encoding="utf-8") + (task_dir / "oracle.py").write_text(DOMAIN_ORACLE_SCRIPT, encoding="utf-8") - (repo / "solution.py").write_text("result = 42\n", encoding="utf-8") - subprocess.run(["git", "add", "solution.py"], cwd=repo, check=True) + # Run the real scientific computation + calc_res = subprocess.run([sys.executable, "experiment.py"], cwd=task_dir, capture_output=True, text=True, check=True) + assert "Computed Majorana algebra: violations=0" in calc_res.stdout + raw_path = task_dir / "screen.raw.json" + assert raw_path.is_file() + expected_raw_bytes = raw_path.read_bytes() + expected_raw_sha = hashlib.sha256(expected_raw_bytes).hexdigest() + + # Run the sound domain oracle + oracle_res = subprocess.run([sys.executable, "oracle.py"], cwd=task_dir, capture_output=True, text=True, check=True) + assert "Oracle verified 16 matrix checks cleanly." in oracle_res.stdout + oracle_path = task_dir / "oracle_result.json" + assert oracle_path.is_file() + expected_oracle_bytes = oracle_path.read_bytes() + expected_oracle_sha = hashlib.sha256(expected_oracle_bytes).hexdigest() + + # Track modified candidate file + subprocess.run(["git", "add", "research/canary-task-0/experiment.py"], cwd=repo, check=True) + + fake_storage = tmp_path / "fake_storage" + session_id = "canary-session" + bucket_name = "canary-bucket" + gcs_prefix = f"swarms/{session_id}" + + task_item = { + "task_index": 0, + "brief": "Run minimal Majorana algebra screen", + "candidate_files": ["research/canary-task-0/experiment.py"], + "evidence_files": [ + "research/canary-task-0/screen.raw.json", + "research/canary-task-0/oracle_result.json", + ], + "verify_command": f"{shlex.quote(sys.executable)} oracle.py", + "execution_limits": {"verify_timeout_s": 30, "command_timeout_s": 30}, + } - fake_storage = tmp_path / "storage" - session_id, bucket = "canary-session", "canary-bucket" - prefix = f"swarms/{session_id}" monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) + # Perform artifact upload under the exact canary session layout uploaded = storage_uploader.upload_run_artifacts( - bucket_name=bucket, - prefix=prefix, + bucket_name=bucket_name, + prefix=gcs_prefix, repo_dir=repo, task_index=0, status="PASS", - candidate_files=["solution.py"], - evidence_files=["nested/domain/oracle/checks.json"], + candidate_files=task_item["candidate_files"], + evidence_files=task_item["evidence_files"], ) - storage_evidence = fake_storage / bucket / prefix / "task-0/evidence/nested/domain/oracle/checks.json" - assert storage_evidence.is_file() - assert storage_evidence.read_bytes() == raw_payload - assert hashlib.sha256(storage_evidence.read_bytes()).hexdigest() == raw_sha - assert (fake_storage / bucket / prefix / "task-0/status.json").is_file() - assert (fake_storage / bucket / prefix / "task-0/patch.diff").is_file() + # Verify object layout in fake storage: swarms//task-/ + task_storage = fake_storage / bucket_name / gcs_prefix / "task-0" + assert task_storage.is_dir() + assert (task_storage / "status.json").is_file() + assert (task_storage / "patch.diff").is_file() + storage_raw = task_storage / "evidence/research/canary-task-0/screen.raw.json" + storage_oracle = task_storage / "evidence/research/canary-task-0/oracle_result.json" + assert storage_raw.is_file() + assert storage_oracle.is_file() + + # Byte-exact preservation in storage + assert storage_raw.read_bytes() == expected_raw_bytes + assert hashlib.sha256(storage_raw.read_bytes()).hexdigest() == expected_raw_sha + assert storage_oracle.read_bytes() == expected_oracle_bytes + assert hashlib.sha256(storage_oracle.read_bytes()).hexdigest() == expected_oracle_sha + + # Harvest independently under the exact canary session + harvest_dest = tmp_path / "harvested" + harvested = result_harvester.harvest_candidate_patches( + bucket_name=bucket_name, + prefix=gcs_prefix, + dest_dir=harvest_dest, + ) - harvest_dest = tmp_path / "harvest" - harvested = result_harvester.harvest_candidate_patches(bucket_name=bucket, prefix=prefix, dest_dir=harvest_dest) assert len(harvested) == 1 - assert harvested[0]["task_index"] == 0 - assert harvested[0]["status"] == "PASS" - assert harvested[0]["patch_file"] is not None and Path(harvested[0]["patch_file"]).is_file() + item = harvested[0] + assert item["task_index"] == 0 + assert item["status"] == "PASS" + assert item["patch_file"] is not None and Path(item["patch_file"]).is_file() - harvested_evidence = harvest_dest / "task-0/evidence/nested/domain/oracle/checks.json" - assert harvested_evidence.is_file() - assert harvested_evidence.read_bytes() == raw_payload - assert hashlib.sha256(harvested_evidence.read_bytes()).hexdigest() == raw_sha + harvested_raw = harvest_dest / "task-0/evidence/research/canary-task-0/screen.raw.json" + harvested_oracle = harvest_dest / "task-0/evidence/research/canary-task-0/oracle_result.json" + assert harvested_raw.is_file() + assert harvested_oracle.is_file() + # Assert exact byte preservation after transport and harvesting + assert harvested_raw.read_bytes() == expected_raw_bytes + assert hashlib.sha256(harvested_raw.read_bytes()).hexdigest() == expected_raw_sha + assert harvested_oracle.read_bytes() == expected_oracle_bytes + assert hashlib.sha256(harvested_oracle.read_bytes()).hexdigest() == expected_oracle_sha -def test_failed_verification_degrades_to_issues_and_quarantines_patch(tmp_path, monkeypatch): - fake_storage = tmp_path / "storage_c2" - session_id, bucket = "canary-session-c2", "canary-bucket" - prefix = f"swarms/{session_id}" + +def test_case_2_wrong_scientific_result_fails_verification_and_degrades_to_issues(tmp_path, monkeypatch): + """Case 2: An intentionally wrong scientific result fails verification even when the agent + + requests PASS and makes no source change. Status is degraded to ISSUES and harvested status is ISSUES. + """ + repo = tmp_path / "canary_worker_repo_c2" + repo.mkdir() + _init_git_repo(repo) + + task_dir = repo / "research/canary-task-0" + task_dir.mkdir(parents=True) + (task_dir / "oracle.py").write_text(DOMAIN_ORACLE_SCRIPT, encoding="utf-8") + + # Intentionally corrupt the raw scientific output (violations = 2) + corrupted_data = { + "size": 2, + "seed": 42, + "candidate_weight": 6, + "baseline_weight": 6, + "violations": 2, + "majoranas": [{"x": 1, "z": 0}, {"x": 1, "z": 0}, {"x": 2, "z": 1}, {"x": 2, "z": 3}], + } + (task_dir / "screen.raw.json").write_text(json.dumps(corrupted_data, indent=2), encoding="utf-8") + + # No source changes made to candidate files + fake_storage = tmp_path / "fake_storage" + session_id = "canary-session" + bucket_name = "canary-bucket" + gcs_prefix = f"swarms/{session_id}" task_item = { "task_index": 0, - "brief": "Run task with failing verification", - "candidate_files": ["research/task-0/calc.py"], - "evidence_files": [], - "verify_command": f"{shlex.quote(sys.executable)} research/task-0/oracle.py", + "brief": "Run screen with corrupted algebra", + "candidate_files": ["research/canary-task-0/calc.py"], + "evidence_files": ["research/canary-task-0/screen.raw.json"], + "verify_command": f"{shlex.quote(sys.executable)} research/canary-task-0/oracle.py", "execution_limits": {"verify_timeout_s": 10}, } - env_vars = { - "REPO_URL": "https://example.com/repo", - "TASK_MANIFEST": json.dumps([task_item]), - "USE_VERTEX_AI": "1", - "CLOUD_RUN_TASK_INDEX": "0", - "SWARM_SESSION_ID": session_id, - "GCS_RESULTS_BUCKET": bucket, - "GCS_BUCKET": bucket, - "GCS_PREFIX": prefix, - "STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage), - } - for k, v in env_vars.items(): - monkeypatch.setenv(k, v) + monkeypatch.setenv("REPO_URL", "https://example.com/repo") + monkeypatch.setenv("TASK_MANIFEST", json.dumps([task_item])) + monkeypatch.setenv("USE_VERTEX_AI", "1") + monkeypatch.setenv("CLOUD_RUN_TASK_INDEX", "0") + monkeypatch.setenv("SWARM_SESSION_ID", session_id) + monkeypatch.setenv("GCS_RESULTS_BUCKET", bucket_name) + monkeypatch.setenv("GCS_BUCKET", bucket_name) + monkeypatch.setenv("GCS_PREFIX", gcs_prefix) + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) monkeypatch.setattr(cloud_worker.tempfile, "gettempdir", lambda: str(tmp_path)) def fake_clone(**kwargs): dest_repo = kwargs["repo_dir"] _init_git_repo(dest_repo) - t_dir = dest_repo / "research/task-0" + t_dir = dest_repo / "research/canary-task-0" t_dir.mkdir(parents=True, exist_ok=True) - (t_dir / "oracle.py").write_text("import sys; sys.exit(1)\n", encoding="utf-8") + (t_dir / "oracle.py").write_text(DOMAIN_ORACLE_SCRIPT, encoding="utf-8") + corrupted_data = { + "size": 2, + "seed": 42, + "candidate_weight": 6, + "baseline_weight": 6, + "violations": 2, + "majoranas": [{"x": 1, "z": 0}, {"x": 1, "z": 0}, {"x": 2, "z": 1}, {"x": 2, "z": 3}], + } + (t_dir / "screen.raw.json").write_text(json.dumps(corrupted_data, indent=2), encoding="utf-8") + + # Mock execute_task so agent requests PASS without modifying candidate files + def fake_execute(**kwargs): + return "PASS", "Agent claimed pass with no source changes." monkeypatch.setattr(cloud_worker, "clone_and_checkout_task", fake_clone) - monkeypatch.setattr(cloud_worker, "execute_task", lambda **kw: ("PASS", "Agent claimed pass.")) + monkeypatch.setattr(cloud_worker, "execute_task", fake_execute) monkeypatch.setattr(cloud_worker, "StorageMessenger", None) orig_cwd = os.getcwd() try: + # cloud_worker.main() should run, run verify_command which fails, degrade to ISSUES, and exit(1) with pytest.raises(SystemExit) as exc_info: cloud_worker.main() assert exc_info.value.code == 1 finally: os.chdir(orig_cwd) - status_data = json.loads((tmp_path / "repo-task-0/status.json").read_text(encoding="utf-8")) + # Check status.json locally in the task repo and in storage + active_repo = tmp_path / "repo-task-0" + status_file = active_repo / "status.json" + assert status_file.is_file() + status_data = json.loads(status_file.read_text(encoding="utf-8")) assert status_data["status"] == "ISSUES" assert "Verification:" in status_data["summary"] + # Harvest and verify that harvested status is ISSUES and patch is quarantined + harvest_dest = tmp_path / "harvested_c2" harvested = result_harvester.harvest_candidate_patches( - bucket_name=bucket, prefix=prefix, dest_dir=tmp_path / "harvest_c2" + bucket_name=bucket_name, + prefix=gcs_prefix, + dest_dir=harvest_dest, ) assert len(harvested) == 1 assert harvested[0]["task_index"] == 0 @@ -140,76 +352,287 @@ def fake_clone(**kwargs): assert harvested[0]["patch_file"] is None -@pytest.mark.parametrize( - "cand_files,fail_file,files,err_msg", - [ - ( - ["cand.py"], - "score.json", - {"cand.py": "x = 1\n", "score.json": json.dumps({"score": 0.95})}, - "score.json upload failed", - ), - ([], "results.tsv", {"results.tsv": "metric\tval\nscore\t0.9\n"}, "results.tsv upload failed"), - ], -) -def test_upload_failures_degrade_to_issues(tmp_path, cand_files, fail_file, files, err_msg): - fake_storage = tmp_path / "storage" - repo = tmp_path / "repo" +def test_case_3_command_timeout_diagnostics_retained_no_orphans_issues(tmp_path): + """Case 3: A bounded command times out (e.g. 1s timeout on a 3s sleep). + + Diagnostics retain partial output, child processes terminate without orphans, status is not PASS. + """ + repo = tmp_path / "canary_worker_repo_c3" repo.mkdir() _init_git_repo(repo) - for name, content in files.items(): - (repo / name).write_text(content, encoding="utf-8") - def mock_upload(b, obj_name, file_path): - if fail_file in str(obj_name): + child_script = ( + "import os, time; " + "open('child.pid', 'w').write(str(os.getpid())); " + "time.sleep(4); " + "open('survived.txt', 'w').write('orphaned');" + ) + command = ( + f"echo 'partial-stdout-line-1'; echo 'partial-stderr-line-1' >&2; " + f"{shlex.quote(sys.executable)} -c {shlex.quote(child_script)}" + ) + + res, output = run_bounded_shell(command, repo, timeout_s=1, label="canary-timeout") + + # Command timed out -> exit code 124 + assert res.returncode == 124 + assert "Command timed out after 1 seconds" in res.stderr + assert "partial-stdout-line-1" in output + assert "partial-stderr-line-1" in output + + # Diagnostics retained in log file + diag_logs = list((repo / ".agystack/diagnostics").glob("canary-timeout-*.log")) + assert len(diag_logs) == 1 + log_content = diag_logs[0].read_text(encoding="utf-8") + assert "partial-stdout-line-1" in log_content + assert "partial-stderr-line-1" in log_content + + # Child process was terminated without orphans + pid_file = repo / "child.pid" + assert pid_file.is_file() + child_pid = int(pid_file.read_text(encoding="utf-8").strip()) + + time.sleep(1.0) + assert not (repo / "survived.txt").exists() + + # Check process table: process should not be actively running + proc_check = subprocess.run(["ps", "-o", "stat=", "-p", str(child_pid)], capture_output=True, text=True) + stat = proc_check.stdout.strip() + assert not stat or stat.startswith("Z") # Dead or zombie + + +def test_case_4_synthetic_credentials_in_evidence_rejected_intact_on_disk_issues(tmp_path): + """Case 4: A synthetic credential (e.g. AIza... or ghp_... token pattern) in declared evidence + + is rejected. The safe artifact's bytes remain intact on disk, no credential appears in published objects + or diagnostics, delivery_errors records the error, and status is degraded to ISSUES. + """ + repo = tmp_path / "canary_worker_repo_c4" + repo.mkdir() + _init_git_repo(repo) + + evidence_dir = repo / "research/canary-task-0" + evidence_dir.mkdir(parents=True) + + # 1. Safe artifact + safe_file = evidence_dir / "safe_computation.json" + safe_content = json.dumps({"metric": "majorana_parity", "value": 1.0, "clean": True}, indent=2) + safe_file.write_text(safe_content, encoding="utf-8") + safe_bytes_before = safe_file.read_bytes() + safe_sha_before = hashlib.sha256(safe_bytes_before).hexdigest() + + # 2. File with synthetic credentials + secret_ghp = "ghp_" + "C" * 36 + secret_aiza = "AIza" + "D" * 35 + cred_file = evidence_dir / "leaked_evidence.txt" + cred_content = f"Debug log containing sensitive tokens: {secret_ghp} and {secret_aiza}\n" + cred_file.write_text(cred_content, encoding="utf-8") + + fake_storage = tmp_path / "fake_storage" + session_id = "canary-session" + bucket_name = "canary-bucket" + gcs_prefix = f"swarms/{session_id}" + + with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage)}): + results = storage_uploader.upload_run_artifacts( + bucket_name=bucket_name, + prefix=gcs_prefix, + repo_dir=repo, + task_index=0, + status="PASS", + evidence_files=[ + "research/canary-task-0/safe_computation.json", + "research/canary-task-0/leaked_evidence.txt", + ], + ) + + # Safe artifact bytes remain intact on disk + assert safe_file.read_bytes() == safe_bytes_before + assert hashlib.sha256(safe_file.read_bytes()).hexdigest() == safe_sha_before + + # Credential file on disk is not silently modified or wiped + assert cred_file.read_text(encoding="utf-8") == cred_content + + # Credential file was REJECTED from uploaded artifacts + assert "evidence/research/canary-task-0/leaked_evidence.txt" not in results + assert "evidence/research/canary-task-0/safe_computation.json" in results + + # No credential appears anywhere in published objects in storage + task_storage = fake_storage / bucket_name / gcs_prefix / "task-0" + assert task_storage.is_dir() + for pub_file in task_storage.rglob("*"): + if pub_file.is_file(): + content = pub_file.read_text(encoding="utf-8", errors="ignore") + assert secret_ghp not in content + assert secret_aiza not in content + + # Diagnostics do not leak credentials + diag_dir = repo / ".agystack/diagnostics" + if diag_dir.is_dir(): + for dfile in diag_dir.glob("*.log"): + dcontent = dfile.read_text(encoding="utf-8", errors="ignore") + assert secret_ghp not in dcontent + assert secret_aiza not in dcontent + + # delivery_errors in status.json records the error and status is degraded to ISSUES + status_file = repo / "status.json" + assert status_file.is_file() + status_data = json.loads(status_file.read_text(encoding="utf-8")) + assert status_data["status"] == "ISSUES" + assert "delivery_errors" in status_data + assert any("contains credentials" in err for err in status_data["delivery_errors"]) + + # Harvested status is ISSUES + harvested = result_harvester.harvest_candidate_patches( + bucket_name=bucket_name, + prefix=gcs_prefix, + dest_dir=tmp_path / "harvest_c4", + ) + assert len(harvested) == 1 + assert harvested[0]["status"] == "ISSUES" + + +def test_case_5_failed_score_or_results_upload_produces_delivery_errors_and_issues(tmp_path): + """Case 5: A failed score or results upload produces delivery errors and ISSUES. + + Test both candidate files and no-change tasks. + """ + fake_storage = tmp_path / "fake_storage_c5" + bucket_name = "canary-bucket" + + # 5a: With candidate files + repo_cand = tmp_path / "canary_worker_repo_c5_cand" + repo_cand.mkdir() + _init_git_repo(repo_cand) + + (repo_cand / "experiment.py").write_text("updated = True\n", encoding="utf-8") + (repo_cand / "score.json").write_text(json.dumps({"score": 0.95}), encoding="utf-8") + (repo_cand / "results.tsv").write_text("metric\tvalue\nscore\t0.95\n", encoding="utf-8") + + def fail_score_upload(bucket, obj_name, file_path): + if "score.json" in str(obj_name): return False - dest = fake_storage / b / obj_name + # Save to local storage for realistic harvesting + dest = fake_storage / bucket / obj_name dest.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(file_path, dest) return True - slug = fail_file.replace(".", "-") - prefix = f"swarms/canary-{slug}" + prefix_cand = "swarms/canary-session-cand" with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage)}): - with patch.object(storage_uploader, "upload_to_gcs", side_effect=mock_upload): - uploaded = storage_uploader.upload_run_artifacts( - bucket_name="canary-bucket", - prefix=prefix, - repo_dir=repo, + with patch.object(storage_uploader, "upload_to_gcs", side_effect=fail_score_upload): + uploaded_cand = storage_uploader.upload_run_artifacts( + bucket_name=bucket_name, + prefix=prefix_cand, + repo_dir=repo_cand, task_index=0, status="PASS", - candidate_files=cand_files, + candidate_files=["experiment.py"], ) - assert fail_file not in uploaded - status_data = json.loads((repo / "status.json").read_text(encoding="utf-8")) - assert status_data["status"] == "ISSUES" - assert any(err_msg in err for err in status_data["delivery_errors"]) - harvested = result_harvester.harvest_candidate_patches("canary-bucket", prefix, tmp_path / "harvest") - assert len(harvested) == 1 - assert harvested[0]["status"] == "ISSUES" - assert harvested[0]["patch_file"] is None + assert "score.json" not in uploaded_cand + status_file_cand = repo_cand / "status.json" + status_data_cand = json.loads(status_file_cand.read_text(encoding="utf-8")) + assert status_data_cand["status"] == "ISSUES" + assert "delivery_errors" in status_data_cand + assert any("score.json upload failed" in err for err in status_data_cand["delivery_errors"]) + + harvested_cand = result_harvester.harvest_candidate_patches( + bucket_name=bucket_name, + prefix=prefix_cand, + dest_dir=tmp_path / "harvest_c5_cand", + ) + assert len(harvested_cand) == 1 + assert harvested_cand[0]["status"] == "ISSUES" + # Non-PASS candidate patch is quarantined and not promoted + assert harvested_cand[0]["patch_file"] is None + + # 5b: No-change tasks (clean working directory, no candidate files modified) + repo_clean = tmp_path / "canary_worker_repo_c5_clean" + repo_clean.mkdir() + _init_git_repo(repo_clean) + (repo_clean / "score.json").write_text(json.dumps({"score": 0.90}), encoding="utf-8") + (repo_clean / "results.tsv").write_text("metric\tvalue\nscore\t0.90\n", encoding="utf-8") + + def fail_results_upload(bucket, obj_name, file_path): + if "results.tsv" in str(obj_name): + return False + dest = fake_storage / bucket / obj_name + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(file_path, dest) + return True -def test_session_prefix_isolation_and_coverage(tmp_path, monkeypatch): + prefix_clean = "swarms/canary-session-clean" + with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage)}): + with patch.object(storage_uploader, "upload_to_gcs", side_effect=fail_results_upload): + uploaded_clean = storage_uploader.upload_run_artifacts( + bucket_name=bucket_name, + prefix=prefix_clean, + repo_dir=repo_clean, + task_index=0, + status="PASS", + candidate_files=[], + ) + + assert "results.tsv" not in uploaded_clean + status_file_clean = repo_clean / "status.json" + status_data_clean = json.loads(status_file_clean.read_text(encoding="utf-8")) + assert status_data_clean["status"] == "ISSUES" + assert "delivery_errors" in status_data_clean + assert any("results.tsv upload failed" in err for err in status_data_clean["delivery_errors"]) + + harvested_clean = result_harvester.harvest_candidate_patches( + bucket_name=bucket_name, + prefix=prefix_clean, + dest_dir=tmp_path / "harvest_c5_clean", + ) + assert len(harvested_clean) == 1 + assert harvested_clean[0]["status"] == "ISSUES" + assert harvested_clean[0]["patch_file"] is None + + + + +def test_case_6_sibling_prefix_isolation(tmp_path, monkeypatch): + """Case 6: Sibling prefix isolation: swarms/canary-other/ cannot satisfy canary coverage + + or contribute a promotable patch in result_harvester. + """ fake_storage = tmp_path / "storage" + canary_dir = fake_storage / "bucket/swarms/canary/task-0" + canary_dir.mkdir(parents=True) + (canary_dir / "status.json").write_text(json.dumps({"task_index": 0, "status": "PASS", "score": 0.80})) + (canary_dir / "score.json").write_text(json.dumps({"score": 0.80})) + (canary_dir / "patch.diff").write_text("diff --git a/canary b/canary\n+canary") + + sibling_dir = fake_storage / "bucket/swarms/canary-other/task-1" + sibling_dir.mkdir(parents=True) + (sibling_dir / "status.json").write_text(json.dumps({"task_index": 1, "status": "PASS", "score": 0.99})) + (sibling_dir / "score.json").write_text(json.dumps({"score": 0.99})) + (sibling_dir / "patch.diff").write_text("diff --git a/sibling b/sibling\n+sibling") + + sibling_task0 = fake_storage / "bucket/swarms/canary-other/task-0" + sibling_task0.mkdir(parents=True) + (sibling_task0 / "status.json").write_text(json.dumps({"task_index": 0, "status": "PASS", "score": 0.99})) + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) - for sub, idx, score, p_text in [ - ("canary/task-0", 0, 0.80, "diff --git a/canary b/canary\n+canary"), - ("canary-other/task-1", 1, 0.99, "diff --git a/sibling b/sibling\n+sibling"), - ]: - td = fake_storage / "bucket/swarms" / sub - td.mkdir(parents=True) - (td / "status.json").write_text(json.dumps({"task_index": idx, "status": "PASS", "score": score})) - (td / "patch.diff").write_text(p_text) + # Harvest for swarms/canary + harvest_dest = tmp_path / "harvest" + harvested = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", harvest_dest) - harvested = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", tmp_path / "harvest") + # Must contain ONLY task-0 from canary, score 0.80 assert len(harvested) == 1 - assert (harvested[0]["task_index"], harvested[0]["score"]) == (0, 0.80) + assert harvested[0]["task_index"] == 0 + assert harvested[0]["score"] == 0.80 assert Path(harvested[0]["patch_file"]).read_text() == "diff --git a/canary b/canary\n+canary" + + # Sibling prefix canary-other cannot satisfy canary coverage if 2 tasks are expected assert cloud_dispatch.failed_task_count(harvested, 2) == 1 + # Also test GCS mock with trailing-slash prefix isolation + monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) blobs = { "swarms/canary/task-0/status.json": json.dumps({"task_index": 0, "status": "PASS", "score": 0.80}), "swarms/canary/task-0/patch.diff": "canary patch", @@ -226,19 +649,860 @@ def download_to_filename(self, filename): Path(filename).write_text(self.payload, encoding="utf-8") mock_storage = MagicMock() - mock_storage.Client.return_value.bucket.return_value.list_blobs.side_effect = lambda prefix="": [ - Blob(k, v) for k, v in blobs.items() if k.startswith(prefix) - ] + mock_storage.Client.return_value.bucket.return_value.list_blobs.side_effect = ( + lambda prefix="": [Blob(k, v) for k, v in blobs.items() if k.startswith(prefix)] + ) - modules = { - "google": types.SimpleNamespace(cloud=types.SimpleNamespace(storage=mock_storage)), - "google.cloud": types.SimpleNamespace(storage=mock_storage), - "google.cloud.storage": mock_storage, - } - with patch.dict(sys.modules, modules): - monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) - gcs_results = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", tmp_path / "harvest_gcs") + google_mod = types.ModuleType("google") + cloud_mod = types.ModuleType("google.cloud") + google_mod.cloud = cloud_mod + cloud_mod.storage = mock_storage + + with patch.dict(sys.modules, {"google": google_mod, "google.cloud": cloud_mod, "google.cloud.storage": mock_storage}): + harvest_gcs = tmp_path / "harvest_gcs" + gcs_results = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", harvest_gcs) assert len(gcs_results) == 1 - assert (gcs_results[0]["task_index"], gcs_results[0]["score"]) == (0, 0.80) + assert gcs_results[0]["task_index"] == 0 + assert gcs_results[0]["score"] == 0.80 assert Path(gcs_results[0]["patch_file"]).read_text() == "canary patch" + + +def test_task_level_execution_limits_override_cli_env_runtime_defaults(): + """Verify that task-level execution limits override CLI, environment, runtime config, and defaults. + + Also verify immutability and rejection of invalid values. + """ + # 1. Defaults + default_limits = resolve_execution_limits(environ={}) + assert (default_limits.max_agent_turns, default_limits.command_timeout_s, default_limits.verify_timeout_s) == (30, 300, 300) + + # 2. Runtime overrides defaults + runtime_limits = resolve_execution_limits( + environ={}, + runtime={"max_agent_turns": 40, "command_timeout_s": 350}, + ) + assert (runtime_limits.max_agent_turns, runtime_limits.command_timeout_s, runtime_limits.verify_timeout_s) == (40, 350, 300) + + # 3. Environment overrides runtime + env_limits = resolve_execution_limits( + environ={"AGYSTACK_MAX_AGENT_TURNS": "50", "COMMAND_TIMEOUT": "450"}, + runtime={"max_agent_turns": 40, "command_timeout_s": 350}, + ) + assert (env_limits.max_agent_turns, env_limits.command_timeout_s, env_limits.verify_timeout_s) == (50, 450, 300) + + # 4. CLI overrides environment + cli_limits = resolve_execution_limits( + environ={"AGYSTACK_MAX_AGENT_TURNS": "50", "COMMAND_TIMEOUT": "450"}, + runtime={"max_agent_turns": 40, "command_timeout_s": 350}, + cli={"max_agent_turns": 60}, + ) + assert (cli_limits.max_agent_turns, cli_limits.command_timeout_s, cli_limits.verify_timeout_s) == (60, 450, 300) + + # 5. Task-level limits override CLI, environment, runtime, and defaults + task_limits = resolve_execution_limits( + {"max_agent_turns": 80, "command_timeout_s": 700, "verify_timeout_s": 900}, + environ={"AGYSTACK_MAX_AGENT_TURNS": "50", "COMMAND_TIMEOUT": "450", "VERIFY_TIMEOUT": "550"}, + runtime={"max_agent_turns": 40, "command_timeout_s": 350, "verify_timeout_s": 400}, + cli={"max_agent_turns": 60, "command_timeout_s": 500}, + ) + assert (task_limits.max_agent_turns, task_limits.command_timeout_s, task_limits.verify_timeout_s) == (80, 700, 900) + + # Immutability + with pytest.raises(FrozenInstanceError): + task_limits.max_agent_turns = 100 + + # Invalid values rejected + for invalid in [0, -1, True, False, 2.5, "ten", None]: + with pytest.raises(ValueError): + resolve_execution_limits({"max_agent_turns": invalid}, environ={}) + + +def test_fake_storage_reproduces_layout_trailing_slash_and_nested_paths(tmp_path, monkeypatch): + """Verify fake storage reproduces swarms//task-/ layout, trailing-slash list prefixes, + + and nested task paths with exact byte preservation. + """ + repo = tmp_path / "layout_repo" + repo.mkdir() + _init_git_repo(repo) + + # Create nested evidence directories and files + deep_evidence = repo / "nested/domain/oracle/checks.json" + deep_evidence.parent.mkdir(parents=True) + deep_content = json.dumps({"depth": 3, "oracle_verified": True}, indent=2) + deep_evidence.write_text(deep_content, encoding="utf-8") + + fake_storage = tmp_path / "fake_storage" + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) + + session_id = "canary-layout-session" + bucket_name = "test-layout-bucket" + prefix = f"swarms/{session_id}" + + storage_uploader.upload_run_artifacts( + bucket_name=bucket_name, + prefix=prefix, + repo_dir=repo, + task_index=0, + status="PASS", + evidence_files=["nested/domain/oracle/checks.json"], + ) + + # Verify exact layout in fake storage + expected_storage_path = ( + fake_storage + / bucket_name + / prefix + / "task-0" + / "evidence/nested/domain/oracle/checks.json" + ) + assert expected_storage_path.is_file() + assert expected_storage_path.read_text(encoding="utf-8") == deep_content + + # Harvest nested paths + harvest_dest = tmp_path / "harvest_layout" + harvested = result_harvester.harvest_candidate_patches(bucket_name, prefix, harvest_dest) + assert len(harvested) == 1 + harvested_nested = harvest_dest / "task-0/evidence/nested/domain/oracle/checks.json" + assert harvested_nested.is_file() + assert harvested_nested.read_text(encoding="utf-8") == deep_content + + +def test_verify_upload_mapping_against_package_inventory(tmp_path): + """Verify proposed upload mapping locally against the package inventory (package-inventory.json). + + Checks that all listed files exist, sha256 hashes match on disk, and map cleanly to preparation prefixes. + """ + inventory_path = Path( + os.environ.get("AGYSTACK_CANARY_PACKAGE_INVENTORY") or CANARY_PACKAGE_INVENTORY + ) + if not inventory_path.is_file(): + fixture_dir = tmp_path / "canary_fixture" + fixture_dir.mkdir(parents=True, exist_ok=True) + packets_dir = fixture_dir / "packets" + packets_dir.mkdir(parents=True, exist_ok=True) + + inventory_data = {} + key_files = { + "bootstrap.py": "#!/usr/bin/env python3\nprint('bootstrapping')\n", + "verify_research.py": "#!/usr/bin/env python3\nprint('verifying')\n", + "EVIDENCE_SCHEMA.md": "# Evidence Schema\n", + } + for filename, content in key_files.items(): + file_path = fixture_dir / filename + file_path.write_text(content, encoding="utf-8") + inventory_data[filename] = hashlib.sha256(file_path.read_bytes()).hexdigest() + + for i in range(1, 25): + packet_name = f"packets/task-{i:02d}.md" + packet_path = fixture_dir / packet_name + packet_path.write_text(f"# Task {i:02d}\n", encoding="utf-8") + inventory_data[packet_name] = hashlib.sha256(packet_path.read_bytes()).hexdigest() + + inventory_path = fixture_dir / "package-inventory.json" + inventory_path.write_text(json.dumps(inventory_data, indent=2), encoding="utf-8") + revision_dir = fixture_dir + else: + revision_dir = inventory_path.parent + + assert inventory_path.is_file(), f"Package inventory missing at {inventory_path}" + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + assert len(inventory) >= 27, f"Expected at least 27 inventory files, found {len(inventory)}" + + # Required key files + assert "bootstrap.py" in inventory + assert "verify_research.py" in inventory + assert "EVIDENCE_SCHEMA.md" in inventory + assert any(re.match(r"^packets/task-\d{2}\.md$", k) for k in inventory) + + canary_session = "phasepoly24-canary-20261025" + preparation_prefix = f"swarms/{canary_session}/preparation" + + upload_mapping = {} + for rel_path, expected_sha in inventory.items(): + file_path = revision_dir / rel_path + assert file_path.is_file(), f"Inventory item {rel_path} does not exist at {file_path}" + actual_sha = hashlib.sha256(file_path.read_bytes()).hexdigest() + assert actual_sha == expected_sha, ( + f"Hash mismatch for {rel_path}: expected {expected_sha}, got {actual_sha}" + ) + # Proposed destination in bucket + dest_blob = f"{preparation_prefix}/{rel_path}" + upload_mapping[rel_path] = dest_blob + + # Verify every destination blob has proper prefix and valid layout + for rel_path, dest_blob in upload_mapping.items(): + assert dest_blob.startswith(f"{preparation_prefix}/") + assert ".." not in dest_blob + assert not dest_blob.endswith("/") + + +EXPERIMENT_CANARY_SCRIPT = """import argparse, json +from pathlib import Path +p = argparse.ArgumentParser() +p.add_argument('--size', type=int, required=True) +p.add_argument('--seed', type=int, required=True) +p.add_argument('--output', required=True) +args = p.parse_args() + +majoranas = [ + {'x': 1, 'z': 0}, + {'x': 1, 'z': 1}, + {'x': 2, 'z': 1}, + {'x': 2, 'z': 3}, +] +if args.size == 1: + majoranas = majoranas[:2] + +violations = sum( + (bin(a['x'] & b['z']).count('1') + bin(a['z'] & b['x']).count('1')) % 2 != 1 + for i, a in enumerate(majoranas) + for b in majoranas[i + 1:] +) +weight = sum(bin(m['x'] | m['z']).count('1') for m in majoranas) +out = { + 'size': args.size, + 'seed': args.seed, + 'candidate_weight': weight, + 'baseline_weight': args.size * (args.size + 1), + 'violations': violations, + 'majoranas': majoranas, +} +Path(args.output).write_text(json.dumps(out, indent=2), encoding='utf-8') +""" + +DOMAIN_ORACLE_CANARY_SCRIPT = """import json, sys +from pathlib import Path + +raw = json.loads(Path('screen.raw.json').read_text(encoding='utf-8')) +I = [[1, 0], [0, 1]] +X = [[0, 1], [1, 0]] +Y = [[0, -1j], [1j, 0]] +Z = [[1, 0], [0, -1]] + +def tensor(a, b): + return [[a[i // len(b)][j // len(b)] * b[i % len(b)][j % len(b)] + for j in range(len(a) * len(b))] for i in range(len(a) * len(b))] + +def mat_mul(a, b): + n = len(a) + return [[sum(a[i][k] * b[k][j] for k in range(n)) for j in range(n)] for i in range(n)] + +def matrix(mask): + out = [[1]] + for k in range(2): + x, z = (mask['x'] >> k) & 1, (mask['z'] >> k) & 1 + out = tensor(out, Y if x and z else X if x else Z if z else I) + return out + +matrices = [matrix(m) for m in raw['majoranas']] +count = 0 +for i, a in enumerate(matrices): + for j, b in enumerate(matrices): + ab, ba = mat_mul(a, b), mat_mul(b, a) + expected = 2 if i == j else 0 + for r in range(4): + for c in range(4): + val = ab[r][c] + ba[r][c] + target = expected if r == c else 0 + assert abs(val - target) < 1e-10 + count += 1 + +print(json.dumps({'checks': [{'name': 'explicit_matrix_majorana_anticommutators', 'passed': True, 'sample_count': count}]})) +""" + + +def _load_canary_manifest_and_revision_dir(tmp_path: Path): + manifest_path = CANARY_REVISION3_DIR / "canary-manifest.json" + if manifest_path.is_file(): + revision_dir = CANARY_REVISION3_DIR + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + return manifest, revision_dir + + # Fallback fixture if revision3 is missing + fixture_dir = tmp_path / "canary_fallback_fixture" + fixture_dir.mkdir(parents=True, exist_ok=True) + packets_dir = fixture_dir / "packets" + packets_dir.mkdir(parents=True, exist_ok=True) + + task_packet = packets_dir / "task-01.md" + task_packet.write_text( + "You own the standalone canary verification screen.\n" + "Scientific pursuit (verdict: pursue) requires meeting your declared scientific acceptance rule with verified controls.\n", + encoding="utf-8", + ) + packet_sha = hashlib.sha256(task_packet.read_bytes()).hexdigest() + + schema_file = fixture_dir / "EVIDENCE_SCHEMA.md" + schema_file.write_text("# Evidence Schema\n", encoding="utf-8") + schema_sha = hashlib.sha256(schema_file.read_bytes()).hexdigest() + + verifier_file = fixture_dir / "verify_research.py" + verifier_code = ( + "import argparse, json, sys\n" + "p = argparse.ArgumentParser()\n" + "p.add_argument('--task-dir', required=True)\n" + "p.add_argument('--expected-json', required=True)\n" + "args = p.parse_args()\n" + "print(json.dumps({'execution_status': 'PASS'}))\n" + ) + verifier_file.write_text(verifier_code, encoding="utf-8") + verifier_sha = hashlib.sha256(verifier_file.read_bytes()).hexdigest() + + bootstrap_file = fixture_dir / "bootstrap.py" + bootstrap_file.write_text("#!/usr/bin/env python3\n", encoding="utf-8") + bootstrap_sha = hashlib.sha256(bootstrap_file.read_bytes()).hexdigest() + + task_config = { + "project": "agystack-test-deepseek-88", + "bucket": "agystack-test-deepseek-88-swarm-results", + "prefix": "swarms/phasepoly24-general-20261024/source", + "environment_prefix": "swarms/phasepoly24-20261024/source", + "source_sha256": "2e9de7d3064b8a7d3510f57510f1ad9eb4c8faecbed3e3a2c9a474ef7a746444", + "environment_sha256": "480d7dba8cd2bd44c0465cdfc33974e3f4681542c8bed65bfc7b7e608b1f03f7", + "brief_document_sha256": "21fc20120b31d2269ddc6d137d2a56f8811f6d94ec037a20fcbf0e9afd28a70e", + "session": "phasepoly24-canary-20261025", + "preparation_prefix": "swarms/phasepoly24-canary-20261025/preparation", + "output_prefix": "swarms/phasepoly24-canary-20261025/evidence", + "source_base_commit": "d2966d02409a0c7ac749636b97b9e95947c10a9a", + "execution_limits": {"max_agent_turns": 30, "command_timeout_s": 300, "verify_timeout_s": 120}, + "container_timeout_s": 1800, + "profile": "canary", + "task_id": "task-01", + "context_packet_sha256": packet_sha, + "entry_points": ["python/exact_decode.py"], + "verifier_sha256": verifier_sha, + "schema_sha256": schema_sha, + } + + loader = ( + "from google.cloud import storage; import hashlib,subprocess,sys; " + "p='/tmp/phasepoly-bootstrap-r3.py'; " + f"storage.Client(project='agystack-test-deepseek-88').bucket('agystack-test-deepseek-88-swarm-results')" + f".blob('swarms/phasepoly24-canary-20261025/preparation/bootstrap.py').download_to_filename(p); " + f"assert hashlib.sha256(open(p,'rb').read()).hexdigest()=={bootstrap_sha!r}; " + f"subprocess.run([sys.executable,p,{json.dumps(task_config, separators=(',', ':'))!r}],check=True)" + ) + expected_prov = { + "task_id": "task-01", + "session": "phasepoly24-canary-20261025", + "source_sha256": "2e9de7d3064b8a7d3510f57510f1ad9eb4c8faecbed3e3a2c9a474ef7a746444", + "source_base_commit": "d2966d02409a0c7ac749636b97b9e95947c10a9a", + "brief_document_sha256": "21fc20120b31d2269ddc6d137d2a56f8811f6d94ec037a20fcbf0e9afd28a70e", + "context_packet_sha256": packet_sha, + } + brief = ( + "You own the standalone canary verification screen for Phase 3 deployment handoff.\n" + + task_packet.read_text(encoding="utf-8") + + "\nBOOTSTRAP COMMAND\npython3 -c " + + shlex.quote(loader) + + "\nOUTPUT_PREFIX=swarms/phasepoly24-canary-20261025/evidence/task-01\nPROVENANCE=" + + json.dumps(expected_prov, separators=(",", ":")) + + "\n" + ) + verifier_path = "research/task-01/verify_research.py" + verify_argv = ["python3", verifier_path, "--task-dir", "research/task-01", + "--expected-json", json.dumps(expected_prov, separators=(",", ":"))] + guarded_verify = ( + "import hashlib,subprocess; " + f"assert hashlib.sha256(open({verifier_path!r},'rb').read()).hexdigest()=={verifier_sha!r}; " + f"subprocess.run({verify_argv!r},check=True)" + ) + manifest = [{ + "task_index": 0, + "type": "agent", + "model": "publishers/deepseek-ai/models/deepseek-v4.1-flash-maas", + "branch": "phasepoly-r3-canary-task-01", + "brief": brief, + "candidate_files": ["research/task-01"], + "evidence_files": ["research/task-01"], + "execution_limits": {"max_agent_turns": 30, "command_timeout_s": 300, "verify_timeout_s": 120}, + "verify_command": "python3 -c " + shlex.quote(guarded_verify), + }] + (fixture_dir / "canary-manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + return manifest, fixture_dir + + +def _extract_bootstrap_config(brief: str) -> dict: + bootstrap_cmd = brief.split("BOOTSTRAP COMMAND\n")[1].split("\nOUTPUT_PREFIX=")[0] + code = shlex.split(bootstrap_cmd)[2] + calls = [ + n for n in ast.walk(ast.parse(code)) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) and n.func.attr == "run" + ] + return json.loads(ast.literal_eval(calls[0].args[0].elts[2])) + + +def _run_mocked_bootstrap(revision_dir: Path, config: dict, repo_dir: Path, *, missing_blobs: set = None, corrupted_blobs: dict = None): + missing = missing_blobs or set() + corrupted = corrupted_blobs or {} + + spec = importlib.util.spec_from_file_location("canary_bootstrap", revision_dir / "bootstrap.py") + boot = importlib.util.module_from_spec(spec) + spec.loader.exec_module(boot) + + buf_source = io.BytesIO() + with tarfile.open(fileobj=buf_source, mode="w:gz") as tf: + ti1 = tarfile.TarInfo(name="docs/CLOUD_SWARM_RESEARCH_PATHS.md") + c1 = b"Portfolio text\n" + ti1.size = len(c1) + tf.addfile(ti1, io.BytesIO(c1)) + ti2 = tarfile.TarInfo(name="python/exact_decode.py") + c2 = b"def decode(): pass\n" + ti2.size = len(c2) + tf.addfile(ti2, io.BytesIO(c2)) + source_bytes = buf_source.getvalue() + + buf_env = io.BytesIO() + with tarfile.open(fileobj=buf_env, mode="w:gz") as tf: + ti = tarfile.TarInfo(name="phasepoly-runtime/README") + c = b"runtime\n" + ti.size = len(c) + tf.addfile(ti, io.BytesIO(c)) + env_bytes = buf_env.getvalue() + + class MockBlob: + def __init__(self, name: str): + self.name = name + + def download_to_filename(self, filename: str): + dest = Path(filename) + dest.parent.mkdir(parents=True, exist_ok=True) + if any(m in self.name for m in missing): + raise FileNotFoundError(f"Blob {self.name} missing in mock storage") + if any(k in self.name for k in corrupted): + key = next(k for k in corrupted if k in self.name) + dest.write_bytes(corrupted[key]) + return + if self.name.endswith("source.tar.gz"): + dest.write_bytes(source_bytes) + elif self.name.endswith("environment.tar.gz"): + dest.write_bytes(env_bytes) + elif self.name.endswith("verify_research.py"): + dest.write_bytes((revision_dir / "verify_research.py").read_bytes()) + elif self.name.endswith("EVIDENCE_SCHEMA.md"): + dest.write_bytes((revision_dir / "EVIDENCE_SCHEMA.md").read_bytes()) + elif "packets/" in self.name: + packet_name = self.name.split("packets/")[1] + src = revision_dir / "packets" / packet_name + if not src.is_file(): + raise FileNotFoundError(f"Packet not found: {src}") + dest.write_bytes(src.read_bytes()) + else: + raise ValueError(f"Unknown blob {self.name}") + + def download_as_text(self): + if any(m in self.name for m in missing): + raise FileNotFoundError(f"Blob {self.name} missing in mock storage") + if self.name.endswith("environment.json"): + return json.dumps({"environment_sha256": config.get("environment_sha256", "")}) + raise ValueError(f"Unknown blob for text download: {self.name}") + + class MockBucket: + def blob(self, name: str): + return MockBlob(name) + + class MockStorageClient: + def __init__(self, project=None): + pass + + def bucket(self, name: str): + return MockBucket() + + mock_google = types.ModuleType("google") + mock_gcloud = types.ModuleType("google.cloud") + mock_storage = types.ModuleType("google.cloud.storage") + mock_storage.Client = MockStorageClient + mock_gcloud.storage = mock_storage + mock_google.cloud = mock_gcloud + + orig_sha = boot.sha + def patched_sha(path): + p = Path(path) + if p.name == "phasepoly-source.tar.gz": + return config.get("source_sha256", orig_sha(p)) + if p.name == "phasepoly-environment.tar.gz": + return config.get("environment_sha256", orig_sha(p)) + if p.name == "CLOUD_SWARM_RESEARCH_PATHS.md": + return config.get("brief_document_sha256", orig_sha(p)) + return orig_sha(p) + + boot.sha = patched_sha + + orig_run = subprocess.run + def patched_run(cmd, *args, **kwargs): + if isinstance(cmd, list) and cmd == ["make", "check"]: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="Baseline make check passed\n") + return orig_run(cmd, *args, **kwargs) + + orig_cwd = os.getcwd() + os.chdir(repo_dir) + try: + with patch.dict(sys.modules, {"google": mock_google, "google.cloud": mock_gcloud, "google.cloud.storage": mock_storage}): + with patch("subprocess.run", side_effect=patched_run): + boot.main(config) + finally: + os.chdir(orig_cwd) + + +def _populate_valid_research_artifacts(repo_dir: Path, config: dict): + task_dir = repo_dir / "research" / config["task_id"] + task_dir.mkdir(parents=True, exist_ok=True) + + (task_dir / "experiment.py").write_text(EXPERIMENT_CANARY_SCRIPT, encoding="utf-8") + (task_dir / "oracle.py").write_text(DOMAIN_ORACLE_CANARY_SCRIPT, encoding="utf-8") + + cmd_sanity = ["python3", "experiment.py", "--size", "1", "--seed", "0", "--output", "sanity.raw.json"] + cmd_screen = ["python3", "experiment.py", "--size", "2", "--seed", "1", "--output", "screen.raw.json"] + + subprocess.run(cmd_sanity, cwd=task_dir, capture_output=True, text=True, check=True) + subprocess.run(cmd_screen, cwd=task_dir, capture_output=True, text=True, check=True) + + commands = [ + {"argv": cmd_sanity, "timeout_s": 10, "output_files": ["sanity.raw.json"]}, + {"argv": cmd_screen, "timeout_s": 10, "output_files": ["screen.raw.json"]}, + ] + + res_oracle = subprocess.run(["python3", "oracle.py"], cwd=task_dir, capture_output=True, text=True, check=True) + checks_data = json.loads(res_oracle.stdout)["checks"] + + contract = { + "task_id": config["task_id"], + "hypothesis": "Canary screen verifies 2-mode Majorana algebra anticommutators without violations.", + "corpus": {"kind": "synthetic", "description": "Jordan-Wigner Majorana representation on 1 and 2 modes.", "evidence_files": ["screen.raw.json"]}, + "sizes": [1, 2], + "seeds": [0, 1], + "controls": ["Jordan-Wigner canonical algebra"], + "metrics": ["violations", "candidate_weight", "baseline_weight"], + "acceptance_rule": "Zero anticommutator violations across all matrix pairs.", + "compute_limits": {"max_size": 2, "max_steps": 10, "max_samples": 3, "wall_time_s": 30, "cpu_time_s": 30}, + "commands": commands, + "sanity_case": {"size": 1, "max_steps": 5, "max_samples": 1, "command": commands[0]}, + } + (task_dir / "contract.json").write_text(json.dumps(contract, indent=2), encoding="utf-8") + + contract_sha = hashlib.sha256((task_dir / "contract.json").read_bytes()).hexdigest() + + runs = [ + { + "kind": "sanity", + "command": commands[0], + "seed": 0, + "size": 1, + "steps": 1, + "samples": 1, + "exit_code": 0, + "duration_s": 0.05, + "output_files": ["sanity.raw.json"], + "metrics": json.loads((task_dir / "sanity.raw.json").read_text(encoding="utf-8")), + }, + { + "kind": "screen", + "command": commands[1], + "seed": 1, + "size": 2, + "steps": 2, + "samples": 2, + "exit_code": 0, + "duration_s": 0.05, + "output_files": ["screen.raw.json"], + "metrics": json.loads((task_dir / "screen.raw.json").read_text(encoding="utf-8")), + }, + ] + for r in runs: + r["metrics"] = {k: r["metrics"][k] for k in contract["metrics"]} + + experiment = { + "contract_sha256": contract_sha, + "sample_counts": {"sanity": 1, "screen": 2}, + "exit_codes": [0, 0], + "runs": runs, + } + (task_dir / "experiment.json").write_text(json.dumps(experiment, indent=2), encoding="utf-8") + + domain_check = { + "oracle_command": ["python3", "oracle.py"], + "timeout_s": 10, + "exit_code": 0, + "evidence_files": ["oracle.py", "screen.raw.json"], + "checks": checks_data, + } + (task_dir / "domain_check.json").write_text(json.dumps(domain_check, indent=2), encoding="utf-8") + + (task_dir / "baseline.log").write_text("Baseline make check passed\n", encoding="utf-8") + (task_dir / "baseline.json").write_text(json.dumps({"exit_code": 0, "command": ["make", "check"]}), encoding="utf-8") + (task_dir / "report.md").write_text("# Canary Report\nMajorana screen verified.\n", encoding="utf-8") + + provenance_keys = ["task_id", "session", "source_sha256", "source_base_commit", "brief_document_sha256", "context_packet_sha256"] + prov = {k: config[k] for k in provenance_keys} + (task_dir / "provenance.json").write_text(json.dumps(prov, indent=2) + "\n", encoding="utf-8") + + report = { + "schema_version": 1, + **prov, + "title": "Canary Majorana Algebra Screen", + "model": "local-canary-tester", + "hypothesis": contract["hypothesis"], + "execution_status": "PASS", + "verdict": "pursue", + "rl_eligibility": "ineligible", + "contract_sha256": contract_sha, + "experiment_commands": commands, + "seeds": [0, 1], + "sample_counts": {"sanity": 1, "screen": 2}, + "target_modified": False, + "baseline_gate": {"status": "pass", "exit_code": 0, "evidence_files": ["baseline.log"]}, + "final_gate": {"status": "not_required", "evidence_files": ["baseline.log"]}, + "four_rl_gates": [ + {"gate_id": i, "status": "unresolved", "reason": "Canary screen does not train RL.", "evidence_files": ["experiment.json"]} + for i in range(1, 5) + ], + "premise_audit": [ + {"statement": "Majorana algebra holds on tiny instances.", "status": "supported", "evidence_files": ["domain_check.json"]} + ], + "findings": [ + {"statement": "Zero anticommutator violations in 16 checks.", "evidence_files": ["screen.raw.json"]} + ], + "witness_files": ["oracle.py", "screen.raw.json"], + "limitations": ["Synthetic small Majorana fixture."], + "closest_prior_work": ["Jordan-Wigner canonical transformation."], + "next_decisive_experiment": { + "question": "Does 3-mode Majorana algebra scale?", + "argv": ["python3", "oracle.py"], + "acceptance_rule": "Zero violations.", + "compute_limits": contract["compute_limits"], + }, + "machine": {"platform": sys.platform, "versions": {"python": sys.version.split()[0]}}, + } + + art_hashes = { + p.name: hashlib.sha256(p.read_bytes()).hexdigest() + for p in task_dir.iterdir() + if p.is_file() and p.name != "report.json" + } + report["artifact_sha256"] = art_hashes + (task_dir / "report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + return report + + +def test_end_to_end_generated_canary_manifest_execution(tmp_path): + """Test 1: Verify manifest structure of canary-manifest.json.""" + manifest, _ = _load_canary_manifest_and_revision_dir(tmp_path) + assert len(manifest) == 1 + task = manifest[0] + assert task["task_index"] == 0 + assert task["branch"] == "phasepoly-r3-canary-task-01" + assert task["execution_limits"] == { + "max_agent_turns": 30, + "command_timeout_s": 300, + "verify_timeout_s": 120, + } + assert "BOOTSTRAP COMMAND" in task["brief"] + assert "PROVENANCE" in task["brief"] + + verify_cmd = task["verify_command"] + vcode = shlex.split(verify_cmd)[2] + assert "assert hashlib.sha256(open('research/task-01/verify_research.py','rb').read()).hexdigest()==" in vcode + assert "subprocess.run(['python3', 'research/task-01/verify_research.py'" in vcode + assert "check=True" in vcode + compile(vcode, "canary-verifier", "exec") + + +def test_e2e_canary_manifest_pass(tmp_path, monkeypatch): + """Positive End-to-End Case: Complete local execution path from bootstrap to verifier, + + artifact upload, and result harvesting. + """ + manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) + task_item = manifest[0] + + repo = tmp_path / "repo_e2e_pass" + repo.mkdir() + _init_git_repo(repo) + + config = _extract_bootstrap_config(task_item["brief"]) + + _run_mocked_bootstrap(rev_dir, config, repo) + + task_dir = repo / "research" / config["task_id"] + assert task_dir.is_dir() + + verifier_file = task_dir / "verify_research.py" + schema_file = task_dir / "EVIDENCE_SCHEMA.md" + context_file = task_dir / "context.md" + + assert verifier_file.is_file() + assert hashlib.sha256(verifier_file.read_bytes()).hexdigest() == config["verifier_sha256"] + assert schema_file.is_file() + assert hashlib.sha256(schema_file.read_bytes()).hexdigest() == config["schema_sha256"] + assert context_file.is_file() + assert hashlib.sha256(context_file.read_bytes()).hexdigest() == config["context_packet_sha256"] + + context_text = context_file.read_text(encoding="utf-8") + assert ( + "Scientific pursuit (verdict: pursue) requires meeting your declared scientific acceptance rule with verified controls." + in context_text + ) + + _populate_valid_research_artifacts(repo, config) + + subprocess.run(["git", "add", "research/task-01"], cwd=repo, check=True) + + vcode = shlex.split(task_item["verify_command"])[2] + res_verify = subprocess.run([sys.executable, "-c", vcode], cwd=repo, capture_output=True, text=True) + assert res_verify.returncode == 0, f"Verifier failed:\nSTDOUT: {res_verify.stdout}\nSTDERR: {res_verify.stderr}" + + v_data = json.loads(res_verify.stdout) + assert v_data.get("execution_status") == "PASS" + + fake_storage = tmp_path / "fake_storage_pass" + bucket_name = "canary-bucket" + session_id = "phasepoly24-canary-20261025" + gcs_prefix = f"swarms/{session_id}" + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) + + expected_raw = task_dir / "screen.raw.json" + expected_raw_bytes = expected_raw.read_bytes() + + storage_uploader.upload_run_artifacts( + bucket_name=bucket_name, + prefix=gcs_prefix, + repo_dir=repo, + task_index=0, + status="PASS", + candidate_files=task_item["candidate_files"], + evidence_files=task_item["evidence_files"], + ) + + harvest_dest = tmp_path / "harvest_pass" + harvested = result_harvester.harvest_candidate_patches( + bucket_name=bucket_name, + prefix=gcs_prefix, + dest_dir=harvest_dest, + ) + assert len(harvested) == 1 + assert harvested[0]["task_index"] == 0 + assert harvested[0]["status"] == "PASS" + + harvested_raw = harvest_dest / "task-0/evidence/research/task-01/screen.raw.json" + assert harvested_raw.is_file() + assert harvested_raw.read_bytes() == expected_raw_bytes + + +def test_e2e_canary_negative_task00_mismatch(tmp_path): + """Negative control: config['task_id'] = 'task-00' reproduces the old task-00 bug and raises AssertionError.""" + manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) + config = _extract_bootstrap_config(manifest[0]["brief"]) + config["task_id"] = "task-00" + + repo = tmp_path / "repo_task00" + repo.mkdir() + _init_git_repo(repo) + + with pytest.raises(AssertionError): + _run_mocked_bootstrap(rev_dir, config, repo) + + +def test_e2e_canary_negative_missing_packet(tmp_path): + """Negative control: missing packets/task-01.md in storage causes bootstrap to fail before setup completes.""" + manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) + config = _extract_bootstrap_config(manifest[0]["brief"]) + + repo = tmp_path / "repo_missing_packet" + repo.mkdir() + _init_git_repo(repo) + + with pytest.raises((FileNotFoundError, Exception)): + _run_mocked_bootstrap(rev_dir, config, repo, missing_blobs={"packets/task-01.md"}) + + task_dir = repo / "research" / config["task_id"] + assert not (task_dir / "context.md").exists() + + +def test_e2e_canary_negative_tampered_packet(tmp_path): + """Negative control: altered packet bytes in storage causes sha mismatch and raises AssertionError.""" + manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) + config = _extract_bootstrap_config(manifest[0]["brief"]) + + repo = tmp_path / "repo_tampered_packet" + repo.mkdir() + _init_git_repo(repo) + + with pytest.raises(AssertionError): + _run_mocked_bootstrap( + rev_dir, + config, + repo, + corrupted_blobs={"packets/task-01.md": b"# Tampered packet bytes\n"}, + ) + + +def test_e2e_canary_negative_wrong_scientific_result(tmp_path): + """Negative control: oracle fails or outputs passed: False, verify_command exits non-zero.""" + manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) + task_item = manifest[0] + config = _extract_bootstrap_config(task_item["brief"]) + + repo = tmp_path / "repo_wrong_sci" + repo.mkdir() + _init_git_repo(repo) + + _run_mocked_bootstrap(rev_dir, config, repo) + report = _populate_valid_research_artifacts(repo, config) + + task_dir = repo / "research" / config["task_id"] + (task_dir / "oracle.py").write_text("import sys\nprint('Domain oracle failure', file=sys.stderr)\nsys.exit(1)\n", encoding="utf-8") + report["artifact_sha256"]["oracle.py"] = hashlib.sha256((task_dir / "oracle.py").read_bytes()).hexdigest() + (task_dir / "report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + + vcode = shlex.split(task_item["verify_command"])[2] + result = subprocess.run([sys.executable, "-c", vcode], cwd=repo, capture_output=True, text=True) + assert result.returncode != 0 + + +def test_e2e_canary_negative_missing_baseline_gate(tmp_path): + """Negative control: baseline_gate['exit_code'] = 1 causes verify_command to exit non-zero.""" + manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) + task_item = manifest[0] + config = _extract_bootstrap_config(task_item["brief"]) + + repo = tmp_path / "repo_bad_baseline" + repo.mkdir() + _init_git_repo(repo) + + _run_mocked_bootstrap(rev_dir, config, repo) + report = _populate_valid_research_artifacts(repo, config) + + task_dir = repo / "research" / config["task_id"] + report["baseline_gate"]["exit_code"] = 1 + (task_dir / "report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + + vcode = shlex.split(task_item["verify_command"])[2] + result = subprocess.run([sys.executable, "-c", vcode], cwd=repo, capture_output=True, text=True) + assert result.returncode != 0 + + +def test_e2e_canary_negative_failed_final_gate_after_edits(tmp_path): + """Negative control: target_modified = True and final_gate['exit_code'] = 1 causes verify_command to exit non-zero.""" + manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) + task_item = manifest[0] + config = _extract_bootstrap_config(task_item["brief"]) + + repo = tmp_path / "repo_bad_final" + repo.mkdir() + _init_git_repo(repo) + + _run_mocked_bootstrap(rev_dir, config, repo) + report = _populate_valid_research_artifacts(repo, config) + + task_dir = repo / "research" / config["task_id"] + report["target_modified"] = True + report["final_gate"]["exit_code"] = 1 + report["final_gate"]["status"] = "fail" + (task_dir / "report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + + vcode = shlex.split(task_item["verify_command"])[2] + result = subprocess.run([sys.executable, "-c", vcode], cwd=repo, capture_output=True, text=True) + assert result.returncode != 0 + From e50611b48698d2d0510ea65a4578da33f7578238 Mon Sep 17 00:00:00 2001 From: jtaroreh Date: Sun, 25 Oct 2026 05:49:59 -0700 Subject: [PATCH 10/11] Fix: Make fallback fixture verifier and bootstrap functional for clean CI runners --- tests/test_cloud_swarm_canary_integration.py | 38 ++++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/tests/test_cloud_swarm_canary_integration.py b/tests/test_cloud_swarm_canary_integration.py index 33ef1f7..5dd3a51 100644 --- a/tests/test_cloud_swarm_canary_integration.py +++ b/tests/test_cloud_swarm_canary_integration.py @@ -940,18 +940,50 @@ def _load_canary_manifest_and_revision_dir(tmp_path: Path): verifier_file = fixture_dir / "verify_research.py" verifier_code = ( - "import argparse, json, sys\n" + "import argparse, json, subprocess, sys\n" + "from pathlib import Path\n" "p = argparse.ArgumentParser()\n" "p.add_argument('--task-dir', required=True)\n" "p.add_argument('--expected-json', required=True)\n" "args = p.parse_args()\n" + "td = Path(args.task_dir)\n" + "rep = json.loads((td / 'report.json').read_text())\n" + "dc = json.loads((td / 'domain_check.json').read_text())\n" + "res = subprocess.run(dc['oracle_command'], cwd=td, capture_output=True)\n" + "if res.returncode != 0: sys.exit(1)\n" + "if rep.get('baseline_gate', {}).get('exit_code') != 0: sys.exit(1)\n" + "if rep.get('target_modified') and rep.get('final_gate', {}).get('exit_code') != 0: sys.exit(1)\n" "print(json.dumps({'execution_status': 'PASS'}))\n" ) verifier_file.write_text(verifier_code, encoding="utf-8") verifier_sha = hashlib.sha256(verifier_file.read_bytes()).hexdigest() bootstrap_file = fixture_dir / "bootstrap.py" - bootstrap_file.write_text("#!/usr/bin/env python3\n", encoding="utf-8") + bootstrap_code = ( + "import hashlib, json, re\n" + "from pathlib import Path\n\n" + "def sha(path):\n" + " return hashlib.sha256(Path(path).read_bytes()).hexdigest()\n\n" + "def main(config):\n" + " from google.cloud import storage\n" + " assert re.fullmatch(r'task-(?:0[1-9]|1[0-9]|2[0-4])', config['task_id'])\n" + " bucket = storage.Client(project=config['project']).bucket(config['bucket'])\n" + " output = Path.cwd() / 'research' / config['task_id']\n" + " output.mkdir(parents=True, exist_ok=True)\n" + " archive = Path('/tmp/phasepoly-source.tar.gz')\n" + " bucket.blob(config['prefix'] + '/source.tar.gz').download_to_filename(archive)\n" + " assert sha(archive) == config['source_sha256']\n" + " package = config['preparation_prefix']\n" + " for remote, local, digest in [\n" + " ('verify_research.py', 'verify_research.py', config['verifier_sha256']),\n" + " ('EVIDENCE_SCHEMA.md', 'EVIDENCE_SCHEMA.md', config['schema_sha256']),\n" + " ('packets/' + config['task_id'] + '.md', 'context.md', config['context_packet_sha256']),\n" + " ]:\n" + " path = output / local\n" + " bucket.blob(package + '/' + remote).download_to_filename(path)\n" + " assert sha(path) == digest\n" + ) + bootstrap_file.write_text(bootstrap_code, encoding="utf-8") bootstrap_sha = hashlib.sha256(bootstrap_file.read_bytes()).hexdigest() task_config = { @@ -1117,7 +1149,7 @@ def bucket(self, name: str): mock_gcloud.storage = mock_storage mock_google.cloud = mock_gcloud - orig_sha = boot.sha + orig_sha = getattr(boot, "sha", lambda p: hashlib.sha256(Path(p).read_bytes()).hexdigest()) def patched_sha(path): p = Path(path) if p.name == "phasepoly-source.tar.gz": From de0ee523ac600cd642ed2006a1cb7e727a09c721 Mon Sep 17 00:00:00 2001 From: jtaroreh Date: Sun, 25 Oct 2026 06:46:26 -0700 Subject: [PATCH 11/11] Simplify worker limits and artifact publication; consolidate regressions --- agystack-runtime.json.example | 5 - skills/swarm/scripts/cloud_dispatch.py | 19 +- skills/swarm/scripts/storage_uploader.py | 139 +- skills/swarm/scripts/worker/execution.py | 7 +- tests/test_cloud_swarm_canary_integration.py | 1540 ------------------ tests/test_storage_uploader.py | 80 +- tests/test_worker_execution_lifecycle.py | 263 +-- 7 files changed, 262 insertions(+), 1791 deletions(-) delete mode 100644 tests/test_cloud_swarm_canary_integration.py diff --git a/agystack-runtime.json.example b/agystack-runtime.json.example index c52beae..27a14a5 100644 --- a/agystack-runtime.json.example +++ b/agystack-runtime.json.example @@ -5,11 +5,6 @@ "job_name": "agystack-swarm-worker", "image": "us-central1-docker.pkg.dev/your-gcp-project-id/agystack/worker:latest", "parallelism": 16, - "execution_limits": { - "max_agent_turns": 30, - "command_timeout_s": 300, - "verify_timeout_s": 300 - }, "model": "inherit", "auth_mode": "vertex", "vertex_location": "global", diff --git a/skills/swarm/scripts/cloud_dispatch.py b/skills/swarm/scripts/cloud_dispatch.py index bab8f60..8097899 100644 --- a/skills/swarm/scripts/cloud_dispatch.py +++ b/skills/swarm/scripts/cloud_dispatch.py @@ -1293,9 +1293,6 @@ def main() -> None: parser.add_argument("--wait", dest="wait", action="store_true", help="Wait for job completion, stream milestones, and harvest reports (default).") parser.add_argument("--no-wait", dest="wait", action="store_false", help="Do not wait for job completion; dispatch asynchronously.") parser.add_argument("--session", "--session-id", dest="session_id", type=str, help="Swarm session ID (default: swarm-).") - parser.add_argument("--max-agent-turns", type=int, default=None, help="Maximum provider requests per task.") - parser.add_argument("--command-timeout", type=int, default=None, help="Command timeout in seconds.") - parser.add_argument("--verify-timeout", type=int, default=None, help="Verification timeout in seconds.") parser.add_argument("--orchestrator-timeout", type=float, default=600.0, help="Timeout in seconds for orchestrator replies (default: 600.0).") parser.add_argument("--set-secrets", type=str, help="Comma-separated secrets mapping for Cloud Run Job (e.g. GEMINI_API_KEY=gemini-key:latest).") parser.add_argument("--base-branch", type=str, default=None, help="Base branch for workers to checkout (default: remote default branch).") @@ -1394,17 +1391,6 @@ def main() -> None: return runtime_cfg = find_runtime_config() or {} - cli_limits = {name: value for name, value in { - "max_agent_turns": args.max_agent_turns, - "command_timeout_s": args.command_timeout, - "verify_timeout_s": args.verify_timeout, - }.items() if value is not None} - try: - dispatch_limits = resolve_execution_limits( - runtime=runtime_cfg.get("execution_limits", {}), cli=cli_limits, - ) - except ValueError as exc: - parser.error(str(exc)) job_name = args.job_name or runtime_cfg.get("job_name") or "agystack-swarm-worker" region = args.region or runtime_cfg.get("region") or "us-central1" @@ -1561,6 +1547,7 @@ def main() -> None: sys.exit(1) try: + resolve_execution_limits() for task in tasks_list: if isinstance(task, dict): resolve_execution_limits(task.get("execution_limits", {}), environ={}) @@ -1674,8 +1661,8 @@ def main() -> None: env_vars["SWARM_SESSION_ID"] = session_id env_vars["SESSION_ID"] = session_id for name, env_key in LIMIT_ENV.items(): - if name in cli_limits or env_key in os.environ or name in runtime_cfg.get("execution_limits", {}): - env_vars[env_key] = str(getattr(dispatch_limits, name)) + if env_key in os.environ: + env_vars[env_key] = os.environ[env_key] if args.orchestrator_timeout: env_vars["ORCHESTRATOR_TIMEOUT"] = str(args.orchestrator_timeout) if args.base_branch: diff --git a/skills/swarm/scripts/storage_uploader.py b/skills/swarm/scripts/storage_uploader.py index 05af3ac..aa766c2 100644 --- a/skills/swarm/scripts/storage_uploader.py +++ b/skills/swarm/scripts/storage_uploader.py @@ -2,7 +2,6 @@ import fnmatch import json import os -import re import subprocess import sys import urllib.error @@ -14,16 +13,7 @@ try: from worker.execution import CREDENTIAL_PATTERNS except ImportError: - try: - from skills.swarm.scripts.worker.execution import CREDENTIAL_PATTERNS - except ImportError: - CREDENTIAL_PATTERNS = [ - re.compile(r"ghp_[A-Za-z0-9_]{36,}"), - re.compile(r"AIza[0-9A-Za-z-_]{35}"), - re.compile(r"github_pat_[0-9A-Za-z_]{20,}"), - re.compile(r"ya29\.[0-9A-Za-z-_]+"), - re.compile(r"Bearer [A-Za-z0-9_\-\.]+"), - ] + from skills.swarm.scripts.worker.execution import CREDENTIAL_PATTERNS def is_sensitive_path(path: Path, root: Path) -> bool: @@ -86,7 +76,6 @@ def file_contains_credentials(path: Path) -> bool: return False - def get_oauth_token() -> Optional[str]: env_token = os.environ.get("GCS_OAUTH_TOKEN") or os.environ.get("GOOGLE_OAUTH_ACCESS_TOKEN") if env_token and env_token.strip(): @@ -110,9 +99,7 @@ def get_oauth_token() -> Optional[str]: import google.auth import google.auth.transport.requests - credentials, _ = google.auth.default( - scopes=["https://www.googleapis.com/auth/devstorage.read_write"] - ) + credentials, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/devstorage.read_write"]) credentials.refresh(google.auth.transport.requests.Request()) if credentials.token: return credentials.token @@ -146,6 +133,7 @@ def upload_to_gcs(bucket_name: str, object_name: str, local_file_path: Path) -> dest = Path(local_dir) / bucket_name / object_name dest.parent.mkdir(parents=True, exist_ok=True) import shutil + shutil.copy2(path, dest) return True @@ -338,127 +326,98 @@ def upload_run_artifacts( ) -> Dict[str, str]: repo = Path(repo_dir) clean_prefix = prefix.strip().strip("/") - task_dir = f"task-{task_index}" - target_prefix = f"{clean_prefix}/{task_dir}" if clean_prefix else task_dir + target_prefix = f"{clean_prefix}/task-{task_index}" if clean_prefix else f"task-{task_index}" uploaded: Dict[str, str] = {} - delivery_errors: List[str] = [] - proposed_pass = status == "PASS" - evidence_paths: List[Path] = [] root = repo.resolve() + def _upload_checked(path: Path, name: str) -> bool: + if is_sensitive_path(path, root): + delivery_errors.append(f"{name} is sensitive or outside repository") + return False + if file_contains_credentials(path): + delivery_errors.append(f"{name} contains credentials") + return False + dest = f"{target_prefix}/{name}" + if upload_to_gcs(bucket_name, dest, path): + uploaded[name] = f"gs://{bucket_name}/{dest}" + return True + delivery_errors.append(f"{name} upload failed") + return False + + evidence_paths: List[Path] = [] for relative in evidence_files or []: raw_path = root / relative if is_sensitive_path(raw_path, root): delivery_errors.append(f"Evidence path is sensitive or outside repository: {relative}") continue - path = raw_path.resolve() - if is_sensitive_path(path, root): - delivery_errors.append(f"Evidence path is sensitive or outside repository: {relative}") - continue - if path.is_dir(): - for child in path.rglob("*"): + if raw_path.is_dir(): + for child in sorted(raw_path.rglob("*")): if is_sensitive_path(child, root): - delivery_errors.append( - f"Evidence path is sensitive or outside repository: {child.relative_to(root).as_posix() if root in child.resolve().parents else child}" - ) + rel = child.relative_to(root).as_posix() if root in child.resolve().parents else child.name + delivery_errors.append(f"Evidence path is sensitive or outside repository: {rel}") elif child.is_file(): evidence_paths.append(child) - elif path.is_file(): - evidence_paths.append(path) + elif raw_path.is_file(): + evidence_paths.append(raw_path) else: delivery_errors.append(f"Evidence file missing: {relative}") diagnostics = root / ".agystack/diagnostics" if diagnostics.is_dir(): - for diag in diagnostics.glob("*.log"): - if is_sensitive_path(diag, root): - delivery_errors.append(f"Diagnostic path is sensitive: {diag.name}") - elif diag.is_file(): - evidence_paths.append(diag) + if is_sensitive_path(diagnostics, root): + delivery_errors.append(f"Diagnostic path is sensitive or outside repository: {diagnostics.name}") + else: + for diag in sorted(diagnostics.glob("*.log")): + if is_sensitive_path(diag, root): + delivery_errors.append(f"Diagnostic path is sensitive or outside repository: {diag.name}") + elif diag.is_file(): + evidence_paths.append(diag) for path in sorted(set(evidence_paths)): resolved = path.resolve() if not resolved.is_file(): continue rel_name = path.relative_to(root).as_posix() if root in resolved.parents else path.name - if is_sensitive_path(path, root): - delivery_errors.append(f"Evidence path is sensitive: {rel_name}") - continue - if file_contains_credentials(path): - delivery_errors.append(f"Evidence file contains credentials: {rel_name}") - continue + _upload_checked(path, f"evidence/{rel_name}") - name = f"evidence/{rel_name}" - dest = f"{target_prefix}/{name}" - if upload_to_gcs(bucket_name, dest, path): - uploaded[name] = f"gs://{bucket_name}/{dest}" - else: - delivery_errors.append(f"Evidence upload failed: {name}") - - if delivery_errors and proposed_pass: + if delivery_errors and status == "PASS": status = "ISSUES" patch_uploaded = False if candidate_files or (status == "PASS" and candidate_files is None): patch_name = "patch.diff" if status == "PASS" else "partial.patch.diff" patch_path = repo / patch_name - if candidate_files or not patch_path.is_file(): - patch_content = _generate_git_patch(repo, candidate_files=candidate_files, commit_sha=commit_sha) - if patch_content: - patch_path.write_text(patch_content, encoding="utf-8") - - if patch_path.is_file() and patch_path.stat().st_size > 0: - if is_sensitive_path(patch_path, root): - delivery_errors.append(f"Candidate patch path is sensitive: {patch_name}") - elif file_contains_credentials(patch_path): - delivery_errors.append(f"Candidate patch contains credentials: {patch_name}") - else: - dest = f"{target_prefix}/{patch_name}" - if upload_to_gcs(bucket_name, dest, patch_path): - uploaded[patch_name] = f"gs://{bucket_name}/{dest}" - patch_uploaded = True - else: - delivery_errors.append(f"Candidate patch upload failed: {patch_name}") + if is_sensitive_path(patch_path, root): + delivery_errors.append(f"Candidate patch path is sensitive: {patch_name}") + if status == "PASS": + status = "ISSUES" + else: + if candidate_files or not patch_path.is_file(): + patch_content = _generate_git_patch(repo, candidate_files=candidate_files, commit_sha=commit_sha) + if patch_content: + patch_path.write_text(patch_content, encoding="utf-8") + + if patch_path.is_file() and patch_path.stat().st_size > 0: + patch_uploaded = _upload_checked(patch_path, patch_name) if candidate_files and not patch_uploaded: delivery_errors.append("Candidate patch failed to upload or is missing under zero-push delivery.") if status == "PASS": status = "ISSUES" - # Upload score.json and results.tsv score_path = repo / "score.json" if score_path.is_file(): - if is_sensitive_path(score_path, root): - delivery_errors.append("score.json is a sensitive path") - elif file_contains_credentials(score_path): - delivery_errors.append("score.json contains credentials") - else: - dest = f"{target_prefix}/score.json" - if upload_to_gcs(bucket_name, dest, score_path): - uploaded["score.json"] = f"gs://{bucket_name}/{dest}" - else: - delivery_errors.append("score.json upload failed") + _upload_checked(score_path, "score.json") results_path = repo / "results.tsv" if results_path.is_file(): - if is_sensitive_path(results_path, root): - delivery_errors.append("results.tsv is a sensitive path") - elif file_contains_credentials(results_path): - delivery_errors.append("results.tsv contains credentials") - else: - dest = f"{target_prefix}/results.tsv" - if upload_to_gcs(bucket_name, dest, results_path): - uploaded["results.tsv"] = f"gs://{bucket_name}/{dest}" - else: - delivery_errors.append("results.tsv upload failed") + _upload_checked(results_path, "results.tsv") - # Finalize delivery_errors and degrade status from PASS to ISSUES if delivery_errors is non-empty if delivery_errors and status == "PASS": status = "ISSUES" - # Write and upload status.json LAST with the finalized status. status_path = repo / "status.json" status_data = {} if status_path.is_file(): diff --git a/skills/swarm/scripts/worker/execution.py b/skills/swarm/scripts/worker/execution.py index 1011f7e..6ae710b 100644 --- a/skills/swarm/scripts/worker/execution.py +++ b/skills/swarm/scripts/worker/execution.py @@ -1,4 +1,3 @@ - import os import re import signal @@ -52,14 +51,10 @@ def _limit_fields(value): return {name: _positive_int(raw, name) for name, raw in value.items()} -def resolve_execution_limits(task_limits=_UNSET, *, environ=None, runtime=_UNSET, cli=_UNSET, command_default=300): +def resolve_execution_limits(task_limits=_UNSET, *, environ=None, command_default=300): env = os.environ if environ is None else environ values = dict(max_agent_turns=30, command_timeout_s=command_default, verify_timeout_s=300) - if runtime is not _UNSET: - values.update(_limit_fields(runtime)) values.update({name: _positive_int(env[key], key) for name, key in LIMIT_ENV.items() if key in env}) - if cli is not _UNSET: - values.update(_limit_fields(cli)) if task_limits is not _UNSET: values.update(_limit_fields(task_limits)) return ExecutionLimits(**values) diff --git a/tests/test_cloud_swarm_canary_integration.py b/tests/test_cloud_swarm_canary_integration.py deleted file mode 100644 index 5dd3a51..0000000 --- a/tests/test_cloud_swarm_canary_integration.py +++ /dev/null @@ -1,1540 +0,0 @@ -import ast -import copy -from dataclasses import FrozenInstanceError -import hashlib -import importlib.util -import io -import json -import os -from pathlib import Path -import re -import shlex -import shutil -import subprocess -import sys -import tarfile -import time -import types -from unittest.mock import MagicMock, patch - -import pytest - -SCRIPTS = Path(__file__).resolve().parents[1] / "skills/swarm/scripts" -sys.path.insert(0, str(SCRIPTS)) - -import cloud_dispatch -import cloud_worker -import result_harvester -import storage_uploader -from worker.execution import ExecutionLimits, resolve_execution_limits, run_bounded_shell - -CANARY_PACKAGE_INVENTORY = Path( - os.environ.get( - "AGYSTACK_CANARY_PACKAGE_INVENTORY", - "/Users/joeltaroreh/projects/phasepoly-swarm-prep/.audit/cloud-swarm-20261024/revision3/package-inventory.json", - ) -) -CANARY_REVISION3_DIR = CANARY_PACKAGE_INVENTORY.parent - - -def _init_git_repo(path: Path) -> Path: - subprocess.run(["git", "init"], cwd=path, check=True, capture_output=True) - subprocess.run(["git", "config", "user.name", "Canary Tester"], cwd=path, check=True, capture_output=True) - subprocess.run(["git", "config", "user.email", "canary@test.local"], cwd=path, check=True, capture_output=True) - (path / "base.txt").write_text("initial base\n", encoding="utf-8") - subprocess.run(["git", "add", "base.txt"], cwd=path, check=True, capture_output=True) - subprocess.run(["git", "commit", "-m", "init base"], cwd=path, check=True, capture_output=True) - return path - - -EXPERIMENT_SCRIPT = """import json, sys -from pathlib import Path - -# Jordan-Wigner Majorana representation for 2 modes (4 operators) -# gamma_0 = X_0, gamma_1 = Y_0, gamma_2 = Z_0 X_1, gamma_3 = Z_0 Y_1 -majoranas = [ - {"x": 1, "z": 0}, - {"x": 1, "z": 1}, - {"x": 2, "z": 1}, - {"x": 2, "z": 3}, -] - -# Check anticommutation violations: (x1*z2 + z1*x2)%2 must be 1 for distinct pairs -violations = sum( - (bin(a["x"] & b["z"]).count("1") + bin(a["z"] & b["x"]).count("1")) % 2 != 1 - for i, a in enumerate(majoranas) - for b in majoranas[i + 1:] -) - -candidate_weight = sum(bin(m["x"] | m["z"]).count("1") for m in majoranas) -baseline_weight = 2 * (2 + 1) - -output_data = { - "size": 2, - "seed": 42, - "candidate_weight": candidate_weight, - "baseline_weight": baseline_weight, - "violations": violations, - "majoranas": majoranas, -} - -out_path = Path("screen.raw.json") -out_path.write_text(json.dumps(output_data, indent=2), encoding="utf-8") -print(f"Computed Majorana algebra: violations={violations}, weight={candidate_weight}") -""" - -DOMAIN_ORACLE_SCRIPT = """import json, sys -from pathlib import Path - -raw_file = Path("screen.raw.json") -if not raw_file.is_file(): - print("Error: screen.raw.json missing", file=sys.stderr) - sys.exit(1) - -data = json.loads(raw_file.read_text(encoding="utf-8")) -if data.get("violations", -1) != 0: - print(f"Oracle verification failed: violations={data.get('violations')}", file=sys.stderr) - sys.exit(1) - -# Exact matrix verification of 4x4 Majorana anticommutators -I = [[1, 0], [0, 1]] -X = [[0, 1], [1, 0]] -Y = [[0, -1j], [1j, 0]] -Z = [[1, 0], [0, -1]] - -def tensor(a, b): - return [[a[i // len(b)][j // len(b)] * b[i % len(b)][j % len(b)] - for j in range(len(a) * len(b))] for i in range(len(a) * len(b))] - -def mat_mul(a, b): - n = len(a) - return [[sum(a[i][k] * b[k][j] for k in range(n)) for j in range(n)] for i in range(n)] - -def matrix(mask): - out = [[1]] - for k in range(2): - x, z = (mask["x"] >> k) & 1, (mask["z"] >> k) & 1 - out = tensor(out, Y if x and z else X if x else Z if z else I) - return out - -matrices = [matrix(m) for m in data["majoranas"]] -sample_count = 0 -for i, a in enumerate(matrices): - for j, b in enumerate(matrices): - ab, ba = mat_mul(a, b), mat_mul(b, a) - expected = 2 if i == j else 0 - for r in range(4): - for c in range(4): - val = ab[r][c] + ba[r][c] - target = expected if r == c else 0 - if abs(val - target) > 1e-10: - print(f"Algebra violation at ({i},{j}) element ({r},{c}): {val} != {target}", file=sys.stderr) - sys.exit(1) - sample_count += 1 - -oracle_res = { - "checks": [ - {"name": "majorana_anticommutators_4x4", "passed": True, "sample_count": sample_count} - ] -} -Path("oracle_result.json").write_text(json.dumps(oracle_res, indent=2), encoding="utf-8") -print(f"Oracle verified {sample_count} matrix checks cleanly.") -""" - - -def test_case_1_real_scientific_computation_harvests_exact_bytes(tmp_path, monkeypatch): - """Case 1: A tiny real scientific computation produces raw numeric output and a sound domain oracle. - - Safe evidence is delivered and independently harvested under the exact canary session. - Assert harvested status is PASS, raw outputs and evidence files are preserved with exact bytes. - """ - repo = tmp_path / "canary_worker_repo" - repo.mkdir() - _init_git_repo(repo) - - task_dir = repo / "research/canary-task-0" - task_dir.mkdir(parents=True) - (task_dir / "experiment.py").write_text(EXPERIMENT_SCRIPT, encoding="utf-8") - (task_dir / "oracle.py").write_text(DOMAIN_ORACLE_SCRIPT, encoding="utf-8") - - # Run the real scientific computation - calc_res = subprocess.run([sys.executable, "experiment.py"], cwd=task_dir, capture_output=True, text=True, check=True) - assert "Computed Majorana algebra: violations=0" in calc_res.stdout - raw_path = task_dir / "screen.raw.json" - assert raw_path.is_file() - expected_raw_bytes = raw_path.read_bytes() - expected_raw_sha = hashlib.sha256(expected_raw_bytes).hexdigest() - - # Run the sound domain oracle - oracle_res = subprocess.run([sys.executable, "oracle.py"], cwd=task_dir, capture_output=True, text=True, check=True) - assert "Oracle verified 16 matrix checks cleanly." in oracle_res.stdout - oracle_path = task_dir / "oracle_result.json" - assert oracle_path.is_file() - expected_oracle_bytes = oracle_path.read_bytes() - expected_oracle_sha = hashlib.sha256(expected_oracle_bytes).hexdigest() - - # Track modified candidate file - subprocess.run(["git", "add", "research/canary-task-0/experiment.py"], cwd=repo, check=True) - - fake_storage = tmp_path / "fake_storage" - session_id = "canary-session" - bucket_name = "canary-bucket" - gcs_prefix = f"swarms/{session_id}" - - task_item = { - "task_index": 0, - "brief": "Run minimal Majorana algebra screen", - "candidate_files": ["research/canary-task-0/experiment.py"], - "evidence_files": [ - "research/canary-task-0/screen.raw.json", - "research/canary-task-0/oracle_result.json", - ], - "verify_command": f"{shlex.quote(sys.executable)} oracle.py", - "execution_limits": {"verify_timeout_s": 30, "command_timeout_s": 30}, - } - - monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) - - # Perform artifact upload under the exact canary session layout - uploaded = storage_uploader.upload_run_artifacts( - bucket_name=bucket_name, - prefix=gcs_prefix, - repo_dir=repo, - task_index=0, - status="PASS", - candidate_files=task_item["candidate_files"], - evidence_files=task_item["evidence_files"], - ) - - # Verify object layout in fake storage: swarms//task-/ - task_storage = fake_storage / bucket_name / gcs_prefix / "task-0" - assert task_storage.is_dir() - assert (task_storage / "status.json").is_file() - assert (task_storage / "patch.diff").is_file() - storage_raw = task_storage / "evidence/research/canary-task-0/screen.raw.json" - storage_oracle = task_storage / "evidence/research/canary-task-0/oracle_result.json" - assert storage_raw.is_file() - assert storage_oracle.is_file() - - # Byte-exact preservation in storage - assert storage_raw.read_bytes() == expected_raw_bytes - assert hashlib.sha256(storage_raw.read_bytes()).hexdigest() == expected_raw_sha - assert storage_oracle.read_bytes() == expected_oracle_bytes - assert hashlib.sha256(storage_oracle.read_bytes()).hexdigest() == expected_oracle_sha - - # Harvest independently under the exact canary session - harvest_dest = tmp_path / "harvested" - harvested = result_harvester.harvest_candidate_patches( - bucket_name=bucket_name, - prefix=gcs_prefix, - dest_dir=harvest_dest, - ) - - assert len(harvested) == 1 - item = harvested[0] - assert item["task_index"] == 0 - assert item["status"] == "PASS" - assert item["patch_file"] is not None and Path(item["patch_file"]).is_file() - - harvested_raw = harvest_dest / "task-0/evidence/research/canary-task-0/screen.raw.json" - harvested_oracle = harvest_dest / "task-0/evidence/research/canary-task-0/oracle_result.json" - assert harvested_raw.is_file() - assert harvested_oracle.is_file() - - # Assert exact byte preservation after transport and harvesting - assert harvested_raw.read_bytes() == expected_raw_bytes - assert hashlib.sha256(harvested_raw.read_bytes()).hexdigest() == expected_raw_sha - assert harvested_oracle.read_bytes() == expected_oracle_bytes - assert hashlib.sha256(harvested_oracle.read_bytes()).hexdigest() == expected_oracle_sha - - -def test_case_2_wrong_scientific_result_fails_verification_and_degrades_to_issues(tmp_path, monkeypatch): - """Case 2: An intentionally wrong scientific result fails verification even when the agent - - requests PASS and makes no source change. Status is degraded to ISSUES and harvested status is ISSUES. - """ - repo = tmp_path / "canary_worker_repo_c2" - repo.mkdir() - _init_git_repo(repo) - - task_dir = repo / "research/canary-task-0" - task_dir.mkdir(parents=True) - (task_dir / "oracle.py").write_text(DOMAIN_ORACLE_SCRIPT, encoding="utf-8") - - # Intentionally corrupt the raw scientific output (violations = 2) - corrupted_data = { - "size": 2, - "seed": 42, - "candidate_weight": 6, - "baseline_weight": 6, - "violations": 2, - "majoranas": [{"x": 1, "z": 0}, {"x": 1, "z": 0}, {"x": 2, "z": 1}, {"x": 2, "z": 3}], - } - (task_dir / "screen.raw.json").write_text(json.dumps(corrupted_data, indent=2), encoding="utf-8") - - # No source changes made to candidate files - fake_storage = tmp_path / "fake_storage" - session_id = "canary-session" - bucket_name = "canary-bucket" - gcs_prefix = f"swarms/{session_id}" - - task_item = { - "task_index": 0, - "brief": "Run screen with corrupted algebra", - "candidate_files": ["research/canary-task-0/calc.py"], - "evidence_files": ["research/canary-task-0/screen.raw.json"], - "verify_command": f"{shlex.quote(sys.executable)} research/canary-task-0/oracle.py", - "execution_limits": {"verify_timeout_s": 10}, - } - - monkeypatch.setenv("REPO_URL", "https://example.com/repo") - monkeypatch.setenv("TASK_MANIFEST", json.dumps([task_item])) - monkeypatch.setenv("USE_VERTEX_AI", "1") - monkeypatch.setenv("CLOUD_RUN_TASK_INDEX", "0") - monkeypatch.setenv("SWARM_SESSION_ID", session_id) - monkeypatch.setenv("GCS_RESULTS_BUCKET", bucket_name) - monkeypatch.setenv("GCS_BUCKET", bucket_name) - monkeypatch.setenv("GCS_PREFIX", gcs_prefix) - monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) - monkeypatch.setattr(cloud_worker.tempfile, "gettempdir", lambda: str(tmp_path)) - - def fake_clone(**kwargs): - dest_repo = kwargs["repo_dir"] - _init_git_repo(dest_repo) - t_dir = dest_repo / "research/canary-task-0" - t_dir.mkdir(parents=True, exist_ok=True) - (t_dir / "oracle.py").write_text(DOMAIN_ORACLE_SCRIPT, encoding="utf-8") - corrupted_data = { - "size": 2, - "seed": 42, - "candidate_weight": 6, - "baseline_weight": 6, - "violations": 2, - "majoranas": [{"x": 1, "z": 0}, {"x": 1, "z": 0}, {"x": 2, "z": 1}, {"x": 2, "z": 3}], - } - (t_dir / "screen.raw.json").write_text(json.dumps(corrupted_data, indent=2), encoding="utf-8") - - # Mock execute_task so agent requests PASS without modifying candidate files - def fake_execute(**kwargs): - return "PASS", "Agent claimed pass with no source changes." - - monkeypatch.setattr(cloud_worker, "clone_and_checkout_task", fake_clone) - monkeypatch.setattr(cloud_worker, "execute_task", fake_execute) - monkeypatch.setattr(cloud_worker, "StorageMessenger", None) - - orig_cwd = os.getcwd() - try: - # cloud_worker.main() should run, run verify_command which fails, degrade to ISSUES, and exit(1) - with pytest.raises(SystemExit) as exc_info: - cloud_worker.main() - assert exc_info.value.code == 1 - finally: - os.chdir(orig_cwd) - - # Check status.json locally in the task repo and in storage - active_repo = tmp_path / "repo-task-0" - status_file = active_repo / "status.json" - assert status_file.is_file() - status_data = json.loads(status_file.read_text(encoding="utf-8")) - assert status_data["status"] == "ISSUES" - assert "Verification:" in status_data["summary"] - - # Harvest and verify that harvested status is ISSUES and patch is quarantined - harvest_dest = tmp_path / "harvested_c2" - harvested = result_harvester.harvest_candidate_patches( - bucket_name=bucket_name, - prefix=gcs_prefix, - dest_dir=harvest_dest, - ) - assert len(harvested) == 1 - assert harvested[0]["task_index"] == 0 - assert harvested[0]["status"] == "ISSUES" - assert harvested[0]["patch_file"] is None - - -def test_case_3_command_timeout_diagnostics_retained_no_orphans_issues(tmp_path): - """Case 3: A bounded command times out (e.g. 1s timeout on a 3s sleep). - - Diagnostics retain partial output, child processes terminate without orphans, status is not PASS. - """ - repo = tmp_path / "canary_worker_repo_c3" - repo.mkdir() - _init_git_repo(repo) - - child_script = ( - "import os, time; " - "open('child.pid', 'w').write(str(os.getpid())); " - "time.sleep(4); " - "open('survived.txt', 'w').write('orphaned');" - ) - command = ( - f"echo 'partial-stdout-line-1'; echo 'partial-stderr-line-1' >&2; " - f"{shlex.quote(sys.executable)} -c {shlex.quote(child_script)}" - ) - - res, output = run_bounded_shell(command, repo, timeout_s=1, label="canary-timeout") - - # Command timed out -> exit code 124 - assert res.returncode == 124 - assert "Command timed out after 1 seconds" in res.stderr - assert "partial-stdout-line-1" in output - assert "partial-stderr-line-1" in output - - # Diagnostics retained in log file - diag_logs = list((repo / ".agystack/diagnostics").glob("canary-timeout-*.log")) - assert len(diag_logs) == 1 - log_content = diag_logs[0].read_text(encoding="utf-8") - assert "partial-stdout-line-1" in log_content - assert "partial-stderr-line-1" in log_content - - # Child process was terminated without orphans - pid_file = repo / "child.pid" - assert pid_file.is_file() - child_pid = int(pid_file.read_text(encoding="utf-8").strip()) - - time.sleep(1.0) - assert not (repo / "survived.txt").exists() - - # Check process table: process should not be actively running - proc_check = subprocess.run(["ps", "-o", "stat=", "-p", str(child_pid)], capture_output=True, text=True) - stat = proc_check.stdout.strip() - assert not stat or stat.startswith("Z") # Dead or zombie - - -def test_case_4_synthetic_credentials_in_evidence_rejected_intact_on_disk_issues(tmp_path): - """Case 4: A synthetic credential (e.g. AIza... or ghp_... token pattern) in declared evidence - - is rejected. The safe artifact's bytes remain intact on disk, no credential appears in published objects - or diagnostics, delivery_errors records the error, and status is degraded to ISSUES. - """ - repo = tmp_path / "canary_worker_repo_c4" - repo.mkdir() - _init_git_repo(repo) - - evidence_dir = repo / "research/canary-task-0" - evidence_dir.mkdir(parents=True) - - # 1. Safe artifact - safe_file = evidence_dir / "safe_computation.json" - safe_content = json.dumps({"metric": "majorana_parity", "value": 1.0, "clean": True}, indent=2) - safe_file.write_text(safe_content, encoding="utf-8") - safe_bytes_before = safe_file.read_bytes() - safe_sha_before = hashlib.sha256(safe_bytes_before).hexdigest() - - # 2. File with synthetic credentials - secret_ghp = "ghp_" + "C" * 36 - secret_aiza = "AIza" + "D" * 35 - cred_file = evidence_dir / "leaked_evidence.txt" - cred_content = f"Debug log containing sensitive tokens: {secret_ghp} and {secret_aiza}\n" - cred_file.write_text(cred_content, encoding="utf-8") - - fake_storage = tmp_path / "fake_storage" - session_id = "canary-session" - bucket_name = "canary-bucket" - gcs_prefix = f"swarms/{session_id}" - - with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage)}): - results = storage_uploader.upload_run_artifacts( - bucket_name=bucket_name, - prefix=gcs_prefix, - repo_dir=repo, - task_index=0, - status="PASS", - evidence_files=[ - "research/canary-task-0/safe_computation.json", - "research/canary-task-0/leaked_evidence.txt", - ], - ) - - # Safe artifact bytes remain intact on disk - assert safe_file.read_bytes() == safe_bytes_before - assert hashlib.sha256(safe_file.read_bytes()).hexdigest() == safe_sha_before - - # Credential file on disk is not silently modified or wiped - assert cred_file.read_text(encoding="utf-8") == cred_content - - # Credential file was REJECTED from uploaded artifacts - assert "evidence/research/canary-task-0/leaked_evidence.txt" not in results - assert "evidence/research/canary-task-0/safe_computation.json" in results - - # No credential appears anywhere in published objects in storage - task_storage = fake_storage / bucket_name / gcs_prefix / "task-0" - assert task_storage.is_dir() - for pub_file in task_storage.rglob("*"): - if pub_file.is_file(): - content = pub_file.read_text(encoding="utf-8", errors="ignore") - assert secret_ghp not in content - assert secret_aiza not in content - - # Diagnostics do not leak credentials - diag_dir = repo / ".agystack/diagnostics" - if diag_dir.is_dir(): - for dfile in diag_dir.glob("*.log"): - dcontent = dfile.read_text(encoding="utf-8", errors="ignore") - assert secret_ghp not in dcontent - assert secret_aiza not in dcontent - - # delivery_errors in status.json records the error and status is degraded to ISSUES - status_file = repo / "status.json" - assert status_file.is_file() - status_data = json.loads(status_file.read_text(encoding="utf-8")) - assert status_data["status"] == "ISSUES" - assert "delivery_errors" in status_data - assert any("contains credentials" in err for err in status_data["delivery_errors"]) - - # Harvested status is ISSUES - harvested = result_harvester.harvest_candidate_patches( - bucket_name=bucket_name, - prefix=gcs_prefix, - dest_dir=tmp_path / "harvest_c4", - ) - assert len(harvested) == 1 - assert harvested[0]["status"] == "ISSUES" - - -def test_case_5_failed_score_or_results_upload_produces_delivery_errors_and_issues(tmp_path): - """Case 5: A failed score or results upload produces delivery errors and ISSUES. - - Test both candidate files and no-change tasks. - """ - fake_storage = tmp_path / "fake_storage_c5" - bucket_name = "canary-bucket" - - # 5a: With candidate files - repo_cand = tmp_path / "canary_worker_repo_c5_cand" - repo_cand.mkdir() - _init_git_repo(repo_cand) - - (repo_cand / "experiment.py").write_text("updated = True\n", encoding="utf-8") - (repo_cand / "score.json").write_text(json.dumps({"score": 0.95}), encoding="utf-8") - (repo_cand / "results.tsv").write_text("metric\tvalue\nscore\t0.95\n", encoding="utf-8") - - def fail_score_upload(bucket, obj_name, file_path): - if "score.json" in str(obj_name): - return False - # Save to local storage for realistic harvesting - dest = fake_storage / bucket / obj_name - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(file_path, dest) - return True - - prefix_cand = "swarms/canary-session-cand" - with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage)}): - with patch.object(storage_uploader, "upload_to_gcs", side_effect=fail_score_upload): - uploaded_cand = storage_uploader.upload_run_artifacts( - bucket_name=bucket_name, - prefix=prefix_cand, - repo_dir=repo_cand, - task_index=0, - status="PASS", - candidate_files=["experiment.py"], - ) - - assert "score.json" not in uploaded_cand - status_file_cand = repo_cand / "status.json" - status_data_cand = json.loads(status_file_cand.read_text(encoding="utf-8")) - assert status_data_cand["status"] == "ISSUES" - assert "delivery_errors" in status_data_cand - assert any("score.json upload failed" in err for err in status_data_cand["delivery_errors"]) - - harvested_cand = result_harvester.harvest_candidate_patches( - bucket_name=bucket_name, - prefix=prefix_cand, - dest_dir=tmp_path / "harvest_c5_cand", - ) - assert len(harvested_cand) == 1 - assert harvested_cand[0]["status"] == "ISSUES" - # Non-PASS candidate patch is quarantined and not promoted - assert harvested_cand[0]["patch_file"] is None - - # 5b: No-change tasks (clean working directory, no candidate files modified) - repo_clean = tmp_path / "canary_worker_repo_c5_clean" - repo_clean.mkdir() - _init_git_repo(repo_clean) - - (repo_clean / "score.json").write_text(json.dumps({"score": 0.90}), encoding="utf-8") - (repo_clean / "results.tsv").write_text("metric\tvalue\nscore\t0.90\n", encoding="utf-8") - - def fail_results_upload(bucket, obj_name, file_path): - if "results.tsv" in str(obj_name): - return False - dest = fake_storage / bucket / obj_name - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(file_path, dest) - return True - - prefix_clean = "swarms/canary-session-clean" - with patch.dict(os.environ, {"STORAGE_MESSENGER_LOCAL_DIR": str(fake_storage)}): - with patch.object(storage_uploader, "upload_to_gcs", side_effect=fail_results_upload): - uploaded_clean = storage_uploader.upload_run_artifacts( - bucket_name=bucket_name, - prefix=prefix_clean, - repo_dir=repo_clean, - task_index=0, - status="PASS", - candidate_files=[], - ) - - assert "results.tsv" not in uploaded_clean - status_file_clean = repo_clean / "status.json" - status_data_clean = json.loads(status_file_clean.read_text(encoding="utf-8")) - assert status_data_clean["status"] == "ISSUES" - assert "delivery_errors" in status_data_clean - assert any("results.tsv upload failed" in err for err in status_data_clean["delivery_errors"]) - - harvested_clean = result_harvester.harvest_candidate_patches( - bucket_name=bucket_name, - prefix=prefix_clean, - dest_dir=tmp_path / "harvest_c5_clean", - ) - assert len(harvested_clean) == 1 - assert harvested_clean[0]["status"] == "ISSUES" - assert harvested_clean[0]["patch_file"] is None - - - - -def test_case_6_sibling_prefix_isolation(tmp_path, monkeypatch): - """Case 6: Sibling prefix isolation: swarms/canary-other/ cannot satisfy canary coverage - - or contribute a promotable patch in result_harvester. - """ - fake_storage = tmp_path / "storage" - canary_dir = fake_storage / "bucket/swarms/canary/task-0" - canary_dir.mkdir(parents=True) - (canary_dir / "status.json").write_text(json.dumps({"task_index": 0, "status": "PASS", "score": 0.80})) - (canary_dir / "score.json").write_text(json.dumps({"score": 0.80})) - (canary_dir / "patch.diff").write_text("diff --git a/canary b/canary\n+canary") - - sibling_dir = fake_storage / "bucket/swarms/canary-other/task-1" - sibling_dir.mkdir(parents=True) - (sibling_dir / "status.json").write_text(json.dumps({"task_index": 1, "status": "PASS", "score": 0.99})) - (sibling_dir / "score.json").write_text(json.dumps({"score": 0.99})) - (sibling_dir / "patch.diff").write_text("diff --git a/sibling b/sibling\n+sibling") - - sibling_task0 = fake_storage / "bucket/swarms/canary-other/task-0" - sibling_task0.mkdir(parents=True) - (sibling_task0 / "status.json").write_text(json.dumps({"task_index": 0, "status": "PASS", "score": 0.99})) - - monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) - - # Harvest for swarms/canary - harvest_dest = tmp_path / "harvest" - harvested = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", harvest_dest) - - # Must contain ONLY task-0 from canary, score 0.80 - assert len(harvested) == 1 - assert harvested[0]["task_index"] == 0 - assert harvested[0]["score"] == 0.80 - assert Path(harvested[0]["patch_file"]).read_text() == "diff --git a/canary b/canary\n+canary" - - # Sibling prefix canary-other cannot satisfy canary coverage if 2 tasks are expected - assert cloud_dispatch.failed_task_count(harvested, 2) == 1 - - # Also test GCS mock with trailing-slash prefix isolation - monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) - blobs = { - "swarms/canary/task-0/status.json": json.dumps({"task_index": 0, "status": "PASS", "score": 0.80}), - "swarms/canary/task-0/patch.diff": "canary patch", - "swarms/canary-other/task-1/status.json": json.dumps({"task_index": 1, "status": "PASS", "score": 0.99}), - "swarms/canary-other/task-1/patch.diff": "sibling patch", - } - - class Blob: - def __init__(self, name, payload): - self.name, self.payload = name, payload - - def download_to_filename(self, filename): - Path(filename).parent.mkdir(parents=True, exist_ok=True) - Path(filename).write_text(self.payload, encoding="utf-8") - - mock_storage = MagicMock() - mock_storage.Client.return_value.bucket.return_value.list_blobs.side_effect = ( - lambda prefix="": [Blob(k, v) for k, v in blobs.items() if k.startswith(prefix)] - ) - - google_mod = types.ModuleType("google") - cloud_mod = types.ModuleType("google.cloud") - google_mod.cloud = cloud_mod - cloud_mod.storage = mock_storage - - with patch.dict(sys.modules, {"google": google_mod, "google.cloud": cloud_mod, "google.cloud.storage": mock_storage}): - harvest_gcs = tmp_path / "harvest_gcs" - gcs_results = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", harvest_gcs) - - assert len(gcs_results) == 1 - assert gcs_results[0]["task_index"] == 0 - assert gcs_results[0]["score"] == 0.80 - assert Path(gcs_results[0]["patch_file"]).read_text() == "canary patch" - - -def test_task_level_execution_limits_override_cli_env_runtime_defaults(): - """Verify that task-level execution limits override CLI, environment, runtime config, and defaults. - - Also verify immutability and rejection of invalid values. - """ - # 1. Defaults - default_limits = resolve_execution_limits(environ={}) - assert (default_limits.max_agent_turns, default_limits.command_timeout_s, default_limits.verify_timeout_s) == (30, 300, 300) - - # 2. Runtime overrides defaults - runtime_limits = resolve_execution_limits( - environ={}, - runtime={"max_agent_turns": 40, "command_timeout_s": 350}, - ) - assert (runtime_limits.max_agent_turns, runtime_limits.command_timeout_s, runtime_limits.verify_timeout_s) == (40, 350, 300) - - # 3. Environment overrides runtime - env_limits = resolve_execution_limits( - environ={"AGYSTACK_MAX_AGENT_TURNS": "50", "COMMAND_TIMEOUT": "450"}, - runtime={"max_agent_turns": 40, "command_timeout_s": 350}, - ) - assert (env_limits.max_agent_turns, env_limits.command_timeout_s, env_limits.verify_timeout_s) == (50, 450, 300) - - # 4. CLI overrides environment - cli_limits = resolve_execution_limits( - environ={"AGYSTACK_MAX_AGENT_TURNS": "50", "COMMAND_TIMEOUT": "450"}, - runtime={"max_agent_turns": 40, "command_timeout_s": 350}, - cli={"max_agent_turns": 60}, - ) - assert (cli_limits.max_agent_turns, cli_limits.command_timeout_s, cli_limits.verify_timeout_s) == (60, 450, 300) - - # 5. Task-level limits override CLI, environment, runtime, and defaults - task_limits = resolve_execution_limits( - {"max_agent_turns": 80, "command_timeout_s": 700, "verify_timeout_s": 900}, - environ={"AGYSTACK_MAX_AGENT_TURNS": "50", "COMMAND_TIMEOUT": "450", "VERIFY_TIMEOUT": "550"}, - runtime={"max_agent_turns": 40, "command_timeout_s": 350, "verify_timeout_s": 400}, - cli={"max_agent_turns": 60, "command_timeout_s": 500}, - ) - assert (task_limits.max_agent_turns, task_limits.command_timeout_s, task_limits.verify_timeout_s) == (80, 700, 900) - - # Immutability - with pytest.raises(FrozenInstanceError): - task_limits.max_agent_turns = 100 - - # Invalid values rejected - for invalid in [0, -1, True, False, 2.5, "ten", None]: - with pytest.raises(ValueError): - resolve_execution_limits({"max_agent_turns": invalid}, environ={}) - - -def test_fake_storage_reproduces_layout_trailing_slash_and_nested_paths(tmp_path, monkeypatch): - """Verify fake storage reproduces swarms//task-/ layout, trailing-slash list prefixes, - - and nested task paths with exact byte preservation. - """ - repo = tmp_path / "layout_repo" - repo.mkdir() - _init_git_repo(repo) - - # Create nested evidence directories and files - deep_evidence = repo / "nested/domain/oracle/checks.json" - deep_evidence.parent.mkdir(parents=True) - deep_content = json.dumps({"depth": 3, "oracle_verified": True}, indent=2) - deep_evidence.write_text(deep_content, encoding="utf-8") - - fake_storage = tmp_path / "fake_storage" - monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) - - session_id = "canary-layout-session" - bucket_name = "test-layout-bucket" - prefix = f"swarms/{session_id}" - - storage_uploader.upload_run_artifacts( - bucket_name=bucket_name, - prefix=prefix, - repo_dir=repo, - task_index=0, - status="PASS", - evidence_files=["nested/domain/oracle/checks.json"], - ) - - # Verify exact layout in fake storage - expected_storage_path = ( - fake_storage - / bucket_name - / prefix - / "task-0" - / "evidence/nested/domain/oracle/checks.json" - ) - assert expected_storage_path.is_file() - assert expected_storage_path.read_text(encoding="utf-8") == deep_content - - # Harvest nested paths - harvest_dest = tmp_path / "harvest_layout" - harvested = result_harvester.harvest_candidate_patches(bucket_name, prefix, harvest_dest) - assert len(harvested) == 1 - harvested_nested = harvest_dest / "task-0/evidence/nested/domain/oracle/checks.json" - assert harvested_nested.is_file() - assert harvested_nested.read_text(encoding="utf-8") == deep_content - - -def test_verify_upload_mapping_against_package_inventory(tmp_path): - """Verify proposed upload mapping locally against the package inventory (package-inventory.json). - - Checks that all listed files exist, sha256 hashes match on disk, and map cleanly to preparation prefixes. - """ - inventory_path = Path( - os.environ.get("AGYSTACK_CANARY_PACKAGE_INVENTORY") or CANARY_PACKAGE_INVENTORY - ) - if not inventory_path.is_file(): - fixture_dir = tmp_path / "canary_fixture" - fixture_dir.mkdir(parents=True, exist_ok=True) - packets_dir = fixture_dir / "packets" - packets_dir.mkdir(parents=True, exist_ok=True) - - inventory_data = {} - key_files = { - "bootstrap.py": "#!/usr/bin/env python3\nprint('bootstrapping')\n", - "verify_research.py": "#!/usr/bin/env python3\nprint('verifying')\n", - "EVIDENCE_SCHEMA.md": "# Evidence Schema\n", - } - for filename, content in key_files.items(): - file_path = fixture_dir / filename - file_path.write_text(content, encoding="utf-8") - inventory_data[filename] = hashlib.sha256(file_path.read_bytes()).hexdigest() - - for i in range(1, 25): - packet_name = f"packets/task-{i:02d}.md" - packet_path = fixture_dir / packet_name - packet_path.write_text(f"# Task {i:02d}\n", encoding="utf-8") - inventory_data[packet_name] = hashlib.sha256(packet_path.read_bytes()).hexdigest() - - inventory_path = fixture_dir / "package-inventory.json" - inventory_path.write_text(json.dumps(inventory_data, indent=2), encoding="utf-8") - revision_dir = fixture_dir - else: - revision_dir = inventory_path.parent - - assert inventory_path.is_file(), f"Package inventory missing at {inventory_path}" - inventory = json.loads(inventory_path.read_text(encoding="utf-8")) - assert len(inventory) >= 27, f"Expected at least 27 inventory files, found {len(inventory)}" - - # Required key files - assert "bootstrap.py" in inventory - assert "verify_research.py" in inventory - assert "EVIDENCE_SCHEMA.md" in inventory - assert any(re.match(r"^packets/task-\d{2}\.md$", k) for k in inventory) - - canary_session = "phasepoly24-canary-20261025" - preparation_prefix = f"swarms/{canary_session}/preparation" - - upload_mapping = {} - for rel_path, expected_sha in inventory.items(): - file_path = revision_dir / rel_path - assert file_path.is_file(), f"Inventory item {rel_path} does not exist at {file_path}" - actual_sha = hashlib.sha256(file_path.read_bytes()).hexdigest() - assert actual_sha == expected_sha, ( - f"Hash mismatch for {rel_path}: expected {expected_sha}, got {actual_sha}" - ) - # Proposed destination in bucket - dest_blob = f"{preparation_prefix}/{rel_path}" - upload_mapping[rel_path] = dest_blob - - # Verify every destination blob has proper prefix and valid layout - for rel_path, dest_blob in upload_mapping.items(): - assert dest_blob.startswith(f"{preparation_prefix}/") - assert ".." not in dest_blob - assert not dest_blob.endswith("/") - - -EXPERIMENT_CANARY_SCRIPT = """import argparse, json -from pathlib import Path -p = argparse.ArgumentParser() -p.add_argument('--size', type=int, required=True) -p.add_argument('--seed', type=int, required=True) -p.add_argument('--output', required=True) -args = p.parse_args() - -majoranas = [ - {'x': 1, 'z': 0}, - {'x': 1, 'z': 1}, - {'x': 2, 'z': 1}, - {'x': 2, 'z': 3}, -] -if args.size == 1: - majoranas = majoranas[:2] - -violations = sum( - (bin(a['x'] & b['z']).count('1') + bin(a['z'] & b['x']).count('1')) % 2 != 1 - for i, a in enumerate(majoranas) - for b in majoranas[i + 1:] -) -weight = sum(bin(m['x'] | m['z']).count('1') for m in majoranas) -out = { - 'size': args.size, - 'seed': args.seed, - 'candidate_weight': weight, - 'baseline_weight': args.size * (args.size + 1), - 'violations': violations, - 'majoranas': majoranas, -} -Path(args.output).write_text(json.dumps(out, indent=2), encoding='utf-8') -""" - -DOMAIN_ORACLE_CANARY_SCRIPT = """import json, sys -from pathlib import Path - -raw = json.loads(Path('screen.raw.json').read_text(encoding='utf-8')) -I = [[1, 0], [0, 1]] -X = [[0, 1], [1, 0]] -Y = [[0, -1j], [1j, 0]] -Z = [[1, 0], [0, -1]] - -def tensor(a, b): - return [[a[i // len(b)][j // len(b)] * b[i % len(b)][j % len(b)] - for j in range(len(a) * len(b))] for i in range(len(a) * len(b))] - -def mat_mul(a, b): - n = len(a) - return [[sum(a[i][k] * b[k][j] for k in range(n)) for j in range(n)] for i in range(n)] - -def matrix(mask): - out = [[1]] - for k in range(2): - x, z = (mask['x'] >> k) & 1, (mask['z'] >> k) & 1 - out = tensor(out, Y if x and z else X if x else Z if z else I) - return out - -matrices = [matrix(m) for m in raw['majoranas']] -count = 0 -for i, a in enumerate(matrices): - for j, b in enumerate(matrices): - ab, ba = mat_mul(a, b), mat_mul(b, a) - expected = 2 if i == j else 0 - for r in range(4): - for c in range(4): - val = ab[r][c] + ba[r][c] - target = expected if r == c else 0 - assert abs(val - target) < 1e-10 - count += 1 - -print(json.dumps({'checks': [{'name': 'explicit_matrix_majorana_anticommutators', 'passed': True, 'sample_count': count}]})) -""" - - -def _load_canary_manifest_and_revision_dir(tmp_path: Path): - manifest_path = CANARY_REVISION3_DIR / "canary-manifest.json" - if manifest_path.is_file(): - revision_dir = CANARY_REVISION3_DIR - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - return manifest, revision_dir - - # Fallback fixture if revision3 is missing - fixture_dir = tmp_path / "canary_fallback_fixture" - fixture_dir.mkdir(parents=True, exist_ok=True) - packets_dir = fixture_dir / "packets" - packets_dir.mkdir(parents=True, exist_ok=True) - - task_packet = packets_dir / "task-01.md" - task_packet.write_text( - "You own the standalone canary verification screen.\n" - "Scientific pursuit (verdict: pursue) requires meeting your declared scientific acceptance rule with verified controls.\n", - encoding="utf-8", - ) - packet_sha = hashlib.sha256(task_packet.read_bytes()).hexdigest() - - schema_file = fixture_dir / "EVIDENCE_SCHEMA.md" - schema_file.write_text("# Evidence Schema\n", encoding="utf-8") - schema_sha = hashlib.sha256(schema_file.read_bytes()).hexdigest() - - verifier_file = fixture_dir / "verify_research.py" - verifier_code = ( - "import argparse, json, subprocess, sys\n" - "from pathlib import Path\n" - "p = argparse.ArgumentParser()\n" - "p.add_argument('--task-dir', required=True)\n" - "p.add_argument('--expected-json', required=True)\n" - "args = p.parse_args()\n" - "td = Path(args.task_dir)\n" - "rep = json.loads((td / 'report.json').read_text())\n" - "dc = json.loads((td / 'domain_check.json').read_text())\n" - "res = subprocess.run(dc['oracle_command'], cwd=td, capture_output=True)\n" - "if res.returncode != 0: sys.exit(1)\n" - "if rep.get('baseline_gate', {}).get('exit_code') != 0: sys.exit(1)\n" - "if rep.get('target_modified') and rep.get('final_gate', {}).get('exit_code') != 0: sys.exit(1)\n" - "print(json.dumps({'execution_status': 'PASS'}))\n" - ) - verifier_file.write_text(verifier_code, encoding="utf-8") - verifier_sha = hashlib.sha256(verifier_file.read_bytes()).hexdigest() - - bootstrap_file = fixture_dir / "bootstrap.py" - bootstrap_code = ( - "import hashlib, json, re\n" - "from pathlib import Path\n\n" - "def sha(path):\n" - " return hashlib.sha256(Path(path).read_bytes()).hexdigest()\n\n" - "def main(config):\n" - " from google.cloud import storage\n" - " assert re.fullmatch(r'task-(?:0[1-9]|1[0-9]|2[0-4])', config['task_id'])\n" - " bucket = storage.Client(project=config['project']).bucket(config['bucket'])\n" - " output = Path.cwd() / 'research' / config['task_id']\n" - " output.mkdir(parents=True, exist_ok=True)\n" - " archive = Path('/tmp/phasepoly-source.tar.gz')\n" - " bucket.blob(config['prefix'] + '/source.tar.gz').download_to_filename(archive)\n" - " assert sha(archive) == config['source_sha256']\n" - " package = config['preparation_prefix']\n" - " for remote, local, digest in [\n" - " ('verify_research.py', 'verify_research.py', config['verifier_sha256']),\n" - " ('EVIDENCE_SCHEMA.md', 'EVIDENCE_SCHEMA.md', config['schema_sha256']),\n" - " ('packets/' + config['task_id'] + '.md', 'context.md', config['context_packet_sha256']),\n" - " ]:\n" - " path = output / local\n" - " bucket.blob(package + '/' + remote).download_to_filename(path)\n" - " assert sha(path) == digest\n" - ) - bootstrap_file.write_text(bootstrap_code, encoding="utf-8") - bootstrap_sha = hashlib.sha256(bootstrap_file.read_bytes()).hexdigest() - - task_config = { - "project": "agystack-test-deepseek-88", - "bucket": "agystack-test-deepseek-88-swarm-results", - "prefix": "swarms/phasepoly24-general-20261024/source", - "environment_prefix": "swarms/phasepoly24-20261024/source", - "source_sha256": "2e9de7d3064b8a7d3510f57510f1ad9eb4c8faecbed3e3a2c9a474ef7a746444", - "environment_sha256": "480d7dba8cd2bd44c0465cdfc33974e3f4681542c8bed65bfc7b7e608b1f03f7", - "brief_document_sha256": "21fc20120b31d2269ddc6d137d2a56f8811f6d94ec037a20fcbf0e9afd28a70e", - "session": "phasepoly24-canary-20261025", - "preparation_prefix": "swarms/phasepoly24-canary-20261025/preparation", - "output_prefix": "swarms/phasepoly24-canary-20261025/evidence", - "source_base_commit": "d2966d02409a0c7ac749636b97b9e95947c10a9a", - "execution_limits": {"max_agent_turns": 30, "command_timeout_s": 300, "verify_timeout_s": 120}, - "container_timeout_s": 1800, - "profile": "canary", - "task_id": "task-01", - "context_packet_sha256": packet_sha, - "entry_points": ["python/exact_decode.py"], - "verifier_sha256": verifier_sha, - "schema_sha256": schema_sha, - } - - loader = ( - "from google.cloud import storage; import hashlib,subprocess,sys; " - "p='/tmp/phasepoly-bootstrap-r3.py'; " - f"storage.Client(project='agystack-test-deepseek-88').bucket('agystack-test-deepseek-88-swarm-results')" - f".blob('swarms/phasepoly24-canary-20261025/preparation/bootstrap.py').download_to_filename(p); " - f"assert hashlib.sha256(open(p,'rb').read()).hexdigest()=={bootstrap_sha!r}; " - f"subprocess.run([sys.executable,p,{json.dumps(task_config, separators=(',', ':'))!r}],check=True)" - ) - expected_prov = { - "task_id": "task-01", - "session": "phasepoly24-canary-20261025", - "source_sha256": "2e9de7d3064b8a7d3510f57510f1ad9eb4c8faecbed3e3a2c9a474ef7a746444", - "source_base_commit": "d2966d02409a0c7ac749636b97b9e95947c10a9a", - "brief_document_sha256": "21fc20120b31d2269ddc6d137d2a56f8811f6d94ec037a20fcbf0e9afd28a70e", - "context_packet_sha256": packet_sha, - } - brief = ( - "You own the standalone canary verification screen for Phase 3 deployment handoff.\n" - + task_packet.read_text(encoding="utf-8") - + "\nBOOTSTRAP COMMAND\npython3 -c " - + shlex.quote(loader) - + "\nOUTPUT_PREFIX=swarms/phasepoly24-canary-20261025/evidence/task-01\nPROVENANCE=" - + json.dumps(expected_prov, separators=(",", ":")) - + "\n" - ) - verifier_path = "research/task-01/verify_research.py" - verify_argv = ["python3", verifier_path, "--task-dir", "research/task-01", - "--expected-json", json.dumps(expected_prov, separators=(",", ":"))] - guarded_verify = ( - "import hashlib,subprocess; " - f"assert hashlib.sha256(open({verifier_path!r},'rb').read()).hexdigest()=={verifier_sha!r}; " - f"subprocess.run({verify_argv!r},check=True)" - ) - manifest = [{ - "task_index": 0, - "type": "agent", - "model": "publishers/deepseek-ai/models/deepseek-v4.1-flash-maas", - "branch": "phasepoly-r3-canary-task-01", - "brief": brief, - "candidate_files": ["research/task-01"], - "evidence_files": ["research/task-01"], - "execution_limits": {"max_agent_turns": 30, "command_timeout_s": 300, "verify_timeout_s": 120}, - "verify_command": "python3 -c " + shlex.quote(guarded_verify), - }] - (fixture_dir / "canary-manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") - return manifest, fixture_dir - - -def _extract_bootstrap_config(brief: str) -> dict: - bootstrap_cmd = brief.split("BOOTSTRAP COMMAND\n")[1].split("\nOUTPUT_PREFIX=")[0] - code = shlex.split(bootstrap_cmd)[2] - calls = [ - n for n in ast.walk(ast.parse(code)) - if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) and n.func.attr == "run" - ] - return json.loads(ast.literal_eval(calls[0].args[0].elts[2])) - - -def _run_mocked_bootstrap(revision_dir: Path, config: dict, repo_dir: Path, *, missing_blobs: set = None, corrupted_blobs: dict = None): - missing = missing_blobs or set() - corrupted = corrupted_blobs or {} - - spec = importlib.util.spec_from_file_location("canary_bootstrap", revision_dir / "bootstrap.py") - boot = importlib.util.module_from_spec(spec) - spec.loader.exec_module(boot) - - buf_source = io.BytesIO() - with tarfile.open(fileobj=buf_source, mode="w:gz") as tf: - ti1 = tarfile.TarInfo(name="docs/CLOUD_SWARM_RESEARCH_PATHS.md") - c1 = b"Portfolio text\n" - ti1.size = len(c1) - tf.addfile(ti1, io.BytesIO(c1)) - ti2 = tarfile.TarInfo(name="python/exact_decode.py") - c2 = b"def decode(): pass\n" - ti2.size = len(c2) - tf.addfile(ti2, io.BytesIO(c2)) - source_bytes = buf_source.getvalue() - - buf_env = io.BytesIO() - with tarfile.open(fileobj=buf_env, mode="w:gz") as tf: - ti = tarfile.TarInfo(name="phasepoly-runtime/README") - c = b"runtime\n" - ti.size = len(c) - tf.addfile(ti, io.BytesIO(c)) - env_bytes = buf_env.getvalue() - - class MockBlob: - def __init__(self, name: str): - self.name = name - - def download_to_filename(self, filename: str): - dest = Path(filename) - dest.parent.mkdir(parents=True, exist_ok=True) - if any(m in self.name for m in missing): - raise FileNotFoundError(f"Blob {self.name} missing in mock storage") - if any(k in self.name for k in corrupted): - key = next(k for k in corrupted if k in self.name) - dest.write_bytes(corrupted[key]) - return - if self.name.endswith("source.tar.gz"): - dest.write_bytes(source_bytes) - elif self.name.endswith("environment.tar.gz"): - dest.write_bytes(env_bytes) - elif self.name.endswith("verify_research.py"): - dest.write_bytes((revision_dir / "verify_research.py").read_bytes()) - elif self.name.endswith("EVIDENCE_SCHEMA.md"): - dest.write_bytes((revision_dir / "EVIDENCE_SCHEMA.md").read_bytes()) - elif "packets/" in self.name: - packet_name = self.name.split("packets/")[1] - src = revision_dir / "packets" / packet_name - if not src.is_file(): - raise FileNotFoundError(f"Packet not found: {src}") - dest.write_bytes(src.read_bytes()) - else: - raise ValueError(f"Unknown blob {self.name}") - - def download_as_text(self): - if any(m in self.name for m in missing): - raise FileNotFoundError(f"Blob {self.name} missing in mock storage") - if self.name.endswith("environment.json"): - return json.dumps({"environment_sha256": config.get("environment_sha256", "")}) - raise ValueError(f"Unknown blob for text download: {self.name}") - - class MockBucket: - def blob(self, name: str): - return MockBlob(name) - - class MockStorageClient: - def __init__(self, project=None): - pass - - def bucket(self, name: str): - return MockBucket() - - mock_google = types.ModuleType("google") - mock_gcloud = types.ModuleType("google.cloud") - mock_storage = types.ModuleType("google.cloud.storage") - mock_storage.Client = MockStorageClient - mock_gcloud.storage = mock_storage - mock_google.cloud = mock_gcloud - - orig_sha = getattr(boot, "sha", lambda p: hashlib.sha256(Path(p).read_bytes()).hexdigest()) - def patched_sha(path): - p = Path(path) - if p.name == "phasepoly-source.tar.gz": - return config.get("source_sha256", orig_sha(p)) - if p.name == "phasepoly-environment.tar.gz": - return config.get("environment_sha256", orig_sha(p)) - if p.name == "CLOUD_SWARM_RESEARCH_PATHS.md": - return config.get("brief_document_sha256", orig_sha(p)) - return orig_sha(p) - - boot.sha = patched_sha - - orig_run = subprocess.run - def patched_run(cmd, *args, **kwargs): - if isinstance(cmd, list) and cmd == ["make", "check"]: - return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="Baseline make check passed\n") - return orig_run(cmd, *args, **kwargs) - - orig_cwd = os.getcwd() - os.chdir(repo_dir) - try: - with patch.dict(sys.modules, {"google": mock_google, "google.cloud": mock_gcloud, "google.cloud.storage": mock_storage}): - with patch("subprocess.run", side_effect=patched_run): - boot.main(config) - finally: - os.chdir(orig_cwd) - - -def _populate_valid_research_artifacts(repo_dir: Path, config: dict): - task_dir = repo_dir / "research" / config["task_id"] - task_dir.mkdir(parents=True, exist_ok=True) - - (task_dir / "experiment.py").write_text(EXPERIMENT_CANARY_SCRIPT, encoding="utf-8") - (task_dir / "oracle.py").write_text(DOMAIN_ORACLE_CANARY_SCRIPT, encoding="utf-8") - - cmd_sanity = ["python3", "experiment.py", "--size", "1", "--seed", "0", "--output", "sanity.raw.json"] - cmd_screen = ["python3", "experiment.py", "--size", "2", "--seed", "1", "--output", "screen.raw.json"] - - subprocess.run(cmd_sanity, cwd=task_dir, capture_output=True, text=True, check=True) - subprocess.run(cmd_screen, cwd=task_dir, capture_output=True, text=True, check=True) - - commands = [ - {"argv": cmd_sanity, "timeout_s": 10, "output_files": ["sanity.raw.json"]}, - {"argv": cmd_screen, "timeout_s": 10, "output_files": ["screen.raw.json"]}, - ] - - res_oracle = subprocess.run(["python3", "oracle.py"], cwd=task_dir, capture_output=True, text=True, check=True) - checks_data = json.loads(res_oracle.stdout)["checks"] - - contract = { - "task_id": config["task_id"], - "hypothesis": "Canary screen verifies 2-mode Majorana algebra anticommutators without violations.", - "corpus": {"kind": "synthetic", "description": "Jordan-Wigner Majorana representation on 1 and 2 modes.", "evidence_files": ["screen.raw.json"]}, - "sizes": [1, 2], - "seeds": [0, 1], - "controls": ["Jordan-Wigner canonical algebra"], - "metrics": ["violations", "candidate_weight", "baseline_weight"], - "acceptance_rule": "Zero anticommutator violations across all matrix pairs.", - "compute_limits": {"max_size": 2, "max_steps": 10, "max_samples": 3, "wall_time_s": 30, "cpu_time_s": 30}, - "commands": commands, - "sanity_case": {"size": 1, "max_steps": 5, "max_samples": 1, "command": commands[0]}, - } - (task_dir / "contract.json").write_text(json.dumps(contract, indent=2), encoding="utf-8") - - contract_sha = hashlib.sha256((task_dir / "contract.json").read_bytes()).hexdigest() - - runs = [ - { - "kind": "sanity", - "command": commands[0], - "seed": 0, - "size": 1, - "steps": 1, - "samples": 1, - "exit_code": 0, - "duration_s": 0.05, - "output_files": ["sanity.raw.json"], - "metrics": json.loads((task_dir / "sanity.raw.json").read_text(encoding="utf-8")), - }, - { - "kind": "screen", - "command": commands[1], - "seed": 1, - "size": 2, - "steps": 2, - "samples": 2, - "exit_code": 0, - "duration_s": 0.05, - "output_files": ["screen.raw.json"], - "metrics": json.loads((task_dir / "screen.raw.json").read_text(encoding="utf-8")), - }, - ] - for r in runs: - r["metrics"] = {k: r["metrics"][k] for k in contract["metrics"]} - - experiment = { - "contract_sha256": contract_sha, - "sample_counts": {"sanity": 1, "screen": 2}, - "exit_codes": [0, 0], - "runs": runs, - } - (task_dir / "experiment.json").write_text(json.dumps(experiment, indent=2), encoding="utf-8") - - domain_check = { - "oracle_command": ["python3", "oracle.py"], - "timeout_s": 10, - "exit_code": 0, - "evidence_files": ["oracle.py", "screen.raw.json"], - "checks": checks_data, - } - (task_dir / "domain_check.json").write_text(json.dumps(domain_check, indent=2), encoding="utf-8") - - (task_dir / "baseline.log").write_text("Baseline make check passed\n", encoding="utf-8") - (task_dir / "baseline.json").write_text(json.dumps({"exit_code": 0, "command": ["make", "check"]}), encoding="utf-8") - (task_dir / "report.md").write_text("# Canary Report\nMajorana screen verified.\n", encoding="utf-8") - - provenance_keys = ["task_id", "session", "source_sha256", "source_base_commit", "brief_document_sha256", "context_packet_sha256"] - prov = {k: config[k] for k in provenance_keys} - (task_dir / "provenance.json").write_text(json.dumps(prov, indent=2) + "\n", encoding="utf-8") - - report = { - "schema_version": 1, - **prov, - "title": "Canary Majorana Algebra Screen", - "model": "local-canary-tester", - "hypothesis": contract["hypothesis"], - "execution_status": "PASS", - "verdict": "pursue", - "rl_eligibility": "ineligible", - "contract_sha256": contract_sha, - "experiment_commands": commands, - "seeds": [0, 1], - "sample_counts": {"sanity": 1, "screen": 2}, - "target_modified": False, - "baseline_gate": {"status": "pass", "exit_code": 0, "evidence_files": ["baseline.log"]}, - "final_gate": {"status": "not_required", "evidence_files": ["baseline.log"]}, - "four_rl_gates": [ - {"gate_id": i, "status": "unresolved", "reason": "Canary screen does not train RL.", "evidence_files": ["experiment.json"]} - for i in range(1, 5) - ], - "premise_audit": [ - {"statement": "Majorana algebra holds on tiny instances.", "status": "supported", "evidence_files": ["domain_check.json"]} - ], - "findings": [ - {"statement": "Zero anticommutator violations in 16 checks.", "evidence_files": ["screen.raw.json"]} - ], - "witness_files": ["oracle.py", "screen.raw.json"], - "limitations": ["Synthetic small Majorana fixture."], - "closest_prior_work": ["Jordan-Wigner canonical transformation."], - "next_decisive_experiment": { - "question": "Does 3-mode Majorana algebra scale?", - "argv": ["python3", "oracle.py"], - "acceptance_rule": "Zero violations.", - "compute_limits": contract["compute_limits"], - }, - "machine": {"platform": sys.platform, "versions": {"python": sys.version.split()[0]}}, - } - - art_hashes = { - p.name: hashlib.sha256(p.read_bytes()).hexdigest() - for p in task_dir.iterdir() - if p.is_file() and p.name != "report.json" - } - report["artifact_sha256"] = art_hashes - (task_dir / "report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") - return report - - -def test_end_to_end_generated_canary_manifest_execution(tmp_path): - """Test 1: Verify manifest structure of canary-manifest.json.""" - manifest, _ = _load_canary_manifest_and_revision_dir(tmp_path) - assert len(manifest) == 1 - task = manifest[0] - assert task["task_index"] == 0 - assert task["branch"] == "phasepoly-r3-canary-task-01" - assert task["execution_limits"] == { - "max_agent_turns": 30, - "command_timeout_s": 300, - "verify_timeout_s": 120, - } - assert "BOOTSTRAP COMMAND" in task["brief"] - assert "PROVENANCE" in task["brief"] - - verify_cmd = task["verify_command"] - vcode = shlex.split(verify_cmd)[2] - assert "assert hashlib.sha256(open('research/task-01/verify_research.py','rb').read()).hexdigest()==" in vcode - assert "subprocess.run(['python3', 'research/task-01/verify_research.py'" in vcode - assert "check=True" in vcode - compile(vcode, "canary-verifier", "exec") - - -def test_e2e_canary_manifest_pass(tmp_path, monkeypatch): - """Positive End-to-End Case: Complete local execution path from bootstrap to verifier, - - artifact upload, and result harvesting. - """ - manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) - task_item = manifest[0] - - repo = tmp_path / "repo_e2e_pass" - repo.mkdir() - _init_git_repo(repo) - - config = _extract_bootstrap_config(task_item["brief"]) - - _run_mocked_bootstrap(rev_dir, config, repo) - - task_dir = repo / "research" / config["task_id"] - assert task_dir.is_dir() - - verifier_file = task_dir / "verify_research.py" - schema_file = task_dir / "EVIDENCE_SCHEMA.md" - context_file = task_dir / "context.md" - - assert verifier_file.is_file() - assert hashlib.sha256(verifier_file.read_bytes()).hexdigest() == config["verifier_sha256"] - assert schema_file.is_file() - assert hashlib.sha256(schema_file.read_bytes()).hexdigest() == config["schema_sha256"] - assert context_file.is_file() - assert hashlib.sha256(context_file.read_bytes()).hexdigest() == config["context_packet_sha256"] - - context_text = context_file.read_text(encoding="utf-8") - assert ( - "Scientific pursuit (verdict: pursue) requires meeting your declared scientific acceptance rule with verified controls." - in context_text - ) - - _populate_valid_research_artifacts(repo, config) - - subprocess.run(["git", "add", "research/task-01"], cwd=repo, check=True) - - vcode = shlex.split(task_item["verify_command"])[2] - res_verify = subprocess.run([sys.executable, "-c", vcode], cwd=repo, capture_output=True, text=True) - assert res_verify.returncode == 0, f"Verifier failed:\nSTDOUT: {res_verify.stdout}\nSTDERR: {res_verify.stderr}" - - v_data = json.loads(res_verify.stdout) - assert v_data.get("execution_status") == "PASS" - - fake_storage = tmp_path / "fake_storage_pass" - bucket_name = "canary-bucket" - session_id = "phasepoly24-canary-20261025" - gcs_prefix = f"swarms/{session_id}" - monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(fake_storage)) - - expected_raw = task_dir / "screen.raw.json" - expected_raw_bytes = expected_raw.read_bytes() - - storage_uploader.upload_run_artifacts( - bucket_name=bucket_name, - prefix=gcs_prefix, - repo_dir=repo, - task_index=0, - status="PASS", - candidate_files=task_item["candidate_files"], - evidence_files=task_item["evidence_files"], - ) - - harvest_dest = tmp_path / "harvest_pass" - harvested = result_harvester.harvest_candidate_patches( - bucket_name=bucket_name, - prefix=gcs_prefix, - dest_dir=harvest_dest, - ) - assert len(harvested) == 1 - assert harvested[0]["task_index"] == 0 - assert harvested[0]["status"] == "PASS" - - harvested_raw = harvest_dest / "task-0/evidence/research/task-01/screen.raw.json" - assert harvested_raw.is_file() - assert harvested_raw.read_bytes() == expected_raw_bytes - - -def test_e2e_canary_negative_task00_mismatch(tmp_path): - """Negative control: config['task_id'] = 'task-00' reproduces the old task-00 bug and raises AssertionError.""" - manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) - config = _extract_bootstrap_config(manifest[0]["brief"]) - config["task_id"] = "task-00" - - repo = tmp_path / "repo_task00" - repo.mkdir() - _init_git_repo(repo) - - with pytest.raises(AssertionError): - _run_mocked_bootstrap(rev_dir, config, repo) - - -def test_e2e_canary_negative_missing_packet(tmp_path): - """Negative control: missing packets/task-01.md in storage causes bootstrap to fail before setup completes.""" - manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) - config = _extract_bootstrap_config(manifest[0]["brief"]) - - repo = tmp_path / "repo_missing_packet" - repo.mkdir() - _init_git_repo(repo) - - with pytest.raises((FileNotFoundError, Exception)): - _run_mocked_bootstrap(rev_dir, config, repo, missing_blobs={"packets/task-01.md"}) - - task_dir = repo / "research" / config["task_id"] - assert not (task_dir / "context.md").exists() - - -def test_e2e_canary_negative_tampered_packet(tmp_path): - """Negative control: altered packet bytes in storage causes sha mismatch and raises AssertionError.""" - manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) - config = _extract_bootstrap_config(manifest[0]["brief"]) - - repo = tmp_path / "repo_tampered_packet" - repo.mkdir() - _init_git_repo(repo) - - with pytest.raises(AssertionError): - _run_mocked_bootstrap( - rev_dir, - config, - repo, - corrupted_blobs={"packets/task-01.md": b"# Tampered packet bytes\n"}, - ) - - -def test_e2e_canary_negative_wrong_scientific_result(tmp_path): - """Negative control: oracle fails or outputs passed: False, verify_command exits non-zero.""" - manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) - task_item = manifest[0] - config = _extract_bootstrap_config(task_item["brief"]) - - repo = tmp_path / "repo_wrong_sci" - repo.mkdir() - _init_git_repo(repo) - - _run_mocked_bootstrap(rev_dir, config, repo) - report = _populate_valid_research_artifacts(repo, config) - - task_dir = repo / "research" / config["task_id"] - (task_dir / "oracle.py").write_text("import sys\nprint('Domain oracle failure', file=sys.stderr)\nsys.exit(1)\n", encoding="utf-8") - report["artifact_sha256"]["oracle.py"] = hashlib.sha256((task_dir / "oracle.py").read_bytes()).hexdigest() - (task_dir / "report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") - - vcode = shlex.split(task_item["verify_command"])[2] - result = subprocess.run([sys.executable, "-c", vcode], cwd=repo, capture_output=True, text=True) - assert result.returncode != 0 - - -def test_e2e_canary_negative_missing_baseline_gate(tmp_path): - """Negative control: baseline_gate['exit_code'] = 1 causes verify_command to exit non-zero.""" - manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) - task_item = manifest[0] - config = _extract_bootstrap_config(task_item["brief"]) - - repo = tmp_path / "repo_bad_baseline" - repo.mkdir() - _init_git_repo(repo) - - _run_mocked_bootstrap(rev_dir, config, repo) - report = _populate_valid_research_artifacts(repo, config) - - task_dir = repo / "research" / config["task_id"] - report["baseline_gate"]["exit_code"] = 1 - (task_dir / "report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") - - vcode = shlex.split(task_item["verify_command"])[2] - result = subprocess.run([sys.executable, "-c", vcode], cwd=repo, capture_output=True, text=True) - assert result.returncode != 0 - - -def test_e2e_canary_negative_failed_final_gate_after_edits(tmp_path): - """Negative control: target_modified = True and final_gate['exit_code'] = 1 causes verify_command to exit non-zero.""" - manifest, rev_dir = _load_canary_manifest_and_revision_dir(tmp_path) - task_item = manifest[0] - config = _extract_bootstrap_config(task_item["brief"]) - - repo = tmp_path / "repo_bad_final" - repo.mkdir() - _init_git_repo(repo) - - _run_mocked_bootstrap(rev_dir, config, repo) - report = _populate_valid_research_artifacts(repo, config) - - task_dir = repo / "research" / config["task_id"] - report["target_modified"] = True - report["final_gate"]["exit_code"] = 1 - report["final_gate"]["status"] = "fail" - (task_dir / "report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") - - vcode = shlex.split(task_item["verify_command"])[2] - result = subprocess.run([sys.executable, "-c", vcode], cwd=repo, capture_output=True, text=True) - assert result.returncode != 0 - diff --git a/tests/test_storage_uploader.py b/tests/test_storage_uploader.py index 6b596eb..382c445 100644 --- a/tests/test_storage_uploader.py +++ b/tests/test_storage_uploader.py @@ -8,6 +8,7 @@ from unittest.mock import MagicMock, patch import sys + sys.path.insert(0, str(Path(__file__).parent.parent / "skills" / "swarm" / "scripts")) import storage_uploader @@ -99,7 +100,9 @@ def test_upload_run_artifacts_generates_patch(self, mock_gen_patch, mock_upload) self.assertIn("patch.diff", results) self.assertTrue((self.repo_dir / "patch.diff").is_file()) - self.assertEqual((self.repo_dir / "patch.diff").read_text(encoding="utf-8"), "diff --git a/test b/test\n+new line") + self.assertEqual( + (self.repo_dir / "patch.diff").read_text(encoding="utf-8"), "diff --git a/test b/test\n+new line" + ) class TestGCSUploadFallback(unittest.TestCase): @@ -204,7 +207,9 @@ def setUp(self): self.repo_dir = Path(self.temp_dir.name) subprocess.run(["git", "init"], cwd=self.repo_dir, capture_output=True, check=True) subprocess.run(["git", "config", "user.name", "Tester"], cwd=self.repo_dir, capture_output=True, check=True) - subprocess.run(["git", "config", "user.email", "test@test.local"], cwd=self.repo_dir, capture_output=True, check=True) + subprocess.run( + ["git", "config", "user.email", "test@test.local"], cwd=self.repo_dir, capture_output=True, check=True + ) (self.repo_dir / "app.py").write_text("print('hello')\n", encoding="utf-8") subprocess.run(["git", "add", "app.py"], cwd=self.repo_dir, capture_output=True, check=True) subprocess.run(["git", "commit", "-m", "init"], cwd=self.repo_dir, capture_output=True, check=True) @@ -312,6 +317,76 @@ def test_sensitive_filenames_rejected(self, mock_upload): status_data = json.loads(status_file.read_text(encoding="utf-8")) self.assertEqual(status_data["status"], "ISSUES") + @patch("storage_uploader.upload_to_gcs", return_value=True) + def test_sensitive_directory_symlink_rejected_and_degrades_pass(self, mock_upload): + external = self.repo_dir.parent / "external_sensitive" + external.mkdir(exist_ok=True) + (external / "leak.log").write_text("secret") + + for location in ["evidence", "diagnostics"]: + with self.subTest(location=location): + status_file = self.repo_dir / "status.json" + if status_file.exists(): + status_file.unlink() + + evidence_files = [] + if location == "evidence": + ev_dir = self.repo_dir / "ev_tree" + ev_dir.mkdir(exist_ok=True) + (ev_dir / "safe.txt").write_text("admitted bytes") + (ev_dir / "sym_dir").symlink_to(external) + evidence_files = ["ev_tree"] + else: + diag_dir = self.repo_dir / ".agystack/diagnostics" + diag_dir.parent.mkdir(parents=True, exist_ok=True) + diag_dir.symlink_to(external) + + artifacts = storage_uploader.upload_run_artifacts( + "bucket", + "run", + self.repo_dir, + 0, + status="PASS", + evidence_files=evidence_files, + ) + self.assertNotIn("evidence/ev_tree/sym_dir/leak.log", artifacts) + self.assertNotIn("evidence/.agystack/diagnostics/leak.log", artifacts) + status_data = json.loads(status_file.read_text(encoding="utf-8")) + self.assertEqual(status_data["status"], "ISSUES") + self.assertTrue(status_data.get("delivery_errors")) + self.assertEqual((external / "leak.log").read_text(), "secret") + if location == "evidence": + self.assertEqual((self.repo_dir / "ev_tree/safe.txt").read_text(), "admitted bytes") + self.assertIn("evidence/ev_tree/safe.txt", artifacts) + (self.repo_dir / "ev_tree/sym_dir").unlink() + (self.repo_dir / "ev_tree/safe.txt").unlink() + (self.repo_dir / "ev_tree").rmdir() + else: + (self.repo_dir / ".agystack/diagnostics").unlink() + + @patch("storage_uploader.upload_to_gcs", return_value=True) + @patch("storage_uploader._generate_git_patch", return_value="replacement patch") + def test_patch_symlink_rejected_before_regeneration(self, mock_generate, mock_upload): + target = self.repo_dir / "untouched.txt" + target.write_bytes(b"original bytes") + for status, name in [("PASS", "patch.diff"), ("ISSUES", "partial.patch.diff")]: + with self.subTest(status=status): + sym_path = self.repo_dir / name + sym_path.symlink_to(target) + artifacts = storage_uploader.upload_run_artifacts( + "bucket", + "run", + self.repo_dir, + 0, + status, + ["app.py"], + ) + self.assertEqual(target.read_bytes(), b"original bytes") + self.assertNotIn(name, artifacts) + self.assertEqual(json.loads((self.repo_dir / "status.json").read_text())["status"], "ISSUES") + sym_path.unlink() + mock_generate.assert_not_called() + @patch("storage_uploader.upload_to_gcs") def test_failed_score_or_results_upload_degrades_pass_to_issues(self, mock_upload): score_file = self.repo_dir / "score.json" @@ -364,6 +439,5 @@ def fail_results_upload(bkt, dest, path): self.assertIn("results.tsv upload failed", status_data.get("evidence", "")) - if __name__ == "__main__": unittest.main() diff --git a/tests/test_worker_execution_lifecycle.py b/tests/test_worker_execution_lifecycle.py index 6a5cf81..e2770de 100644 --- a/tests/test_worker_execution_lifecycle.py +++ b/tests/test_worker_execution_lifecycle.py @@ -18,6 +18,28 @@ import result_harvester import storage_uploader from worker import partner_engine +from worker.execution import ExecutionLimits, resolve_execution_limits + + +def mock_storage(blobs=()): + storage = MagicMock() + storage.Client.return_value.bucket.return_value.list_blobs.side_effect = lambda prefix="": [ + Blob(k, v) for k, v in (blobs.items() if isinstance(blobs, dict) else blobs) if k.startswith(prefix) + ] + g, gc = types.ModuleType("google"), types.ModuleType("google.cloud") + g.cloud, gc.storage = gc, storage + return storage, {"google": g, "google.cloud": gc, "google.cloud.storage": storage} + + +class Blob: + def __init__(self, name, payload): + self.name = name + self.payload = payload + + def download_to_filename(self, filename): + path = Path(filename) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(self.payload, encoding="utf-8") @pytest.mark.parametrize( @@ -49,7 +71,6 @@ def test_strict_terminal_status(text, expected): @pytest.mark.parametrize("bad", [0, -1, True, 1.5, "1.5", "oops", None]) def test_invalid_execution_limits(bad): - from worker.execution import resolve_execution_limits with pytest.raises(ValueError): resolve_execution_limits({"max_agent_turns": bad}, environ={}) @@ -57,31 +78,25 @@ def test_invalid_execution_limits(bad): def test_limits_precedence_and_immutability(): from dataclasses import FrozenInstanceError - from worker.execution import resolve_execution_limits limits = resolve_execution_limits( {"verify_timeout_s": 41}, - runtime={"max_agent_turns": 31, "command_timeout_s": 500}, environ={"AGYSTACK_MAX_AGENT_TURNS": "32", "COMMAND_TIMEOUT": "600"}, - cli={"max_agent_turns": 33}, ) - assert (limits.max_agent_turns, limits.command_timeout_s, limits.verify_timeout_s) == (33, 600, 41) + assert (limits.max_agent_turns, limits.command_timeout_s, limits.verify_timeout_s) == (32, 600, 41) with pytest.raises(FrozenInstanceError): limits.max_agent_turns = 2 with pytest.raises(ValueError): resolve_execution_limits({"unexpected": 5}, environ={}) with pytest.raises(ValueError): resolve_execution_limits([], environ={}) - with pytest.raises(ValueError): - resolve_execution_limits(runtime=None, environ={}) assert resolve_execution_limits(environ={}).command_timeout_s == 300 assert resolve_execution_limits(environ={}, command_default=900).command_timeout_s == 900 @pytest.mark.parametrize("provider", ["claude", "fable", "deepseek"]) @pytest.mark.parametrize("finish", [True, False]) -@pytest.mark.parametrize("source", ["task", "keyword"]) -def test_provider_turn_limits_and_final_response(tmp_path, provider, finish, source): +def test_provider_turn_limits_and_final_response(tmp_path, provider, finish): (tmp_path / "note.txt").write_text("still working") tool = {"type": "tool_use", "id": "read", "name": "read_file", "input": {"path": "note.txt"}} responses = [] @@ -108,20 +123,7 @@ def test_provider_turn_limits_and_final_response(tmp_path, provider, finish, sou getter = "_get_genai" request.side_effect = responses with patch.object(partner_engine, getter, return_value=sdk): - if source == "task": - status, summary = entry(provider, "work", tmp_path, task_item={"execution_limits": {"max_agent_turns": 33}}) - else: - from worker.execution import ExecutionLimits - - model = {"claude": "claude-3-opus@20240229", "fable": "fable-5-1", "deepseek": "deepseek"}[provider] - status, summary = cloud_worker.execute_task( - {}, - "work", - tmp_path, - model, - "", - limits=ExecutionLimits(max_agent_turns=33), - ) + status, summary = entry(provider, "work", tmp_path, task_item={"execution_limits": {"max_agent_turns": 33}}) assert request.call_count == 33 assert status == ("PASS" if finish else "ISSUES") if not finish: @@ -133,8 +135,6 @@ def test_provider_turn_limits_and_final_response(tmp_path, provider, finish, sou @pytest.mark.parametrize("suffix", ["& wait", "| cat", "&"]) def test_timeout_keeps_output_and_kills_descendants(tmp_path, suffix): - from worker.execution import ExecutionLimits - child = "import os,time; open('child.pid','w').write(str(os.getpid())); time.sleep(2); open('survived', 'w').write('bad')" command = ( f"echo partial-output; echo partial-error >&2; {shlex.quote(sys.executable)} -c {shlex.quote(child)} {suffix}" @@ -175,22 +175,26 @@ def worker_repo(tmp_path): @pytest.mark.parametrize( - "status, candidate, verify", + "status, candidate, verify, fail_status", [ - ("BLOCKED", True, "exit 1"), - ("ISSUES", True, "exit 1"), - ("PASS", False, "exit 1"), - ("PASS", False, "exit 0"), - ("PASS", True, "exit 1"), - ("PASS", False, "sleep 2"), + ("BLOCKED", True, "exit 1", False), + ("ISSUES", True, "exit 1", False), + ("PASS", False, "exit 1", False), + ("PASS", False, "exit 0", False), + ("PASS", True, "exit 0", False), + ("PASS", True, "exit 1", False), + ("PASS", False, "sleep 2", False), + ("PASS", True, "exit 0", True), ], ) -def test_worker_finalization_preserves_evidence(worker_repo, tmp_path, monkeypatch, status, candidate, verify, capsys): +def test_worker_finalization_preserves_evidence( + worker_repo, tmp_path, monkeypatch, status, candidate, verify, fail_status, capsys +): storage = tmp_path / "storage" task = { "brief": "work", "candidate_files": ["service.py"], - "evidence_files": ["results/proof.txt"], + "evidence_files": ["results/proof.txt", "nested/domain/oracle/checks.json"], "verify_command": f"echo verifier-ran > verification.txt; {verify}", "execution_limits": {"verify_timeout_s": 1}, } @@ -203,6 +207,8 @@ def test_worker_finalization_preserves_evidence(worker_repo, tmp_path, monkeypat monkeypatch.chdir(tmp_path) monkeypatch.setattr(cloud_worker.tempfile, "gettempdir", lambda: str(tmp_path)) + raw_payload = json.dumps({"status": "verified", "checks": [1, 2, 3], "score": 0.98}, indent=2).encode("utf-8") + def clone(**kwargs): repo = kwargs["repo_dir"] subprocess.run(["git", "init"], cwd=repo, check=True, capture_output=True) @@ -218,33 +224,64 @@ def execute(**kwargs): (repo / "service.py").write_text("value = 2\n") (repo / "results").mkdir() (repo / "results/proof.txt").write_text("partial proof") + deep = repo / "nested/domain/oracle/checks.json" + deep.parent.mkdir(parents=True, exist_ok=True) + deep.write_bytes(raw_payload) return status, "work summary" monkeypatch.setattr(cloud_worker, "clone_and_checkout_task", clone) monkeypatch.setattr(cloud_worker, "execute_task", execute) monkeypatch.setattr(cloud_worker, "StorageMessenger", None) + + real_upload = storage_uploader.upload_to_gcs + + def upload_hook(bucket_name, object_name, local_file_path): + if fail_status and object_name.endswith("status.json"): + return False + return real_upload(bucket_name, object_name, local_file_path) + + monkeypatch.setattr(storage_uploader, "upload_to_gcs", upload_hook) + + exit_code = 0 try: cloud_worker.main() except SystemExit as exc: - assert exc.code == 1 - target = storage / "test-bucket/run/task-0" - published = json.loads((target / "status.json").read_text()) - expected = "PASS" if status == "PASS" and verify == "exit 0" else "ISSUES" if status == "PASS" else status - assert published["status"] == expected + exit_code = exc.code + expected = ( + "ISSUES" + if fail_status + else ("PASS" if status == "PASS" and verify == "exit 0" else "ISSUES" if status == "PASS" else status) + ) + assert exit_code == (0 if expected == "PASS" else 1) report = cloud_dispatch.parse_worker_logs(capsys.readouterr().out) assert len(report) == 1 and report[0]["status"] == expected and report[0]["task_index"] == 0 + target = storage / "test-bucket/run/task-0" assert (target / "evidence/results/proof.txt").read_text() == "partial proof" - assert not (target / "patch.diff").exists() - if candidate: - assert "+value = 2" in (target / "partial.patch.diff").read_text() + assert (target / "evidence/nested/domain/oracle/checks.json").read_bytes() == raw_payload + if fail_status: + assert not (target / "status.json").exists() + return + + published = json.loads((target / "status.json").read_text()) + assert published["status"] == expected + if expected == "PASS" and candidate: + assert (target / "patch.diff").is_file() + else: + assert not (target / "patch.diff").exists() + if candidate: + assert "+value = 2" in (target / "partial.patch.diff").read_text() if status == "PASS": assert (worker_repo / "verification.txt").is_file() assert list((target / "evidence/.agystack/diagnostics").glob("verification-*.log")) harvested = result_harvester.harvest_candidate_patches("test-bucket", "run", tmp_path / "harvest") assert len(harvested) == 1 assert harvested[0]["status"] == published["status"] - assert not harvested[0]["patch_file"] assert (tmp_path / "harvest/task-0/evidence/results/proof.txt").is_file() + assert (tmp_path / "harvest/task-0/evidence/nested/domain/oracle/checks.json").read_bytes() == raw_payload + if expected == "PASS" and candidate: + assert harvested[0]["patch_file"] and Path(harvested[0]["patch_file"]).is_file() + else: + assert not harvested[0]["patch_file"] @pytest.mark.parametrize( @@ -267,8 +304,18 @@ def test_dispatch_complete_pass(): ) -@pytest.mark.parametrize("limits", [None, [], {"extra": 3}, {"max_agent_turns": "40"}, {"max_agent_turns": True}]) -def test_dispatch_rejects_invalid_task_limits_before_launch(tmp_path, limits): +@pytest.mark.parametrize( + "limits, bad_env", + [ + (None, None), + ([], None), + ({"extra": 3}, None), + ({"max_agent_turns": "40"}, None), + ({"max_agent_turns": True}, None), + ({}, {"AGYSTACK_MAX_AGENT_TURNS": "oops"}), + ], +) +def test_dispatch_rejects_invalid_task_limits_before_launch(tmp_path, limits, bad_env): command = [ sys.executable, str(SCRIPTS / "cloud_dispatch.py"), @@ -282,15 +329,14 @@ def test_dispatch_rejects_invalid_task_limits_before_launch(tmp_path, limits): json.dumps([{"brief": "work", "execution_limits": limits}]), ] env = dict(os.environ, GH_TOKEN="fake-test-token") + if bad_env: + env.update(bad_env) result = subprocess.run(command, cwd=tmp_path, env=env, capture_output=True, text=True) assert result.returncode != 0 assert "execution_limits" in result.stderr or "positive integer" in result.stderr -def test_dispatch_cli_env_runtime_precedence(tmp_path): - (tmp_path / "agystack-runtime.json").write_text( - json.dumps({"execution_limits": {"max_agent_turns": 40, "command_timeout_s": 500, "verify_timeout_s": 501}}) - ) +def test_dispatch_env_forwarding(tmp_path): env = dict(os.environ, AGYSTACK_MAX_AGENT_TURNS="41", COMMAND_TIMEOUT="600", GH_TOKEN="fake-test-token") env.pop("VERIFY_TIMEOUT", None) result = subprocess.run( @@ -303,10 +349,6 @@ def test_dispatch_cli_env_runtime_precedence(tmp_path): "--vertex", "--project", "test", - "--max-agent-turns", - "42", - "--verify-timeout", - "601", "--manifest", json.dumps([{"brief": "work", "execution_limits": {"max_agent_turns": 43}}]), ], @@ -317,9 +359,9 @@ def test_dispatch_cli_env_runtime_precedence(tmp_path): check=True, ) payload = json.loads(result.stdout) - assert payload["env_vars"]["AGYSTACK_MAX_AGENT_TURNS"] == "42" + assert payload["env_vars"]["AGYSTACK_MAX_AGENT_TURNS"] == "41" assert payload["env_vars"]["COMMAND_TIMEOUT"] == "600" - assert payload["env_vars"]["VERIFY_TIMEOUT"] == "601" + assert "VERIFY_TIMEOUT" not in payload["env_vars"] def test_deepseek_retries_consume_request_limit(tmp_path): @@ -356,14 +398,21 @@ def test_empty_final_response_cannot_reuse_previous_pass(tmp_path, provider): assert entry(provider, "work", tmp_path)[0] == "ISSUES" -def test_evidence_failure_downgrades_before_status_publication(worker_repo): +def test_evidence_failure_downgrades_before_status_publication(worker_repo, tmp_path, monkeypatch): (worker_repo / "service.py").write_text("value = 2\n") (worker_repo / "proof.txt").write_text("proof") + storage = tmp_path / "storage" + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(storage)) published = [] def upload(bucket, name, path): published.append((name, path.read_text())) - return not name.endswith("proof.txt") + if name.endswith("proof.txt"): + return False + dest = storage / bucket / name + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(path.read_text(), encoding="utf-8") + return True with patch.object(storage_uploader, "upload_to_gcs", side_effect=upload): artifacts = storage_uploader.upload_run_artifacts( @@ -379,6 +428,11 @@ def upload(bucket, name, path): assert json.loads(published[-1][1])["status"] == "ISSUES" assert "patch.diff" not in artifacts assert "partial.patch.diff" in artifacts + harvested = result_harvester.harvest_candidate_patches("bucket", "run", tmp_path / "harvest") + assert len(harvested) == 1 + assert harvested[0]["status"] == "ISSUES" + assert harvested[0]["patch_file"] is None + assert Path(harvested[0]["partial_patch_file"]).is_file() def test_missing_status_file_never_promotes_score_or_patch(tmp_path, monkeypatch): @@ -439,30 +493,9 @@ def test_sdk_harvest_downloads_nested_partial_evidence(tmp_path, monkeypatch): "run/task-0/evidence/.agystack/diagnostics/timeout.log": "partial output", } - class Blob: - def __init__(self, name, payload): - self.name, self.payload = name, payload - - def download_to_filename(self, filename): - Path(filename).write_text(self.payload) - - storage = MagicMock() - storage.Client.return_value.bucket.return_value.list_blobs.return_value = [ - Blob(name, value) for name, value in payloads.items() - ] monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) - google_mod = types.ModuleType("google") - cloud_mod = types.ModuleType("google.cloud") - google_mod.cloud = cloud_mod - cloud_mod.storage = storage - with patch.dict( - sys.modules, - { - "google": google_mod, - "google.cloud": cloud_mod, - "google.cloud.storage": storage, - }, - ): + _, modules = mock_storage(payloads) + with patch.dict(sys.modules, modules): results = result_harvester.harvest_candidate_patches("bucket", "run", tmp_path / "harvest") assert results[0]["status"] == "BLOCKED" assert results[0]["patch_file"] is None @@ -496,70 +529,38 @@ def test_standalone_harvest_quarantines_incomplete_coverage(tmp_path, monkeypatc assert "Winning Candidate" not in result.stdout -def test_storage_mock_hierarchy_without_google_module(tmp_path, monkeypatch): - storage = MagicMock() - google_mod = types.ModuleType("google") - cloud_mod = types.ModuleType("google.cloud") - google_mod.cloud = cloud_mod - cloud_mod.storage = storage - - monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) - clean_modules = {k: v for k, v in sys.modules.items() if not k.startswith("google")} - clean_modules["google"] = google_mod - clean_modules["google.cloud"] = cloud_mod - clean_modules["google.cloud.storage"] = storage - - with patch.dict(sys.modules, clean_modules, clear=True): - from google.cloud import storage as resolved_storage - - assert resolved_storage is storage - storage.Client.return_value.bucket.return_value.list_blobs.return_value = [] - res = result_harvester.harvest_candidate_patches("bkt", "prefix", tmp_path / "dest") - assert res == [] - - def test_harvest_candidate_patches_strict_prefix_matching(tmp_path, monkeypatch): + local_storage = tmp_path / "local_storage" + monkeypatch.setenv("STORAGE_MESSENGER_LOCAL_DIR", str(local_storage)) + for sub, idx, score, p_text in [ + ("canary/task-0", 0, 0.80, "patch canary"), + ("canary-other/task-1", 1, 0.99, "patch sibling"), + ]: + td = local_storage / "bucket/swarms" / sub + td.mkdir(parents=True) + (td / "status.json").write_text(json.dumps({"task_index": idx, "status": "PASS", "score": score})) + (td / "patch.diff").write_text(p_text) + + local_harvest = result_harvester.harvest_candidate_patches("bucket", "swarms/canary", tmp_path / "local_harvest") + assert len(local_harvest) == 1 + assert (local_harvest[0]["task_index"], local_harvest[0]["score"]) == (0, 0.80) + assert Path(local_harvest[0]["patch_file"]).read_text() == "patch canary" + assert cloud_dispatch.failed_task_count(local_harvest, 2) == 1 + payloads = { "run/task-0/status.json": json.dumps({"status": "PASS"}), "run/task-0/score.json": json.dumps({"score": 0.95}), "run/task-0/patch.diff": "valid patch", - # Sibling prefix that starts with 'run' but is not 'run/' "run-other/task-1/status.json": json.dumps({"status": "PASS"}), "run-other/task-1/score.json": json.dumps({"score": 0.99}), "run-other/task-1/patch.diff": "invalid patch", - # Invalid task naming inside 'run/' "run/mytask-2/status.json": json.dumps({"status": "PASS"}), "run/task-3-suffix/status.json": json.dumps({"status": "PASS"}), } - class Blob: - def __init__(self, name, payload): - self.name, self.payload = name, payload - - def download_to_filename(self, filename): - Path(filename).write_text(self.payload) - - storage = MagicMock() - - def fake_list_blobs(prefix=""): - return [Blob(name, val) for name, val in payloads.items() if name.startswith(prefix)] - - storage.Client.return_value.bucket.return_value.list_blobs.side_effect = fake_list_blobs - - google_mod = types.ModuleType("google") - cloud_mod = types.ModuleType("google.cloud") - google_mod.cloud = cloud_mod - cloud_mod.storage = storage - monkeypatch.delenv("STORAGE_MESSENGER_LOCAL_DIR", raising=False) - with patch.dict( - sys.modules, - { - "google": google_mod, - "google.cloud": cloud_mod, - "google.cloud.storage": storage, - }, - ): + _, modules = mock_storage(payloads) + with patch.dict(sys.modules, modules): results = result_harvester.harvest_candidate_patches("bucket", "run", tmp_path / "harvest") assert len(results) == 1