diff --git a/.github/workflows/fatal-provider-dependency-review-quality-ci.yml b/.github/workflows/fatal-provider-dependency-review-quality-ci.yml new file mode 100644 index 000000000..0579dcc2e --- /dev/null +++ b/.github/workflows/fatal-provider-dependency-review-quality-ci.yml @@ -0,0 +1,62 @@ +name: Fatal Provider and Dependency Review Quality CI + +on: + pull_request: + branches: [main] + paths: + - ".github/workflows/fatal-provider-dependency-review-quality-ci.yml" + - "scripts/ci/run_opencode_review_model_pool.sh" + - "tests/test_fatal_provider_and_dependency_review_replacement.py" + - "docs/doctoring/opencode-process-group-termination.md" + - "CHANGELOG.md" + +permissions: + contents: read + +concurrency: + group: fatal-provider-dependency-review-quality-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + exact-head-contract: + if: github.event_name != 'pull_request' || github.event.action != 'closed' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout exact source revision + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install exact hash-verified test runner dependencies + env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PIP_NO_INPUT: "1" + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + cat >"${RUNNER_TEMP}/fatal-provider-quality-requirements.txt" <<'EOF' + iniconfig==2.1.0 --hash=sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 + packaging==26.2 --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e + pluggy==1.6.0 --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + pygments==2.20.0 --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + pytest==9.1.1 --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c + EOF + python -m pip install \ + --only-binary=:all: \ + --require-hashes \ + -r "${RUNNER_TEMP}/fatal-provider-quality-requirements.txt" + + - name: Verify fail-first and exact-head contracts + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + test "$(git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha || github.sha }}" + python -m pytest -q tests/test_fatal_provider_and_dependency_review_replacement.py + python -m compileall -q tests/test_fatal_provider_and_dependency_review_replacement.py + bash -n scripts/ci/run_opencode_review_model_pool.sh + git diff --exit-code diff --git a/AGENTS.md b/AGENTS.md index 16f0981c0..76744c844 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,3 +5,4 @@ Materialize accepts only exact SHA-256 pins or a bounded relative `-r` include (no `.`/`..`); a lone `--require-hashes` directive is not trust evidence. See [`docs/doctoring/hourly-nvidia-nim-autofix.md`](docs/doctoring/hourly-nvidia-nim-autofix.md). Conflict-scope roots fail closed when the immediate parent directory is a symbolic link. +Fatal OpenCode provider cleanup must signal the `setsid` process group, not only the timeout wrapper. See [`docs/doctoring/opencode-process-group-termination.md`](docs/doctoring/opencode-process-group-termination.md). diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6fe6621b6..a4ae5ea2c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -25,6 +25,12 @@ flowchart LR Products -->|"standalone or as module"| Buyer ``` +## Fatal-provider process groups + +Each `opencode run` attempt starts under `setsid`. A structured fatal +provider event sends `SIGTERM` then `SIGKILL` to the negative process-group +id so descendants cannot hold stdout/stderr after the launcher returns. + ## Hourly NVIDIA NIM repair gate ```mermaid diff --git a/CHANGELOG.md b/CHANGELOG.md index 1de9130a5..17e34d3b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Launch each OpenCode model attempt with `setsid` and signal the negative process-group id on a fatal provider event so descendant `opencode` processes cannot keep coverage pipes open after the launcher exits. - Materialized base Python locks only when every package line is an exact SHA-256 pin or a bounded relative `-r`/`--requirement` include. A lone `--require-hashes` directive, a dotted include such as `./lock.txt`, or `-r other-hashes.txt` no longer enters the trusted build context. - Refused a conflict-scope repository root whose immediate parent is a symbolic link, so a swapped parent cannot redirect the canonical worktree after the last-component check (CWE-367). - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. diff --git a/CLAUDE.md b/CLAUDE.md index 7127d3c1c..b8d38771c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -114,6 +114,8 @@ repeatable compile command. without running the test suite will break CI. - **100% coverage and 100% docstrings on `scripts/ci/`** are hard gates, not aspirations. New helper code needs matching tests and docstrings. +- **Fatal-provider cleanup** uses `setsid` and the negative process-group id. Do not restore + PID-only `kill` of the timeout wrapper. - **`pull_request_target` trust boundary.** The required review workflows run the *base branch's* trusted scripts. A PR that edits the trusted review workflows can fail its own checks until the base branch catches up; a same-head manual `workflow_dispatch` Strix run may supply review evidence diff --git a/docs/doctoring/opencode-process-group-termination.md b/docs/doctoring/opencode-process-group-termination.md new file mode 100644 index 000000000..06aae0f21 --- /dev/null +++ b/docs/doctoring/opencode-process-group-termination.md @@ -0,0 +1,51 @@ +# OpenCode fatal-provider process-group termination + +## Incident + +The exact-head coverage-evidence job for `.github` pull request #799 reached the repository test suite but did not complete inside its bounded measurement step. A focused reproduction identified `test_fatal_provider_error_kills_hung_opencode_run_early`: the launcher detected a fatal provider event and terminated the `timeout` wrapper, while a descendant fake `opencode` process could remain alive with inherited output pipes. The parent Python process then waited for end-of-file even though the launcher had returned. + +## Decision + +Materialize accepts only exact SHA-256 pins or a bounded relative `-r` include; a lone `--require-hashes` line is not lock evidence. + +Each bounded `opencode run` starts in a new session with `setsid` when that +tool is present (Linux CI / util-linux). On a structured fatal-provider +event, the launcher sends `SIGTERM` to the negative process-group +identifier, waits for bounded group disappearance, and then sends +`SIGKILL` to the same group if necessary. Darwin local runners without +`setsid` keep PID-directed `TERM`/`KILL` so the repository suite still +executes. The ordinary timeout contract remains +`timeout --kill-after=30s`; only the early-fatal cleanup boundary +changes. + +The group signal is deliberately scoped to the session created for one model attempt. It does not target the workflow shell, unrelated model attempts, or the runner process. The production Ubuntu image already installs `util-linux`, which supplies `setsid`. + +CWE-400 describes uncontrolled resource consumption when a child outlives the +intended bound (MITRE, 2026). NIST SP 800-53 Rev. 5 SI-4 requires monitoring +that detects and contains anomalous process behavior rather than treating a +returned parent as a complete cleanup (Joint Task Force, 2020). Killing only +the `timeout` wrapper therefore leaves a descendant that can stall coverage +evidence; the negative process-group identifier is the contained unit. + +## Verification + +The existing behavioral regression uses a fake provider that emits a fatal structured event and sleeps for 120 seconds. Before the change, the test exceeded its 30-second subprocess boundary because a descendant retained the capture pipes. With process-group termination, it completes in under 25 seconds and the complete model-pool test file remains eligible for the exact-head coverage job. Shell syntax validation and the repository-wide evidence command remain required before merge. + +## Rollback + +Rollback requires an independently reviewed change and a replacement mechanism that proves every descendant of a fatal model attempt is reaped without terminating unrelated runner work. Restoring PID-only termination is not acceptable because it reintroduces the pipe-retention failure mode. + +## APA 7th references + +IEEE & The Open Group. (2024). *The Open Group base specifications issue 8: System interfaces, `kill()`*. https://pubs.opengroup.org/onlinepubs/9799919799/functions/kill.html + +Joint Task Force. (2020). *Security and privacy controls for information systems +and organizations* (NIST SP 800-53 Rev. 5). National Institute of Standards and +Technology. https://doi.org/10.6028/NIST.SP.800-53r5 + +MITRE. (2026). *CWE-400: Uncontrolled resource consumption*. +https://cwe.mitre.org/data/definitions/400.html + +Free Software Foundation. (n.d.). *GNU Coreutils manual: `timeout`: Run a command with a time limit*. Retrieved August 7, 2026, from https://www.gnu.org/software/coreutils/manual/html_node/timeout-invocation.html + +Linux man-pages project. (2026, February 8). *setsid(2) — Linux manual page* (Linux man-pages 6.18). https://man7.org/linux/man-pages/man2/setsid.2.html diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh old mode 100644 new mode 100755 index 986982e9a..188493024 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -465,16 +465,34 @@ run_one_model_attempt() { rm -f "$opencode_json_file" "$opencode_stderr_file" "$opencode_export_file" "$candidate_output_file" set +e - timeout --kill-after=30s "${run_timeout_seconds}s" \ - env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ - -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ - opencode run "$(cat "$prompt_file")" \ - --pure \ - --agent "$agent" \ - --model "$model_candidate" \ - --format json \ - --title "PR #${PR_NUMBER} OpenCode bounded review ${model_candidate} attempt ${attempt}/${attempts}" \ - >"$opencode_json_file" 2>"$opencode_stderr_file" & + # Linux CI has setsid (util-linux). Start the timeout wrapper in its own + # session so a fatal-provider abort can terminate the complete provider + # process group. Killing only the timeout wrapper leaves descendants + # holding stdout/stderr pipes open. Darwin local runners lack setsid; + # they keep PID-directed TERM/KILL so the repository suite still runs. + if command -v setsid >/dev/null 2>&1; then + setsid timeout --kill-after=30s "${run_timeout_seconds}s" \ + env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ + opencode run "$(cat "$prompt_file")" \ + --pure \ + --agent "$agent" \ + --model "$model_candidate" \ + --format json \ + --title "PR #${PR_NUMBER} OpenCode bounded review ${model_candidate} attempt ${attempt}/${attempts}" \ + >"$opencode_json_file" 2>"$opencode_stderr_file" & + else + timeout --kill-after=30s "${run_timeout_seconds}s" \ + env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ + opencode run "$(cat "$prompt_file")" \ + --pure \ + --agent "$agent" \ + --model "$model_candidate" \ + --format json \ + --title "PR #${PR_NUMBER} OpenCode bounded review ${model_candidate} attempt ${attempt}/${attempts}" \ + >"$opencode_json_file" 2>"$opencode_stderr_file" & + fi opencode_pid=$! # Some providers (github-models ContextOverflowError) log a fatal error and # then hang instead of exiting, burning the whole run timeout. Watch the JSON @@ -484,12 +502,21 @@ run_one_model_attempt() { if has_fatal_provider_error_event "$opencode_json_file"; then printf 'OpenCode %s attempt %s/%s logged a fatal provider error while still running; killing the hung process instead of waiting out the %ss run timeout.\n' \ "$model_candidate" "$attempt" "$attempts" "$run_timeout_seconds" - kill "$opencode_pid" 2>/dev/null - for _ in $(seq 1 30); do - kill -0 "$opencode_pid" 2>/dev/null || break - sleep 1 - done - kill -9 "$opencode_pid" 2>/dev/null + if command -v setsid >/dev/null 2>&1; then + kill -TERM -- "-$opencode_pid" 2>/dev/null || true + for _ in $(seq 1 30); do + kill -0 -- "-$opencode_pid" 2>/dev/null || break + sleep 1 + done + kill -KILL -- "-$opencode_pid" 2>/dev/null || true + else + kill -TERM "$opencode_pid" 2>/dev/null || true + for _ in $(seq 1 30); do + kill -0 "$opencode_pid" 2>/dev/null || break + sleep 1 + done + kill -KILL "$opencode_pid" 2>/dev/null || true + fi break fi sleep "$fatal_poll_seconds" diff --git a/tests/test_fatal_provider_and_dependency_review_replacement.py b/tests/test_fatal_provider_and_dependency_review_replacement.py new file mode 100644 index 000000000..c0ff00663 --- /dev/null +++ b/tests/test_fatal_provider_and_dependency_review_replacement.py @@ -0,0 +1,22 @@ +"""Fail-first contracts for fatal-provider process-group termination.""" + +from __future__ import annotations + +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def test_fatal_provider_attempt_owns_and_terminates_its_process_group() -> None: + """Fatal-provider cleanup must signal the dedicated attempt process group.""" + launcher = ( + REPO_ROOT / "scripts" / "ci" / "run_opencode_review_model_pool.sh" + ).read_text(encoding="utf-8") + + assert 'setsid timeout --kill-after=30s "${run_timeout_seconds}s"' in launcher + assert 'kill -TERM -- "-$opencode_pid"' in launcher + assert 'kill -0 -- "-$opencode_pid"' in launcher + assert 'kill -KILL -- "-$opencode_pid"' in launcher + assert 'kill "$opencode_pid"' not in launcher + assert 'kill -9 "$opencode_pid"' not in launcher