From 6a004b00bb52fa47ce8c3027eae7af18c2388e6d Mon Sep 17 00:00:00 2001 From: iiChole <230967553+iiChole@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:11:11 +0800 Subject: [PATCH 1/2] feat: add standalone structured delta prototype and reproducible pilot --- .github/workflows/structured-delta.yml | 36 + README.md | 7 + README.zh-cn.md | 7 + docs/STRUCTURED-DELTA.md | 32 + prototypes/structured-delta/.gitignore | 13 + prototypes/structured-delta/PROTOCOL.md | 165 +++ prototypes/structured-delta/README.md | 98 ++ .../structured-delta/SOURCE_PROVENANCE.json | 163 +++ .../structured-delta/experiments/__init__.py | 1 + .../experiments/baselines/__init__.py | 0 .../experiments/baselines/ast_overlap.py | 18 + .../experiments/baselines/embedding.py | 50 + .../experiments/baselines/file_overlap.py | 11 + .../experiments/baselines/git_merge.py | 11 + .../experiments/baselines/llm_intent.py | 44 + .../experiments/benchmark/build.py | 968 ++++++++++++++++++ .../experiments/delta/__init__.py | 0 .../experiments/delta/extractor.py | 449 ++++++++ .../experiments/delta/interaction.py | 190 ++++ .../experiments/delta/model.py | 61 ++ .../experiments/evaluation/__init__.py | 0 .../experiments/evaluation/evaluate.py | 156 +++ .../experiments/evaluation/freeze.py | 55 + .../experiments/evaluation/metrics.py | 67 ++ .../experiments/evaluation/report.py | 296 ++++++ .../experiments/evaluation/runtime.py | 221 ++++ .../experiments/evaluation/validate.py | 66 ++ .../experiments/online/__init__.py | 0 .../experiments/online/sequences.py | 24 + .../experiments/online/simulator.py | 139 +++ .../experiments/tests/__init__.py | 0 .../experiments/tests/test_prototype.py | 172 ++++ prototypes/structured-delta/pyproject.toml | 7 + .../structured-delta/tools/run_pilot.py | 57 ++ .../structured-delta/tools/verify_source.py | 40 + 35 files changed, 3624 insertions(+) create mode 100644 .github/workflows/structured-delta.yml create mode 100644 docs/STRUCTURED-DELTA.md create mode 100644 prototypes/structured-delta/.gitignore create mode 100644 prototypes/structured-delta/PROTOCOL.md create mode 100644 prototypes/structured-delta/README.md create mode 100644 prototypes/structured-delta/SOURCE_PROVENANCE.json create mode 100644 prototypes/structured-delta/experiments/__init__.py create mode 100644 prototypes/structured-delta/experiments/baselines/__init__.py create mode 100644 prototypes/structured-delta/experiments/baselines/ast_overlap.py create mode 100644 prototypes/structured-delta/experiments/baselines/embedding.py create mode 100644 prototypes/structured-delta/experiments/baselines/file_overlap.py create mode 100644 prototypes/structured-delta/experiments/baselines/git_merge.py create mode 100644 prototypes/structured-delta/experiments/baselines/llm_intent.py create mode 100644 prototypes/structured-delta/experiments/benchmark/build.py create mode 100644 prototypes/structured-delta/experiments/delta/__init__.py create mode 100644 prototypes/structured-delta/experiments/delta/extractor.py create mode 100644 prototypes/structured-delta/experiments/delta/interaction.py create mode 100644 prototypes/structured-delta/experiments/delta/model.py create mode 100644 prototypes/structured-delta/experiments/evaluation/__init__.py create mode 100644 prototypes/structured-delta/experiments/evaluation/evaluate.py create mode 100644 prototypes/structured-delta/experiments/evaluation/freeze.py create mode 100644 prototypes/structured-delta/experiments/evaluation/metrics.py create mode 100644 prototypes/structured-delta/experiments/evaluation/report.py create mode 100644 prototypes/structured-delta/experiments/evaluation/runtime.py create mode 100644 prototypes/structured-delta/experiments/evaluation/validate.py create mode 100644 prototypes/structured-delta/experiments/online/__init__.py create mode 100644 prototypes/structured-delta/experiments/online/sequences.py create mode 100644 prototypes/structured-delta/experiments/online/simulator.py create mode 100644 prototypes/structured-delta/experiments/tests/__init__.py create mode 100644 prototypes/structured-delta/experiments/tests/test_prototype.py create mode 100644 prototypes/structured-delta/pyproject.toml create mode 100644 prototypes/structured-delta/tools/run_pilot.py create mode 100644 prototypes/structured-delta/tools/verify_source.py diff --git a/.github/workflows/structured-delta.yml b/.github/workflows/structured-delta.yml new file mode 100644 index 0000000..96801d6 --- /dev/null +++ b/.github/workflows/structured-delta.yml @@ -0,0 +1,36 @@ +name: Structured Delta prototype + +on: + push: + paths: + - "prototypes/structured-delta/**" + - ".github/workflows/structured-delta.yml" + pull_request: + paths: + - "prototypes/structured-delta/**" + - ".github/workflows/structured-delta.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + matrix: + python: ["3.11", "3.14"] + defaults: + run: + working-directory: prototypes/structured-delta + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - run: python -m pip install ruff==0.16.10 + - run: ruff check . && ruff format --check . + - run: python tools/verify_source.py + - run: python -m unittest discover -s experiments/tests -v + - run: python tools/run_pilot.py --out artifacts/ci-pilot diff --git a/README.md b/README.md index d29ef90..8aa9ffd 100644 --- a/README.md +++ b/README.md @@ -524,3 +524,10 @@ spellings of either would mean a hook that never finds its own daemon, and a dae finds a reason not to start a second one. MIT licensed. + +## Structured Delta research prototype + +A standalone Python implementation and reproduction guide are available in +[prototypes/structured-delta](prototypes/structured-delta/README.md). +See [scope and evidence](docs/STRUCTURED-DELTA.md); this prototype is separate +from the product runtime, and generated experimental artifacts stay local. diff --git a/README.zh-cn.md b/README.zh-cn.md index 40dd701..81b6375 100644 --- a/README.zh-cn.md +++ b/README.zh-cn.md @@ -467,3 +467,10 @@ npm run typecheck 而 daemon 永远找不到一个"不必再起一个"的理由。 MIT 许可。 + +## 结构化 Delta 研究原型 + +独立 Python 实现、测试和复现入口见 +[prototypes/structured-delta](prototypes/structured-delta/README.md)。 +[方案范围与证据](docs/STRUCTURED-DELTA.md)说明现有结果及限制； +原型尚未接入产品运行流程，生成的实验数据与结果保留在本地。 diff --git a/docs/STRUCTURED-DELTA.md b/docs/STRUCTURED-DELTA.md new file mode 100644 index 0000000..36cf24e --- /dev/null +++ b/docs/STRUCTURED-DELTA.md @@ -0,0 +1,32 @@ +# Structured program-state delta prototype + +The standalone implementation is in +[prototypes/structured-delta](../prototypes/structured-delta/README.md). +It extracts changes from common-base Python snapshots and uses deterministic +dependency, contract and effect rules to classify concurrent interactions. +It is not integrated into the product's CLI, hooks, routing or acceptance flow. + +The published code includes regression tests and an isolated pilot reproduction +runner. Source has consistent formatting and unused imports removed; AST +provenance is recorded. Original frozen source and experimental evidence remain +unaltered in local research storage. Generated fixtures, results and real-repository +reviewer materials are not tracked in Git. + +The 48-case authored pilot achieved conflict F1 74.3%, with 13 true detections, +6 false positives and 3 missed conflicts. It detected 8/11 cross-file conflicts. +The online pilot showed no positive warning lead. Embedding and LLM baselines +were not run. These findings support further independent evaluation, not a +real-world superiority claim. + +Real-repository blind-transfer preparation selected 70 eligible pairs, below its +100-pair target. Packaging and independent labeling are incomplete; no blind-test +scores exist. The next step is independent annotation and adjudication with +executable branch requirements before detector scoring. + +To validate the published implementation, run from its directory: + +```bash +python3 tools/verify_source.py +python3 -m unittest discover -s experiments/tests -v +python3 tools/run_pilot.py --out artifacts/pilot-001 +``` diff --git a/prototypes/structured-delta/.gitignore b/prototypes/structured-delta/.gitignore new file mode 100644 index 0000000..c944428 --- /dev/null +++ b/prototypes/structured-delta/.gitignore @@ -0,0 +1,13 @@ +__pycache__/ +*.py[cod] +.ruff_cache/ +.venv/ +artifacts/ +experiments/benchmark/cases/ +experiments/results/ +FREEZE.json +blind-transfer-*/private/ +blind-transfer-*/reviewer/ +.env +.env.* +!.env.example diff --git a/prototypes/structured-delta/PROTOCOL.md b/prototypes/structured-delta/PROTOCOL.md new file mode 100644 index 0000000..df85e8b --- /dev/null +++ b/prototypes/structured-delta/PROTOCOL.md @@ -0,0 +1,165 @@ +# Structured program-state delta pilot, protocol v1 + +Date: 2026-10-07. This protocol and all code/case bytes are SHA-256 frozen in +FREEZE.json before the first scored run. Development unit tests and fixture +validation precede freezing; no scored results have been used to choose rules. +Do not edit frozen inputs to improve run-001. Retain failed runs and executor +issues; subsequent changes require a separately versioned protocol/directory. + +## Question and scope + +Can deterministic structured deltas improve detection and localization of +concurrent semantic interactions relative to scope/text heuristics? Only a +Python research prototype: no AgentGit runtime, routing, repair, persistent +memory, hooks, MCP, or live-agent orchestration. Dependencies are standard +Python >=3.11 and Git >=2.38 (merge-tree --write-tree). + +This is a jointly authored, transparent DEVELOPMENT benchmark. Case author and +rule implementer are the same; it is not blind or held-out. 48 microrepositories, +10 independent / 14 compatible / 8 redundant / 16 conflicting, with 36 named +families (names organize analysis, not proof of independent sampling). The +initial sample covers the requested ten situations and includes unsupported +features. Scaling the same templates does not create independent evidence. + +## Inputs, labels and ground truth + +Each directory holds base/, patch_A.diff, patch_B.diff, materialized a/ and b/, +gold.json with interaction label/entity/explanation, and checks/{base,a,b}.py. +Nonconflicting cases include witness/ showing a state that satisfies both sets +of branch requirements. Redundant witnesses use A's state. Gold denotes: + +- independent: no shared behavioral requirement or resolved interaction entity; +- compatible: related scopes/effects can preserve both branch requirements; +- redundant: equivalent change over the case's declared input domain; +- conflicting: the unchanged concurrent transformations break at least one + requirement, or require contradictory contracts for the same entity. + +Textual overlap alone does not decide the label. Checks are finite witnesses, +not proofs of universal equivalence. Base validation checks syntax; branch +checks establish local required behavior. All A and B states must pass their +own checks; every provided joint witness must pass both. Patch application +must reproduce the supplied branch snapshots exactly. No benchmark execution +failure is silently dropped. All 48 cases remain in scoring. + +Detector API accepts only base and changed source snapshots. It has no gold, +requirements, test results, witness, case ID, family, or edit description input. +Gold and checks are accessed after static predictions; Git baseline receives +only Git clean/conflict status, never test outcomes. Tests are isolated Python +subprocesses. This is not a repair experiment. There are no hidden tests and no +claim of hidden-quality generalization. + +## Methods and frozen choices + +B1 file overlap; B2 line overlap and qualified AST symbol overlap (reported +separately); B3 Git merge-tree conflict. These binary heuristics emit conflicting +on overlap/conflict and independent otherwise. They cannot emit compatible or +redundant. Report four-class macro F1 as requested, but do not infer superiority +from macro F1 alone: binary precision/recall and compatible false positives are +necessary comparisons. B2 AST means enclosing changed AST definitions, not a +full tree-edit-distance algorithm. Line positions use common-base lines and +half-integer insertion anchors. + +B4: opt-in external diff embedding via JSON command, cosine >=0.80 predicts +conflicting; no threshold selection on this benchmark, no fake local embedding. +No configured command means NOT RUN, not a score of zero. B5: opt-in command +first describes each actual diff, then judges only the descriptions, returning +one of four labels and up to three entities. This is a description bottleneck, +not a full-source LLM baseline. It is not actual live agents' self-reported +intent. Raw descriptions, judgments, reported usage, model ID and errors are +retained. Missing usage is unknown. Partial runs report coverage and are not +ranked against full-coverage methods. No external command is configured in v1. + +Structured delta: Python AST definitions, create/delete/body-equal rename, +statement edit operations, resolved local/relative imports and aliases, calls, +function signatures/annotations, literal return types/dict keys, module globals +and constant-key read/write effects, requirements.txt entries and flat JSON. +Identity-test normalization proves only a very narrow form of redundancy; it +does not equate == with is or perform arbitrary algebraic equivalence. + +Rule priority: canonical transformation equality => redundant; deletion/rename +against changed implementations or consumers => conflict; incompatible observed +call signature or changed return/value contract consumed by another patch => +conflict; (Full only) changed read/write interference, divergent return behavior +or signature replacements and duplicate definitions => conflict; resolved shared +scope/dependency otherwise => compatible; otherwise independent. Scores are +ordinal rule strengths, not probabilities or cosine values. + +Conservative contract rules may reject safe schema/config changes. Calls/reads +include before and after observations; obsolete assumptions can therefore +remain. No alias heap analysis, CFG/path sensitivity, general type inference, +transitive effect analysis, dynamic lookup/JSON consumer resolution or version +solver. Unresolved observations are retained; forced four-class output is not +a safety guarantee. Constant-key analysis is shallow. Source annotations are +observations, not enforced runtime types. Package manifest extraction does not +resolve installed distributions to module imports. + +Ablation ladder: File; File+Symbol; File+Symbol+Dependency; +File+Symbol+Dependency+Contract; Full Structured Delta. First three use binary +intersection rules; Contract uses incompatibility and compatible fallback; Full +adds actual write effects, divergent returns and canonical redundancy. These +are rule/feature bundle ablations, not a trained model with one fixed decoder. +Effects cannot be interpreted as isolated causal contributions of features. + +## Metrics and falsification criterion + +Four-class macro F1 (all four classes, undefined class F1=0), conflict precision +and recall, compatible->conflicting rate, cross-file conflict recall (changed +file sets are disjoint), entity Top-1 and Recall@3. Localization is exact entity +identity, independent of class correctness, denominator all non-independent +cases with gold entities. File-only predictions do not earn symbol localization +credit. A file-level localization score is not substituted for the entity metric. +One gold entity per case in v1, so Recall@3 is a hit rate. Undefined denominators +are null. Report counts, per-class confusion, every family, all FP/FN, not only +aggregate. Report binary conflict F1 alongside mandatory metrics. + +Bounded positive support requires Full conflict F1 to exceed the strongest +available B1/B2/B3 conflict F1, reduce compatible false positives, and improve +cross-file conflict recall. Failure of those criteria does not support even the +bounded claim. Meeting them supports only these authored cases, not general +semantic correctness, comparison to unavailable B4/B5, or real-world superiority. +No p-value based generalization claim is planned for this constructed sample. + +## Commutativity + +Strict attempt: git apply A then B and B then A, with atomic patch application, +recording both apply statuses, both branch requirement results if applicable, +and exact full-source SHA-256 state equality when both orders apply. A failed +order has tests=null and resulting-state comparison=null, never a pass. + +Separately record true Git three-way merge-tree in both parent orders and the +same checks and state hashes. This is a base-aware textual reconciliation, +not function composition or automatic repair. Preserve conflict-marked trees +for inspection. Text application or three-way noncommutativity is an interaction +signal only: compatible edits may overlap, and commuting cross-file edits may +break runtime tests. Never derive gold or structured predictions from it. + +## Online protocol + +12 preregistered cases, 6 interleaved events each: A1 B1 A2 B2 A3 B3. A1/B1 add +an unused helper; A2/B2 apply the first changed file; A3/B3 complete the remaining +files and remove helper. These are synthetic progress scaffolds, not natural +agent trajectories. All incremental diffs are stored and checked against their +snapshots. Active delta is always base->current branch, not only latest diff. +After every event recompute prediction and trial merge both active branches. +Requirements activate as soon as a branch is functionally complete (ignoring +progress helper), not merely at A3/B3, preventing artificial positive lead. +Always check syntax. Unfinished branches have no extra behavior assertions. + +Record first predicted conflict, first merge failure, first clean-merge test +failure, failure minus detection in event ticks (positive=earlier detection, +negative=late); null if unavailable. Report semantic-test lead separately and +real detector wall time in ms. False intervention = ever warning on final +nonconflicting gold; false early intervention additionally precedes first failure +or occurs when no failure is observed. Both definitions are reported so textual +failures do not legitimize warnings on compatible edits. Final labels are a +limited proxy for transient ground truth. Online run starts only after static +pairwise experiment succeeds. No repair or context-routing decisions occur. + +## Most important next experiment + +Freeze the detector, then independently collect and annotate >=100 real +concurrent edit pairs from repositories excluded from development, with executable +branch/joint requirements and independent adjudication. Include compatible +contract changes and dynamic cross-file dependencies; lock model providers and +thresholds on a separate calibration set for B4/B5. This one blinded transfer +test is more informative than expanding authored templates or adding routing. diff --git a/prototypes/structured-delta/README.md b/prototypes/structured-delta/README.md new file mode 100644 index 0000000..1175d15 --- /dev/null +++ b/prototypes/structured-delta/README.md @@ -0,0 +1,98 @@ +# Structured program-state delta prototype + +This standalone Python prototype detects interactions between two changes from a +common source snapshot. It uses AST symbols, import/call dependencies, signatures, +limited return contracts and state read/write observations to classify changes as +`independent`, `compatible`, `redundant` or `conflicting` and locate affected entities. +It is separate from AgentGit's product runtime and has no automatic repair or +live-agent integration. + +Requirements: Python **>=3.11** and Git **>=2.38**. Runtime dependencies are entirely +in the Python standard library; no API keys or external services are needed. + +## Use the detector + +Run from this directory: + +```python +from experiments.delta.extractor import extract +from experiments.delta.interaction import detect + +base = {"api.py": "def fetch(): return 1\n", "client.py": "VALUE = 1\n"} +a = {**base, "api.py": "def load(): return 1\n"} +b = {**base, "client.py": "from api import fetch\ndef run(): return fetch() + 1\n"} + +prediction = detect(extract(base, a), extract(base, b)) +print(prediction.to_dict()) +# label: conflicting; entity: api.py::fetch +``` + +Inputs are dictionaries of repository-relative paths to source text. Detection +accepts source snapshots only, not ground truth, tests or edit descriptions. +Rule scores are ordinal strengths, not probabilities. Unsupported dynamic +lookups and runtime configuration relationships may remain unresolved. + +## Validate and reproduce the development pilot + +```bash +cd prototypes/structured-delta +python3 tools/verify_source.py +python3 -m unittest discover -s experiments/tests -v +python3 tools/run_pilot.py --out artifacts/pilot-001 +``` + +The runner creates a fresh, ignored directory, copies the published source and +original protocol, generates 48 fixtures, validates branch requirements and +joint witnesses, freezes inputs, runs static and online evaluations, then writes +`REPORT.md`. Existing output directories are rejected. Full case files, patches, +checks, logs, freezes and results stay in that directory; they are not tracked. +This is a reproduction using the cleaned source, not a rewrite of the historical +frozen run. The original protocol's "36 named families" is a narrative counting +error: the generator produces **35** families. + +The historical development pilot had conflict F1 **74.3%**, conflict precision +**68.4%**, recall **81.2%**, compatible false positives **3/14**, cross-file conflict +recall **8/11**, and entity Top-1 **34/38**. Its authored cases and rules shared an +author and were not a blind holdout. The online pilot detected 5/6 conflicts with +zero positive lead and 2/6 false interventions. These are limited development +results, not evidence of real-repository generalization or superiority to an LLM. + +`SOURCE_PROVENANCE.json` records original and published source hashes and AST +digests. Formatting and removal of seven unused imports are the only changes to +the published implementation. Historical source, frozen inputs and results are +preserved locally. [PROTOCOL.md](PROTOCOL.md) is the unmodified original protocol. + +## Optional model baselines + +The embedding and LLM description adapters in `experiments/baselines/` only invoke +an explicitly supplied JSON provider command. They do not discover credentials or +call a service automatically. Without a provider they report `not_configured`. +Provider commands receive JSON on stdin and return JSON on stdout; their module +docstrings describe the formats. Model IDs, available usage, errors and coverage +are retained. The pilot runner does not enable model providers. + +## Layout and blind-transfer status + +| Path | Purpose | +| --- | --- | +| `experiments/delta/` | Structured observations and deterministic interaction rules | +| `experiments/baselines/` | File, line, symbol, Git and optional model comparisons | +| `experiments/benchmark/` | Development fixture generator | +| `experiments/evaluation/` | Validation, freezing, metrics and reporting | +| `experiments/online/` | Synthetic incremental simulation | +| `experiments/tests/` | Detector, metric and Git regression tests | +| `tools/` | Source provenance verification and isolated reproduction | +| `artifacts/` | Locally generated data and results; ignored by Git | + +A separate real-repository annotation preparation selected **70** eligible pairs +under its preregistered sampling limits, below the target of 100. Packaging was +interrupted and independent annotation has not started. There are no blind-test +scores. Repository caches, source archives, provenance mappings and reviewer +materials are local research artifacts and are excluded from this publication. + +For style checks, use Ruff 0.16.10: + +```bash +ruff check . +ruff format --check . +``` diff --git a/prototypes/structured-delta/SOURCE_PROVENANCE.json b/prototypes/structured-delta/SOURCE_PROVENANCE.json new file mode 100644 index 0000000..dbdaabc --- /dev/null +++ b/prototypes/structured-delta/SOURCE_PROVENANCE.json @@ -0,0 +1,163 @@ +{ + "source_date": "2026-10-07", + "original_freeze_sha256": "05c32a8d8dbbefbe5408b7a70c25ff4abdf40ac023a2c2335d0a4569a16d0970", + "protocol_sha256": "54cad569cf2242073884738a5ce229db24cfd48abaed73a1adfb6e5bd363a528", + "changes": "Ruff formatting and seven unused imports removed; all remaining ASTs identical to frozen originals.", + "modules": { + "experiments/__init__.py": { + "original_sha256": "303e4a013112bc04899edb2d66680d4856e30b79258f3cc28333e99e973e3b78", + "published_sha256": "303e4a013112bc04899edb2d66680d4856e30b79258f3cc28333e99e973e3b78", + "program_ast_sha256": "08febec1203888913739133cf412415dde251f771db1982c0b03cb1d76da1a48", + "removed_unused_imports": [] + }, + "experiments/baselines/__init__.py": { + "original_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "published_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "program_ast_sha256": "ad2e13b69c4fc1fda46413f740f426fc1793cfc6d6da8d226ba619f1aef48be7", + "removed_unused_imports": [] + }, + "experiments/baselines/ast_overlap.py": { + "original_sha256": "3eb04806389f0f4443fc09bda741d60ae924afd87d659dc7eb7cd9ff6c20bc93", + "published_sha256": "6a156b104c5e05c8122fa8c786cad789d9634abac1bf2d746caece1b4026a4d4", + "program_ast_sha256": "0b7e2c55a217974d596aa9ff232db6c1b057c717a4b05a10a53550680869e64b", + "removed_unused_imports": [] + }, + "experiments/baselines/embedding.py": { + "original_sha256": "b55e9cfb9bb17e22fcd6029f55fb8ba314f46754abd8aa87e899a944ea5f6d90", + "published_sha256": "b21a809416fa0a6e43f63200c68d25884140c45912ef0794999b3fc9ffb49444", + "program_ast_sha256": "4a94ba0546613ca159fc970635bb9c4a7fff4f34b1ed851d5dcbdb6f396f7ebe", + "removed_unused_imports": [] + }, + "experiments/baselines/file_overlap.py": { + "original_sha256": "943457daba77dd5780eaf6a3b3221fafed16f205859c24669827df1815b9a8ef", + "published_sha256": "5cf29d56660981a00207e4884f7a3e8e7f5ed3c787615f2099b7a950ad94e790", + "program_ast_sha256": "dc4f1e8ecdb2e7122b2ff748efbaee50dd46da64dc49762f1d47f1b8dc320dd5", + "removed_unused_imports": [] + }, + "experiments/baselines/git_merge.py": { + "original_sha256": "9890f74b9c015ea12653b86c6deb2389f47371dbd72c83ed304ddc4f3cebeac5", + "published_sha256": "e2233dce3b295fad4fd135fb35021fde1d660d11471e643eb3dd6f3ed0dce3ce", + "program_ast_sha256": "c2cb7f664b29339c2b701dd91aed1a248e4726b4f3e4c08f505af8d97c10a6a9", + "removed_unused_imports": [] + }, + "experiments/baselines/llm_intent.py": { + "original_sha256": "77cb7a058683aa5e381fef8214dc1c14253a90e916ccaa9eff93c896214e4a5b", + "published_sha256": "e43dc098d8da1c29d0b2fe8b6c76b48601a917605a17e531f4d64d8934d08054", + "program_ast_sha256": "e4a4f3d4acbb4f6f6ac6fe80031f8e3695844e40d695b32b25a39391af0de19e", + "removed_unused_imports": [] + }, + "experiments/benchmark/build.py": { + "original_sha256": "472a22aab9a9aa33011fd62791ab94e3242f110657ad178f93f7a57471501410", + "published_sha256": "ad197e5ef0f3e5b8a6d557dcb1e00dec707f6a449dc7763fd192cef1813c3718", + "program_ast_sha256": "8ba61efbe04df540bfbe802c32ad6ad3f4ce2395ee49be2f9caabb2848e24cfc", + "removed_unused_imports": [] + }, + "experiments/delta/__init__.py": { + "original_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "published_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "program_ast_sha256": "ad2e13b69c4fc1fda46413f740f426fc1793cfc6d6da8d226ba619f1aef48be7", + "removed_unused_imports": [] + }, + "experiments/delta/extractor.py": { + "original_sha256": "5d5f06bdda32007ba6f34257ffb368bef6b5125a2fa52916aeb2f70d4308b2fb", + "published_sha256": "ed271ab1b05dbfbb7a26b2c80f2dd8603abf67b6079553de74fa237dfe00898a", + "program_ast_sha256": "02a62047c8d0719b4fc90f68b697e5beb7b24363ba8d438ee4de8b71a60bd609", + "removed_unused_imports": [] + }, + "experiments/delta/interaction.py": { + "original_sha256": "34ffdd983956a7e00ed0e52a684094c0bac8499ede08cf111ab36a86bdd8ce4f", + "published_sha256": "010b90920b702c7fbf03936290d060abe2ff08cdce24d53e3f5a803c11db1754", + "program_ast_sha256": "3def3e6dc452130156c5f2f60d79b0419c1c9cec7d35ac654d618013a56943fe", + "removed_unused_imports": [] + }, + "experiments/delta/model.py": { + "original_sha256": "7463425ea8bbf61e9a39968802a511991b91e43a7cb8eaf1492f625ae80da7a0", + "published_sha256": "5462a45f1951fac2820e2bf410a61a946da464c4a73d63a80b6c89a7a05a2f3b", + "program_ast_sha256": "1e70d37045f7587f0ce64713a04f1594c72a2b3fb8838264248f4461fefa5ccc", + "removed_unused_imports": [] + }, + "experiments/evaluation/__init__.py": { + "original_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "published_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "program_ast_sha256": "ad2e13b69c4fc1fda46413f740f426fc1793cfc6d6da8d226ba619f1aef48be7", + "removed_unused_imports": [] + }, + "experiments/evaluation/evaluate.py": { + "original_sha256": "a396889a515954261702718335d642621ae0b2165b84ad36263eaef2f2cbf666", + "published_sha256": "2a0fa4d4f2840f158ed99907b0e23e6a8ba2c67e9097637c7c1a2005562a6ad8", + "program_ast_sha256": "a630c9fc72907145eaadc7a105a7b597c50313c4c957eb6587f266a81da1b349", + "removed_unused_imports": [] + }, + "experiments/evaluation/freeze.py": { + "original_sha256": "969f7987af75373662d6b42fa4f12e4e05c523ad3f115aa2dcc13753f2e37807", + "published_sha256": "4df8b069f78ad80c4f8dcc9b24208dbf87bbeced6cf08dee456e6b3190df4bf9", + "program_ast_sha256": "afd3ac3fc1dd44c70ed5ec26e77d809b807cd993e5614767eb17e3892a5f2915", + "removed_unused_imports": [] + }, + "experiments/evaluation/metrics.py": { + "original_sha256": "acdf0062997ee8a1584e8402b741d8598ec9812a1dc8581888f144c79d024401", + "published_sha256": "5c452bc86751a0ba493cf98d52c7187fbfcc7ca028f9b420ca165ce3fc02a4a6", + "program_ast_sha256": "759a410c56eb9f18f1dd7a1ee6dd7bafe6253ff7787c52af52543162720e08b2", + "removed_unused_imports": [] + }, + "experiments/evaluation/report.py": { + "original_sha256": "35c54e5999c957ad63d9f39632718b13621889a6410bf09ebece269732c880b8", + "published_sha256": "707b52d43190a4f4dbc9dc14d34f5f0251204cb473fc8eff9ffa4d0f5222130c", + "program_ast_sha256": "2f0dc50f9c0d28b19d5f439043f21f083cd43fc23d1c754b6aa74a8d453b1ca9", + "removed_unused_imports": [ + "Counter" + ] + }, + "experiments/evaluation/runtime.py": { + "original_sha256": "11afc99694ac12b49696731dcb3218e898599a405218c83f53a3646befe9ac41", + "published_sha256": "e0fe979c533c120d0008ed6a250a78dca3839eab050fcfdb7fded4a4a4a66d77", + "program_ast_sha256": "1be4c2dfeca897ee8f272a4bb287fdbb56ff4f491589c7680131ac8d9ce3784f", + "removed_unused_imports": [ + "io", + "tarfile" + ] + }, + "experiments/evaluation/validate.py": { + "original_sha256": "88d08dd2316c2473bff4e3549c9dec2f33e04b3f743077eccc3ab1759ac58174", + "published_sha256": "cca64706b27c61ef770c5f999e7adab0450594da40987cf9794d5a78efe0e30f", + "program_ast_sha256": "63c626297e04963dc88ed91c3752fc8313456e0e83fd9cd129a382fc16160a48", + "removed_unused_imports": [] + }, + "experiments/online/__init__.py": { + "original_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "published_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "program_ast_sha256": "ad2e13b69c4fc1fda46413f740f426fc1793cfc6d6da8d226ba619f1aef48be7", + "removed_unused_imports": [] + }, + "experiments/online/sequences.py": { + "original_sha256": "fad006138e17606d9022f6c28d26c0c5aa5a0d71176b9bac8863d4a63b3083c5", + "published_sha256": "50742fb4b5f04991dca4ca14c634d2e10ef59bdf5a8a2c3c11394245aa0064a8", + "program_ast_sha256": "6a3f42759362bd013048093a9ba82918006f58c3e91f22bbb0af54b726b2a41b", + "removed_unused_imports": [] + }, + "experiments/online/simulator.py": { + "original_sha256": "1b2efa1d5738dda82ccfbdeee8574fc8477f53d5a9d8c2e7124e5084b71adbbc", + "published_sha256": "c654804584b4f8f152d08278c39127c7365970b9b6ab4b93125af2c655319a8e", + "program_ast_sha256": "2b450ec1cd9eb46906c44d8198e8c9d5ca66a90a104ea5c07926949c83a8a86e", + "removed_unused_imports": [ + "patch" + ] + }, + "experiments/tests/__init__.py": { + "original_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "published_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "program_ast_sha256": "ad2e13b69c4fc1fda46413f740f426fc1793cfc6d6da8d226ba619f1aef48be7", + "removed_unused_imports": [] + }, + "experiments/tests/test_prototype.py": { + "original_sha256": "e3ccded56a270097564000319388727e5e063bfe25841b0b7beb09d0172e07c6", + "published_sha256": "16b2fad62f0362c5b38f82cc38794c8ee13acd36f9ddaa116b2d94a4f299245b", + "program_ast_sha256": "55e891e729a60f09681ebc7abf87a2466754df43d41617d8faa44f6bedc24124", + "removed_unused_imports": [ + "Path", + "json", + "tempfile" + ] + } + } +} diff --git a/prototypes/structured-delta/experiments/__init__.py b/prototypes/structured-delta/experiments/__init__.py new file mode 100644 index 0000000..ba7f492 --- /dev/null +++ b/prototypes/structured-delta/experiments/__init__.py @@ -0,0 +1 @@ +"""Standalone structured program-delta experiment; no AgentGit runtime integration.""" diff --git a/prototypes/structured-delta/experiments/baselines/__init__.py b/prototypes/structured-delta/experiments/baselines/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prototypes/structured-delta/experiments/baselines/ast_overlap.py b/prototypes/structured-delta/experiments/baselines/ast_overlap.py new file mode 100644 index 0000000..c0f669c --- /dev/null +++ b/prototypes/structured-delta/experiments/baselines/ast_overlap.py @@ -0,0 +1,18 @@ +from ..delta.model import Prediction + + +def predict(a, b, mode="symbol"): + if mode == "line": + entities = [ + f"file:{p}" + for p in sorted(set(a.files) & set(b.files)) + if set(a.lines[p]) & set(b.lines[p]) + ] + else: + entities = sorted(set(a.symbols) & set(b.symbols)) + return Prediction( + "conflicting" if entities else "independent", + entities[:3], + float(bool(entities)), + [mode + " overlap"], + ) diff --git a/prototypes/structured-delta/experiments/baselines/embedding.py b/prototypes/structured-delta/experiments/baselines/embedding.py new file mode 100644 index 0000000..fa614f5 --- /dev/null +++ b/prototypes/structured-delta/experiments/baselines/embedding.py @@ -0,0 +1,50 @@ +"""Optional external embedding baseline; never replaced with a fake embedding. + +Provider command receives JSON on stdin: {task: 'embed', texts:[diffA,diffB]}. +Return {vectors:[[...],[...]], model: str, usage: object|null} on stdout. +The provider owns auth/network. No automatic model discovery or invocation. +""" + +import json +import math +import shlex +import subprocess +from ..delta.model import Prediction + + +def invoke(command, payload): + process = subprocess.run( + shlex.split(command), + input=json.dumps(payload), + text=True, + capture_output=True, + timeout=180, + check=True, + ) + return json.loads(process.stdout) + + +def predict(command, diff_a, diff_b, a, b, threshold=0.80): + raw = invoke(command, {"task": "embed", "texts": [diff_a, diff_b]}) + x, y = raw["vectors"] + if ( + not x + or len(x) != len(y) + or not all(isinstance(v, (float, int)) and math.isfinite(v) for v in x + y) + ): + raise ValueError("Invalid embedding vectors") + norm = math.sqrt(sum(v * v for v in x) * sum(v * v for v in y)) + if not norm: + raise ValueError("Zero embedding norm") + sim = sum(u * v for u, v in zip(x, y)) / norm + # This is intentionally a similarity baseline, not our definition of conflict. + candidates = sorted(set(a.symbols) & set(b.symbols)) + [ + "file:" + p for p in sorted(set(a.files) & set(b.files)) + ] + return Prediction( + "conflicting" if sim >= threshold else "independent", + candidates[:3], + sim, + ["Fixed embedding cosine threshold"], + {"model": raw.get("model"), "usage": raw.get("usage"), "threshold": threshold}, + ) diff --git a/prototypes/structured-delta/experiments/baselines/file_overlap.py b/prototypes/structured-delta/experiments/baselines/file_overlap.py new file mode 100644 index 0000000..a9dbc9d --- /dev/null +++ b/prototypes/structured-delta/experiments/baselines/file_overlap.py @@ -0,0 +1,11 @@ +from ..delta.model import Prediction + + +def predict(a, b): + entities = ["file:" + p for p in sorted(set(a.files) & set(b.files))] + return Prediction( + "conflicting" if entities else "independent", + entities[:3], + float(bool(entities)), + ["File overlap"], + ) diff --git a/prototypes/structured-delta/experiments/baselines/git_merge.py b/prototypes/structured-delta/experiments/baselines/git_merge.py new file mode 100644 index 0000000..878ee65 --- /dev/null +++ b/prototypes/structured-delta/experiments/baselines/git_merge.py @@ -0,0 +1,11 @@ +from ..delta.model import Prediction + + +def predict(merge_result): + entities = ["file:" + p for p in merge_result["conflict_paths"]] + return Prediction( + "conflicting" if not merge_result["clean"] else "independent", + entities[:3], + float(not merge_result["clean"]), + ["Git merge-tree textual/structural merge"], + ) diff --git a/prototypes/structured-delta/experiments/baselines/llm_intent.py b/prototypes/structured-delta/experiments/baselines/llm_intent.py new file mode 100644 index 0000000..b885674 --- /dev/null +++ b/prototypes/structured-delta/experiments/baselines/llm_intent.py @@ -0,0 +1,44 @@ +"""Optional two-stage edit-description baseline, based on actual diffs, not gold. + +Provider command accepts task='describe' / task='judge', returning description +or label/entities, plus model and usage. Prompts and raw outputs are persisted. +""" + +from .embedding import invoke +from ..delta.model import Prediction, LABELS + + +def predict(command, diff_a, diff_b): + traces = [] + descriptions = [] + for diff in (diff_a, diff_b): + request = { + "task": "describe", + "instruction": "Describe the behavior of this code edit using only the diff. Identify affected entities as path::symbol or dependency:name. Do not assume unstated intent.", + "diff": diff, + } + response = invoke(command, request) + if not isinstance(response.get("description"), str): + raise ValueError("Missing edit description") + descriptions.append(response["description"]) + traces.append({"request": request, "response": response}) + request = { + "task": "judge", + "instruction": "Judge concurrent changes from a common base. Return label: independent, compatible, redundant, or conflicting; and up to 3 interaction entities. Same scope is not sufficient for conflict.", + "descriptions": descriptions, + } + response = invoke(command, request) + traces.append({"request": request, "response": response}) + if ( + response.get("label") not in LABELS + or not isinstance(response.get("entities"), list) + or not all(isinstance(e, str) for e in response["entities"]) + ): + raise ValueError("Invalid model judgment") + return Prediction( + response["label"], + response["entities"][:3], + 0, + ["LLM edit descriptions + judgment"], + {"traces": traces}, + ) diff --git a/prototypes/structured-delta/experiments/benchmark/build.py b/prototypes/structured-delta/experiments/benchmark/build.py new file mode 100644 index 0000000..826a136 --- /dev/null +++ b/prototypes/structured-delta/experiments/benchmark/build.py @@ -0,0 +1,968 @@ +"""48 authored micro-repositories. Extend with another add() call; no case cap. + +Gold and witness code live only in benchmark artifacts. Detectors do not import +this module. Cases are a development benchmark, NOT independently held-out data. +""" + +import argparse +import difflib +import json +import textwrap +from collections import Counter +from pathlib import Path + +CASES = [] + + +def clean(s): + return textwrap.dedent(s).lstrip("\n").rstrip() + "\n" + + +def add( + title, + label, + family, + base, + a, + b, + entity, + why, + check_a, + check_b, + check_base="", + witness=None, + online=False, +): + base = {p: clean(s) for p, s in base.items()} + + def version(changes): + out = dict(base) + for p, s in changes.items(): + if s is None: + out.pop(p, None) + else: + out[p] = clean(s) + return out + + aa, bb = version(a), version(b) + CASES.append( + dict( + id=f"case{len(CASES) + 1:03}", + title=title, + label=label, + family=family, + base=base, + a=aa, + b=bb, + entities=[] if entity is None else [entity], + explanation=why, + checks={ + "base": clean(check_base), + "a": clean(check_a), + "b": clean(check_b), + }, + witness=version(witness) + if witness is not None + else (aa if label == "redundant" else None), + online=online, + ) + ) + + +def cases(): + if CASES: + return CASES + # Independent: varied scopes, including same-file and shared dictionary roots. + add( + "Unrelated arithmetic modules", + "independent", + "unrelated_files", + {"left.py": "def inc(x): return x+1", "right.py": "def twice(x): return x*2"}, + {"left.py": "def inc(x): return x+2"}, + {"right.py": "def twice(x): return x*3"}, + None, + "Different modules have no calls or shared state.", + "from left import inc\nassert inc(1)==3", + "from right import twice\nassert twice(2)==6", + witness={ + "left.py": "def inc(x): return x+2", + "right.py": "def twice(x): return x*3", + }, + online=True, + ) + add( + "Separate functions in one file", + "independent", + "disjoint_symbols", + {"ops.py": "def left(): return 1\n\ndef right(): return 2"}, + {"ops.py": "def left(): return 10\n\ndef right(): return 2"}, + {"ops.py": "def left(): return 1\n\ndef right(): return 20"}, + None, + "No state or call edge between the functions.", + "from ops import left\nassert left()==10", + "from ops import right\nassert right()==20", + witness={"ops.py": "def left(): return 10\n\ndef right(): return 20"}, + ) + add( + "Separate classes", + "independent", + "disjoint_symbols", + { + "model.py": 'class Cat:\n def sound(self): return "mew"\n\nclass Dog:\n def sound(self): return "woof"' + }, + { + "model.py": 'class Cat:\n def sound(self): return "purr"\n\nclass Dog:\n def sound(self): return "woof"' + }, + { + "model.py": 'class Cat:\n def sound(self): return "mew"\n\nclass Dog:\n def sound(self): return "bark"' + }, + None, + "Neither class refers to the other.", + 'from model import Cat\nassert Cat().sound()=="purr"', + 'from model import Dog\nassert Dog().sound()=="bark"', + witness={ + "model.py": 'class Cat:\n def sound(self): return "purr"\n\nclass Dog:\n def sound(self): return "bark"' + }, + ) + add( + "Separate state keys", + "independent", + "field_sensitive", + { + "state.py": 'VALUES={"a":0,"b":0}\ndef set_a(): VALUES["a"]=1\ndef set_b(): VALUES["b"]=1' + }, + { + "state.py": 'VALUES={"a":0,"b":0}\ndef set_a(): VALUES["a"]=2\ndef set_b(): VALUES["b"]=1' + }, + { + "state.py": 'VALUES={"a":0,"b":0}\ndef set_a(): VALUES["a"]=1\ndef set_b(): VALUES["b"]=3' + }, + None, + "Distinct dictionary keys can be assigned independently.", + 'import state\nstate.set_a()\nassert state.VALUES["a"]==2', + 'import state\nstate.set_b()\nassert state.VALUES["b"]==3', + witness={ + "state.py": 'VALUES={"a":0,"b":0}\ndef set_a(): VALUES["a"]=2\ndef set_b(): VALUES["b"]=3' + }, + ) + add( + "Library import unrelated to formatter", + "independent", + "unrelated_files", + { + "calc.py": "def root(x): return x**0.5", + "fmt.py": "def format_name(x): return x", + }, + {"calc.py": "import math\ndef root(x): return math.sqrt(x)"}, + {"fmt.py": "def format_name(x): return x.upper()"}, + None, + "Import has no dependency on formatter.", + "from calc import root\nassert root(9)==3", + 'from fmt import format_name\nassert format_name("a")=="A"', + witness={ + "calc.py": "import math\ndef root(x): return math.sqrt(x)", + "fmt.py": "def format_name(x): return x.upper()", + }, + ) + add( + "Annotation and unrelated new endpoint", + "independent", + "unrelated_files", + {"a.py": "def f(x): return x", "b.py": "def g(): return 0"}, + {"a.py": "def f(x: int) -> int: return x"}, + {"b.py": "def g(): return 0\ndef h(): return 4"}, + None, + "Type annotation has no consumer in other patch.", + "from a import f\nassert f(3)==3", + "from b import h\nassert h()==4", + witness={ + "a.py": "def f(x: int) -> int: return x", + "b.py": "def g(): return 0\ndef h(): return 4", + }, + ) + add( + "Independent configuration keys", + "independent", + "disjoint_config", + {"config.py": 'TIMEOUT=10\nCOLOR="blue"'}, + {"config.py": 'TIMEOUT=20\nCOLOR="blue"'}, + {"config.py": 'TIMEOUT=10\nCOLOR="red"'}, + None, + "No consumer couples timeout and color.", + "from config import TIMEOUT\nassert TIMEOUT==20", + 'from config import COLOR\nassert COLOR=="red"', + witness={"config.py": 'TIMEOUT=20\nCOLOR="red"'}, + ) + add( + "Comment and behavior elsewhere", + "independent", + "nonsemantic", + {"a.py": "# identity\ndef f(x): return x", "b.py": "def g(): return 1"}, + {"a.py": "# Return input unchanged.\ndef f(x): return x"}, + {"b.py": "def g(): return 2"}, + None, + "Comment has no runtime effect.", + "from a import f\nassert f(2)==2", + "from b import g\nassert g()==2", + witness={ + "a.py": "# Return input unchanged.\ndef f(x): return x", + "b.py": "def g(): return 2", + }, + ) + add( + "Same spelling in different modules", + "independent", + "qualified_names", + {"a.py": "VERSION=1", "b.py": "VERSION=1"}, + {"a.py": "VERSION=1\ndef parse(x): return int(x)"}, + {"b.py": 'VERSION=1\ndef parse(x): return x.split(",")'}, + None, + "Qualified symbols differ despite identical names.", + 'from a import parse\nassert parse("2")==2', + 'from b import parse\nassert parse("a,b")==["a","b"]', + witness={ + "a.py": "VERSION=1\ndef parse(x): return int(x)", + "b.py": 'VERSION=1\ndef parse(x): return x.split(",")', + }, + ) + add( + "Delete unused file and edit another", + "independent", + "file_lifecycle", + {"old.py": "UNUSED=1", "live.py": "def value(): return 1"}, + {"old.py": None}, + {"live.py": "def value(): return 2"}, + None, + "Deleted file has no live imports.", + 'from pathlib import Path\nassert not Path("old.py").exists()', + "from live import value\nassert value()==2", + witness={"old.py": None, "live.py": "def value(): return 2"}, + ) + # Compatible: shared scope/edge, both requirements can coexist. + parse = {"parser.py": "import logging\ndef parse(x):\n return int(x)"} + validate = 'import logging\ndef parse(x):\n if x is None: raise ValueError("empty")\n return int(x)' + log = 'import logging\ndef parse(x):\n logging.info("parse")\n return int(x)' + ca = 'from parser import parse\ntry: parse(None)\nexcept ValueError: pass\nelse: raise AssertionError("validation missing")' + cb = 'from unittest.mock import patch\nfrom parser import parse\nwith patch("parser.logging.info") as log:\n assert parse("2")==2\n log.assert_called_once_with("parse")' + add( + "Validation plus logging", + "compatible", + "same_function_additive", + parse, + {"parser.py": validate}, + {"parser.py": log}, + "parser.py::parse", + "Input validation and successful-call logging can both be preserved.", + ca, + cb, + witness={ + "parser.py": 'import logging\ndef parse(x):\n if x is None: raise ValueError("empty")\n logging.info("parse")\n return int(x)' + }, + online=True, + ) + add( + "Overlapping one-line replacements", + "compatible", + "overlapping_lines", + {"parser.py": "import logging\ndef parse(x): return int(x)"}, + {"parser.py": validate}, + {"parser.py": log}, + "parser.py::parse", + "Both replace the same physical line but request independent added effects.", + ca, + cb, + witness={ + "parser.py": 'import logging\ndef parse(x):\n if x is None: raise ValueError("empty")\n logging.info("parse")\n return int(x)' + }, + ) + add( + "Docstring and implementation", + "compatible", + "same_function_additive", + {"ops.py": "def double(x):\n return x*2"}, + {"ops.py": 'def double(x):\n """Multiply by two."""\n return x*2'}, + {"ops.py": "def double(x):\n return x+x"}, + "ops.py::double", + "Documentation and equivalent arithmetic coexist.", + 'from ops import double\nassert double.__doc__=="Multiply by two."', + "from ops import double\nassert double(4)==8", + witness={ + "ops.py": 'def double(x):\n """Multiply by two."""\n return x+x' + }, + ) + add( + "Optional argument and logging", + "compatible", + "signature_compatible", + {"ops.py": "import logging\ndef scale(x):\n return x*2"}, + {"ops.py": "import logging\ndef scale(x, factor=2):\n return x*factor"}, + { + "ops.py": 'import logging\ndef scale(x):\n logging.info("scale")\n return x*2' + }, + "ops.py::scale", + "Default preserves old calls while logging adds an effect.", + "from ops import scale\nassert scale(2,3)==6", + 'from unittest.mock import patch\nfrom ops import scale\nwith patch("ops.logging.info") as p:\n assert scale(2)==4\n p.assert_called_once()', + witness={ + "ops.py": 'import logging\ndef scale(x, factor=2):\n logging.info("scale")\n return x*factor' + }, + ) + add( + "Producer optional parameter and old-shape caller", + "compatible", + "cross_file_compatible", + { + "api.py": "def get(x): return x", + "client.py": "from api import get\ndef run(): return get(1)", + }, + {"api.py": "def get(x, extra=0): return x+extra"}, + {"client.py": "from api import get\ndef run(): return get(2)"}, + "api.py::get", + "New parameter is optional; existing arity remains valid.", + "from api import get\nassert get(2,3)==5", + "from client import run\nassert run()==2", + witness={ + "api.py": "def get(x, extra=0): return x+extra", + "client.py": "from api import get\ndef run(): return get(2)", + }, + online=True, + ) + add( + "Add result key with tolerant consumer", + "compatible", + "contract_compatible", + { + "api.py": 'def get(): return {"id":1}', + "client.py": 'from api import get\ndef run(): return get()["id"]', + }, + {"api.py": 'def get(): return {"id":1,"name":"A"}'}, + {"client.py": 'from api import get\ndef run(): return get()["id"]+1'}, + "api.py::get", + "Consumer accesses retained key; added key is backward compatible.", + 'from api import get\nassert get()["name"]=="A"', + "from client import run\nassert run()==2", + witness={ + "api.py": 'def get(): return {"id":1,"name":"A"}', + "client.py": 'from api import get\ndef run(): return get()["id"]+1', + }, + online=True, + ) + add( + "Config change with tolerant consumer", + "compatible", + "config_compatible", + { + "config.py": "LIMIT=10", + "client.py": "from config import LIMIT\ndef valid(x): return x10: raise ValueError("large")\n return x*x' + }, + "ops.py::square", + "Disjoint rejected domains preserve the common valid domain.", + "from ops import square\ntry: square(-1)\nexcept ValueError: pass\nelse: raise AssertionError()", + "from ops import square\ntry: square(11)\nexcept ValueError: pass\nelse: raise AssertionError()", + witness={ + "ops.py": 'def square(x):\n if x<0: raise ValueError("negative")\n if x>10: raise ValueError("large")\n return x*x' + }, + ) + add( + "Equivalent producer optimization and new consumer", + "compatible", + "dependency_compatible", + { + "api.py": "def double(x: int): return x*2", + "client.py": "from api import double\ndef run(): return double(1)", + }, + {"api.py": "def double(x: int): return x+x"}, + {"client.py": "from api import double\ndef run(): return double(3)"}, + "api.py::double", + "Integer-domain optimization preserves the called behavior.", + "from api import double\nassert double(4)==8", + "from client import run\nassert run()==6", + witness={ + "api.py": "def double(x: int): return x+x", + "client.py": "from api import double\ndef run(): return double(3)", + }, + ) + add( + "Type annotation plus runtime guard", + "compatible", + "annotation_compatible", + {"ops.py": "def length(x):\n return len(x)"}, + {"ops.py": "def length(x: str) -> int:\n return len(x)"}, + { + "ops.py": 'def length(x):\n if x is None: raise ValueError("empty")\n return len(x)' + }, + "ops.py::length", + "Annotation and guard impose no contradictory runtime result.", + 'from ops import length\nassert length.__annotations__["x"] is str', + "from ops import length\ntry: length(None)\nexcept ValueError: pass\nelse: raise AssertionError()", + witness={ + "ops.py": 'def length(x: str) -> int:\n if x is None: raise ValueError("empty")\n return len(x)' + }, + ) + add( + "Compatible dependency patch release", + "compatible", + "dependency_compatible", + { + "requirements.txt": "tiny==1.0", + "tiny.py": "def encode(x): return str(x)", + "client.py": "import tiny\ndef run(): return tiny.encode(1)", + }, + {"requirements.txt": "tiny==1.1"}, + {"client.py": "import tiny\ndef run(): return tiny.encode(2)"}, + "dependency:tiny", + "Local package API remains unchanged; pinned version metadata alone is not breakage.", + 'from pathlib import Path\nassert "1.1" in Path("requirements.txt").read_text()', + 'from client import run\nassert run()=="2"', + witness={ + "requirements.txt": "tiny==1.1", + "client.py": "import tiny\ndef run(): return tiny.encode(2)", + }, + ) + add( + "New keyword-only optional parameter", + "compatible", + "signature_compatible", + { + "api.py": "def get(x): return x", + "client.py": "from api import get\ndef run(): return get(x=1)", + }, + {"api.py": "def get(x, *, trace=False): return x"}, + {"client.py": "from api import get\ndef run(): return get(x=2)"}, + "api.py::get", + "Caller uses an accepted keyword and does not need optional trace.", + "from api import get\nassert get(3,trace=True)==3", + "from client import run\nassert run()==2", + witness={ + "api.py": "def get(x, *, trace=False): return x", + "client.py": "from api import get\ndef run(): return get(x=2)", + }, + ) + # Redundancy: some canonicalized, some intentionally outside the rule subset. + add( + "Equivalent None identity guards", + "redundant", + "null_equivalence", + {"ops.py": "def value(x):\n return x"}, + {"ops.py": "def value(x):\n if x is None: return 0\n return x"}, + {"ops.py": "def value(x):\n if None is x: return 0\n return x"}, + "ops.py::value", + "Identity comparison operand reversal is equivalent.", + "from ops import value\nassert value(None)==0 and value(2)==2", + "from ops import value\nassert value(None)==0 and value(3)==3", + online=True, + ) + add( + "Same logging edit", + "redundant", + "exact_duplicate", + parse, + {"parser.py": log}, + {"parser.py": log}, + "parser.py::parse", + "Both patches add exactly the same single logging call.", + cb, + cb, + ) + add( + "Same default with different formatting", + "redundant", + "ast_equivalence", + {"ops.py": "def f(x): return x"}, + {"ops.py": "def f(x=1): return x"}, + {"ops.py": "def f( x = 1 ):\n return x"}, + "ops.py::f", + "Whitespace differs, signatures and bodies are identical.", + "from ops import f\nassert f()==1", + "from ops import f\nassert f()==1", + ) + add( + "Equivalent conditional branches", + "redundant", + "null_equivalence", + {"ops.py": "def f(x): return x"}, + { + "ops.py": "def f(x):\n if x is None:\n return 0\n else:\n return x" + }, + { + "ops.py": "def f(x):\n if x is not None:\n return x\n else:\n return 0" + }, + "ops.py::f", + "Inverting the identity test and swapping return arms is equivalent.", + "from ops import f\nassert f(None)==0 and f(4)==4", + "from ops import f\nassert f(None)==0 and f(7)==7", + ) + add( + "Equivalent integer addition", + "redundant", + "algebraic_equivalence", + {"ops.py": "def f(x: int): return x"}, + {"ops.py": "def f(x: int): return x+1"}, + {"ops.py": "def f(x: int): return 1+x"}, + "ops.py::f", + "On the declared plain-integer domain addition commutes.", + "from ops import f\nassert all(f(i)==i+1 for i in range(-10,11))", + "from ops import f\nassert f(8)==9", + ) + add( + "Equivalent local variable rename", + "redundant", + "alpha_equivalence", + {"ops.py": "def f(x): return x"}, + {"ops.py": "def f(x):\n result=x*2\n return result"}, + {"ops.py": "def f(x):\n answer=x*2\n return answer"}, + "ops.py::f", + "Only the local temporary name differs.", + "from ops import f\nassert f(3)==6", + "from ops import f\nassert f(4)==8", + ) + add( + "Equivalent string format", + "redundant", + "format_equivalence", + {"ops.py": "def greet(x: str): return x"}, + {"ops.py": 'def greet(x: str): return f"Hello {x}"'}, + {"ops.py": 'def greet(x: str): return "Hello {}".format(x)'}, + "ops.py::greet", + "For string inputs the two formatting forms have identical output.", + 'from ops import greet\nassert greet("A")=="Hello A"', + 'from ops import greet\nassert greet("B")=="Hello B"', + ) + add( + "Duplicate dictionary field addition", + "redundant", + "exact_duplicate", + {"api.py": 'def data(): return {"id":1}'}, + {"api.py": 'def data(): return {"id":1,"ok":True}'}, + {"api.py": 'def data(): return {"id":1,"ok":True}'}, + "api.py::data", + "Both produce the same extended return contract.", + 'from api import data\nassert data()["ok"] is True', + 'from api import data\nassert data()=={"id":1,"ok":True}', + ) + # Conflicting: 16 cases, cross-file dependencies plus expected hard failures. + add( + "Delete class versus extend class", + "conflicting", + "delete_extend", + {"model.py": "class Foo:\n def value(self): return 1"}, + {"model.py": "# Foo removed"}, + { + "model.py": "class Foo:\n def value(self): return 1\n def extra(self): return 2" + }, + "model.py::Foo", + "A requires Foo absent while B requires its new method.", + 'import model\nassert not hasattr(model,"Foo")', + "from model import Foo\nassert Foo().extra()==2", + online=True, + ) + add( + "API rename versus new old-name consumer", + "conflicting", + "rename_use", + {"api.py": "def fetch(): return 1", "client.py": "VALUE=1"}, + {"api.py": "def load(): return 1"}, + {"client.py": "from api import fetch\ndef run(): return fetch()+1"}, + "api.py::fetch", + "B imports an API removed by A.", + 'import api\nassert api.load()==1 and not hasattr(api,"fetch")', + "from client import run\nassert run()==2", + online=True, + ) + add( + "Different functions share mutable scalar", + "conflicting", + "shared_state", + { + "state.py": "COUNT=0\ndef reset():\n global COUNT\n COUNT=0\ndef step():\n global COUNT\n COUNT+=1\n return COUNT" + }, + { + "state.py": "COUNT=0\ndef reset():\n global COUNT\n COUNT=10\ndef step():\n global COUNT\n COUNT+=1\n return COUNT" + }, + { + "state.py": "COUNT=0\ndef reset():\n global COUNT\n COUNT=0\ndef step():\n global COUNT\n assert COUNT<5\n COUNT+=1\n return COUNT" + }, + "state.py::COUNT", + "Reset changes state range; B assumes reset establishes a count below five.", + "import state\nstate.reset()\nassert state.COUNT==10", + "import state\nstate.reset()\nassert state.step()==1", + online=True, + ) + add( + "Cross-file configuration unit", + "conflicting", + "shared_config", + { + "config.py": "TIMEOUT=5", + "client.py": "from config import TIMEOUT\ndef seconds(): return TIMEOUT", + }, + {"config.py": "TIMEOUT=5000"}, + {"client.py": "from config import TIMEOUT\ndef seconds(): return TIMEOUT*2"}, + "config.py::TIMEOUT", + "A stores milliseconds while B continues using seconds.", + "from config import TIMEOUT\nassert TIMEOUT==5000", + "from client import seconds\nassert seconds()==10", + online=True, + ) + add( + "Producer scalar to mapping", + "conflicting", + "return_contract", + { + "api.py": "def value(): return 3", + "client.py": "from api import value\ndef run(): return value()", + }, + {"api.py": 'def value(): return {"value":3}'}, + {"client.py": "from api import value\ndef run(): return value()+1"}, + "api.py::value", + "Numeric consumer cannot add one to mapping return.", + 'from api import value\nassert value()=={"value":3}', + "from client import run\nassert run()==4", + online=True, + ) + add( + "Producer key rename", + "conflicting", + "return_contract", + { + "api.py": 'def user(): return {"name":"A"}', + "client.py": 'from api import user\ndef label(): return user()["name"]', + }, + {"api.py": 'def user(): return {"display":"A"}'}, + { + "client.py": 'from api import user\ndef label(): return user()["name"].upper()' + }, + "api.py::user", + "Old key is absent in the new return mapping.", + 'from api import user\nassert "name" not in user() and user()["display"]=="A"', + 'from client import label\nassert label()=="A"', + ) + add( + "New required positional argument", + "conflicting", + "signature_break", + { + "api.py": "def scale(x): return x*2", + "client.py": "from api import scale\ndef run(): return scale(1)", + }, + {"api.py": "def scale(x, factor): return x*factor"}, + {"client.py": "from api import scale\ndef run(): return scale(3)"}, + "api.py::scale", + "The new consumer call lacks the newly required factor.", + "from api import scale\nassert scale(3,4)==12", + "from client import run\nassert run()==6", + ) + add( + "Rename keyword argument", + "conflicting", + "signature_break", + { + "api.py": "def get(name): return name", + "client.py": 'from api import get\ndef run(): return get(name="A")', + }, + {"api.py": "def get(key): return key"}, + {"client.py": 'from api import get\ndef run(): return get(name="B")'}, + "api.py::get", + "B passes a keyword no longer accepted by the producer.", + 'from api import get\nassert get(key="C")=="C"', + 'from client import run\nassert run()=="B"', + ) + add( + "Dependency removal invalidates import", + "conflicting", + "dependency_removal", + { + "requirements.txt": "tiny==1.0", + "tiny.py": "def encode(x): return str(x)", + "client.py": "VALUE=1", + }, + {"requirements.txt": "# no dependencies", "tiny.py": None}, + {"client.py": "import tiny\ndef run(): return tiny.encode(2)"}, + "tiny.py::encode", + "Bundled deterministic dependency fixture is removed; B adds a live import.", + 'from pathlib import Path\nassert not Path("tiny.py").exists()', + 'from client import run\nassert run()=="2"', + ) + add( + "Dependency upgrade changes local API", + "conflicting", + "dependency_upgrade", + { + "requirements.txt": "codec==1.0", + "codec.py": "def encode(x): return str(x)", + "client.py": "import codec\ndef run(): return codec.encode(1)", + }, + { + "requirements.txt": "codec==2.0", + "codec.py": "def encode(x): return str(x).encode()", + }, + {"client.py": 'import codec\ndef run(): return "value:"+codec.encode(2)'}, + "codec.py::encode", + "Vendored dependency fixture now returns bytes, while consumer concatenates str.", + 'import codec\nassert codec.encode(1)==b"1"', + 'from client import run\nassert run()=="value:2"', + ) + add( + "Different replacement values", + "conflicting", + "write_write", + {"config.py": 'MODE="base"'}, + {"config.py": 'MODE="fast"'}, + {"config.py": 'MODE="safe"'}, + "config.py::MODE", + "One scalar cannot equal both required values.", + 'from config import MODE\nassert MODE=="fast"', + 'from config import MODE\nassert MODE=="safe"', + ) + add( + "Incompatible return semantics", + "conflicting", + "return_value", + {"ops.py": "def fee(x): return x"}, + {"ops.py": "def fee(x): return x*2"}, + {"ops.py": "def fee(x): return x+2"}, + "ops.py::fee", + "At x=3 the two required outputs are six and five.", + "from ops import fee\nassert fee(3)==6", + "from ops import fee\nassert fee(3)==5", + ) + add( + "Dynamic dispatch after rename", + "conflicting", + "dynamic_dispatch", + { + "api.py": "def fetch(): return 1", + "client.py": 'import api\ndef run(): return getattr(api,"fetch")()', + }, + {"api.py": "def load(): return 1"}, + {"client.py": 'import api\ndef run(): return getattr(api,"fetch")()+1'}, + "api.py::fetch", + "String-based dynamic lookup still asks for removed API.", + 'import api\nassert api.load()==1 and not hasattr(api,"fetch")', + "from client import run\nassert run()==2", + online=True, + ) + add( + "Environment variable contract", + "conflicting", + "dynamic_configuration", + { + "producer.py": 'import os\ndef setup(): os.environ["APP_MODE"]="text"', + "client.py": 'import os\ndef run(): return os.environ.get("APP_MODE")', + }, + {"producer.py": 'import os\ndef setup(): os.environ["APP_MODE"]="binary"'}, + { + "client.py": 'import os\ndef run():\n assert os.environ.get("APP_MODE")=="text"\n return "ok"' + }, + "environment:APP_MODE", + "Producer changes environment protocol consumed in a different module.", + 'import os\nfrom producer import setup\nsetup()\nassert os.environ["APP_MODE"]=="binary"', + 'from producer import setup\nfrom client import run\nsetup()\nassert run()=="ok"', + ) + add( + "JSON configuration consumer", + "conflicting", + "dynamic_configuration", + { + "config.json": '{"mode":"text"}', + "client.py": 'import json\ndef read():\n return json.load(open("config.json"))["mode"]', + }, + {"config.json": '{"mode":"binary"}'}, + { + "client.py": 'import json\ndef read():\n mode=json.load(open("config.json"))["mode"]\n assert mode=="text"\n return mode.upper()' + }, + "config.json::mode", + "File-based dynamic dependency is not an import edge.", + 'import json\nassert json.load(open("config.json"))["mode"]=="binary"', + 'from client import read\nassert read()=="TEXT"', + ) + add( + "Shared string state protocol", + "conflicting", + "shared_state", + { + "state.py": 'MODE="text"\ndef setup():\n global MODE\n MODE="text"\ndef run(): return MODE' + }, + { + "state.py": 'MODE="text"\ndef setup():\n global MODE\n MODE="binary"\ndef run(): return MODE' + }, + { + "state.py": 'MODE="text"\ndef setup():\n global MODE\n MODE="text"\ndef run():\n assert MODE=="text"\n return MODE.upper()' + }, + "state.py::MODE", + "Changed writer violates another function's new state precondition.", + 'import state\nstate.setup()\nassert state.MODE=="binary"', + 'import state\nstate.setup()\nassert state.run()=="TEXT"', + ) + return CASES + + +def patch(base, target): + return "".join( + "".join( + difflib.unified_diff( + base.get(p, "").splitlines(True), + target.get(p, "").splitlines(True), + fromfile="a/" + p if p in base else "/dev/null", + tofile="b/" + p if p in target else "/dev/null", + ) + ) + for p in sorted(base.keys() | target.keys()) + if base.get(p) != target.get(p) + ) + + +def write_state(root, state): + for p, source in state.items(): + dest = root / p + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(source) + + +def build(root): + root.mkdir(parents=True, exist_ok=True) + for c in cases(): + path = root / c["id"] + if path.exists(): + raise FileExistsError(f"Refusing to overwrite benchmark: {path}") + path.mkdir() + for key in ("base", "a", "b"): + write_state(path / key, c[key]) + for key in ("a", "b"): + (path / f"patch_{key.upper()}.diff").write_text(patch(c["base"], c[key])) + (path / "checks").mkdir() + for key, source in c["checks"].items(): + (path / "checks" / f"{key}.py").write_text(source) + if c["witness"] is not None: + write_state(path / "witness", c["witness"]) + metadata = { + k: v + for k, v in c.items() + if k not in {"base", "a", "b", "checks", "witness"} + } + metadata["cross_file"] = not bool( + { + p + for p in c["base"].keys() | c["a"].keys() + if c["base"].get(p) != c["a"].get(p) + } + & { + p + for p in c["base"].keys() | c["b"].keys() + if c["base"].get(p) != c["b"].get(p) + } + ) + metadata["witness_available"] = c["witness"] is not None + (path / "gold.json").write_text( + json.dumps(metadata, ensure_ascii=False, indent=2) + "\n" + ) + if c["online"]: + # Three real source snapshots per branch: harmless helper, first changed + # file, final state. Helpers remain until final cleanup. Steps + # are syntactic and deterministic, not synthesized from gold labels. + from ..online.sequences import stages, functional_complete + + events = [] + for index, (sa, sb) in enumerate( + zip(stages(c["base"], c["a"], "a"), stages(c["base"], c["b"], "b")), 1 + ): + for branch, snapshot in (("a", sa), ("b", sb)): + write_state(path / "online" / f"{branch}{index}", snapshot) + prev = ( + c["base"] + if index == 1 + else (stages(c["base"], c[branch], branch)[index - 2]) + ) + dest = path / "online" / f"{branch}{index}.diff" + dest.write_text(patch(prev, snapshot)) + events.append( + { + "branch": branch, + "stage": index, + "state": f"{branch}{index}", + "patch": f"{branch}{index}.diff", + "requirements_active": functional_complete( + snapshot, c[branch] + ), + } + ) + (path / "online" / "schedule.json").write_text( + json.dumps(events, indent=2) + "\n" + ) + summary = { + "cases": len(CASES), + "labels": dict(Counter(c["label"] for c in CASES)), + "families": dict(Counter(c["family"] for c in CASES)), + "online_cases": sum(c["online"] for c in CASES), + } + (root / "statistics.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--out", type=Path, default=Path("experiments/benchmark/cases")) + build(parser.parse_args().out) diff --git a/prototypes/structured-delta/experiments/delta/__init__.py b/prototypes/structured-delta/experiments/delta/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prototypes/structured-delta/experiments/delta/extractor.py b/prototypes/structured-delta/experiments/delta/extractor.py new file mode 100644 index 0000000..dd87e46 --- /dev/null +++ b/prototypes/structured-delta/experiments/delta/extractor.py @@ -0,0 +1,449 @@ +"""Python AST + simple requirements/JSON analysis; no tests, labels or intent input. + +This is deliberately bounded analysis, not an equivalence prover or type checker. +""" + +import ast +import copy +import difflib +import json +from pathlib import PurePosixPath +from .model import ProgramDelta, Symbol + + +def dump(node): + return ast.dump(node, include_attributes=False) if node is not None else "" + + +class Canonical(ast.NodeTransformer): + def visit_Compare(self, node): + node = self.generic_visit(node) + if len(node.ops) == 1 and isinstance(node.ops[0], (ast.Is, ast.IsNot)): + if isinstance(node.left, ast.Constant) and node.left.value is None: + node.left, node.comparators[0] = node.comparators[0], node.left + return node + + def visit_If(self, node): + node = self.generic_visit(node) + # Invert only an explicit identity test, preserving Python == semantics. + if ( + len(node.body) == len(node.orelse) == 1 + and isinstance(node.body[0], ast.Return) + and isinstance(node.orelse[0], ast.Return) + and isinstance(node.test, ast.Compare) + and len(node.test.ops) == 1 + and isinstance(node.test.ops[0], ast.IsNot) + ): + node.test.ops[0] = ast.Is() + node.body, node.orelse = node.orelse, node.body + return node + + +def canonical(node): + return dump(Canonical().visit(copy.deepcopy(node))) + + +def module_path(name, files, current="", level=0): + if level: + parts = list(PurePosixPath(current).parent.parts) + parts = parts[: len(parts) - level + 1] + name = ".".join(parts + ([name] if name else [])) + path = name.replace(".", "/") + for candidate in (path + ".py", path + "/__init__.py"): + if candidate in files: + return candidate + return None + + +def literal_type(n): + if n is None: + return "None" + if isinstance(n, ast.Constant): + return type(n.value).__name__ + if isinstance(n, ast.Dict): + keys = [ + k.value + for k in n.keys + if isinstance(k, ast.Constant) and isinstance(k.value, str) + ] + return "dict:" + ",".join(sorted(keys)) + if isinstance(n, (ast.List, ast.ListComp)): + return "list" + if isinstance(n, ast.Tuple): + return "tuple:" + str(len(n.elts)) + if ( + isinstance(n, ast.Call) + and isinstance(n.func, ast.Name) + and n.func.id in {"str", "int", "float", "dict", "list", "bool", "set"} + ): + return n.func.id + return "unknown" + + +def state(snapshot): + symbols, imports, unresolved = {}, {}, [] + for path, source in sorted(snapshot.items()): + if not path.endswith(".py"): + continue + try: + tree = ast.parse(source, filename=path) + except SyntaxError as exc: + unresolved.append(f"{path}:{exc.lineno}: {exc.msg}") + continue + aliases, module_imports = {}, set() + globals_ = { + n.id + for stmt in tree.body + if isinstance(stmt, (ast.Assign, ast.AnnAssign)) + for n in ast.walk(stmt) + if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store) + } + defs = { + stmt.name + for stmt in tree.body + if isinstance(stmt, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)) + } + # Module-local aliases, including relative and package imports. + for n in ast.walk(tree): + if isinstance(n, ast.ImportFrom): + target = module_path(n.module or "", snapshot, path, n.level) + for alias in n.names: + entity = ( + f"{target}::{alias.name}" + if target + else f"dependency:{n.module}" + ) + aliases[alias.asname or alias.name] = entity + module_imports.add(entity) + elif isinstance(n, ast.Import): + for alias in n.names: + target = module_path(alias.name, snapshot) + aliases[alias.asname or alias.name.split(".")[0]] = ( + f"{target}::" + if target + else f"dependency:{alias.name.split('.')[0]}" + ) + module_imports.add( + aliases[alias.asname or alias.name.split(".")[0]] + ) + imports[path] = sorted(module_imports) + + def resolve(n, locals_): + if isinstance(n, ast.Name): + if n.id in locals_: + return None + if n.id in aliases: + return aliases[n.id] + if n.id in globals_ or n.id in defs: + return f"{path}::{n.id}" + if isinstance(n, ast.Attribute): + parent = resolve(n.value, locals_) + if parent: + return parent + ("" if parent.endswith("::") else ".") + n.attr + if isinstance(n, ast.Subscript): + parent = resolve(n.value, locals_) + if parent and isinstance(n.slice, ast.Constant): + return parent + "[" + repr(n.slice.value) + "]" + return None + + def collect(n, qual): + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + entity = f"{path}::{qual}{n.name}" + is_func = not isinstance(n, ast.ClassDef) + sig, returns, rexpr, reads, writes, calls = {}, [], [], set(), {}, [] + local_names = set() + if is_func: + args = n.args + declared_global = { + name + for k in ast.walk(n) + if isinstance(k, ast.Global) + for name in k.names + } + local_names = { + k.id + for k in ast.walk(n) + if isinstance(k, ast.Name) and isinstance(k.ctx, ast.Store) + } - declared_global + local_names |= { + k.arg for k in ast.walk(args) if isinstance(k, ast.arg) + } + positional = args.posonlyargs + args.args + sig = { + "parameters": [a.arg for a in positional], + "required": len(positional) - len(args.defaults), + "maximum": None if args.vararg else len(positional), + "posonly": len(args.posonlyargs), + "kwonly": [a.arg for a in args.kwonlyargs], + "required_kwonly": [ + a.arg + for a, d in zip(args.kwonlyargs, args.kw_defaults) + if d is None + ], + "kwargs": bool(args.kwarg), + "annotations": [ + ast.unparse(a.annotation) if a.annotation else "" + for a in positional + ], + "return": ast.unparse(n.returns) if n.returns else "", + } + parents = { + child: parent + for parent in ast.walk(n) + for child in ast.iter_child_nodes(parent) + } + for k in ast.walk(n): + if isinstance(k, ast.Return): + returns.append(literal_type(k.value)) + rexpr.append(canonical(k.value)) + if isinstance( + k, (ast.Name, ast.Attribute, ast.Subscript) + ) and isinstance(k.ctx, ast.Load): + ref = resolve(k, local_names) + parent = parents.get(k) + # Attribute/subscript access uses its most specific resolved entity. + # Avoid inventing interference between distinct constant keys. + covered = isinstance( + parent, (ast.Attribute, ast.Subscript) + ) and resolve(parent, local_names) + if ref and not covered: + reads.add(ref) + if isinstance(k, (ast.Assign, ast.AnnAssign, ast.AugAssign)): + targets = ( + k.targets if isinstance(k, ast.Assign) else [k.target] + ) + for t in targets: + ref = resolve(t, local_names) + if ref: + writes.setdefault(ref, []).append(canonical(k)) + if isinstance(k, ast.Call): + ref = resolve(k.func, local_names) + if ref: + calls.append( + { + "target": ref, + "argc": len(k.args), + "keywords": [v.arg for v in k.keywords], + "dynamic": any( + isinstance(a, ast.Starred) for a in k.args + ) + or any(v.arg is None for v in k.keywords), + "source": entity, + } + ) + # Common collection mutation methods are writes to the collection. + if isinstance( + k.func, ast.Attribute + ) and k.func.attr in { + "append", + "extend", + "update", + "pop", + "clear", + "add", + "remove", + "setdefault", + }: + parent = resolve(k.func.value, local_names) + if parent: + writes.setdefault(parent, []).append( + canonical(k) + ) + if isinstance(k.func, ast.Name) and k.func.id in { + "getattr", + "eval", + "exec", + "globals", + }: + unresolved.append(f"{entity}: dynamic call {k.func.id}") + symbols[entity] = Symbol( + entity, + "function" if is_func else "class", + n.lineno, + n.end_lineno, + canonical(n), + sig, + sorted(set(returns)), + rexpr, + sorted(reads), + writes, + calls, + [canonical(s) for s in n.body], + ) + for child in n.body: + if isinstance( + child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ): + collect(child, qual + n.name + ".") + elif isinstance(n, (ast.Assign, ast.AnnAssign)) and not qual: + for t in n.targets if isinstance(n, ast.Assign) else [n.target]: + if isinstance(t, ast.Name): + entity = f"{path}::{t.id}" + symbols[entity] = Symbol( + entity, + "state", + n.lineno, + n.end_lineno, + canonical(n), + returns=[literal_type(n.value)], + value=canonical(n.value), + ) + + for node in tree.body: + collect(node, "") + for path, source in snapshot.items(): + if path == "requirements.txt": + for row, line in enumerate(source.splitlines(), 1): + if line.strip() and not line.startswith("#"): + import re + + name = re.split(r"[<>=!~\[; ]", line.strip())[0].replace("-", "_") + key = "dependency:" + name + symbols[key] = Symbol( + key, "dependency", row, row, line.strip(), value=line.strip() + ) + elif path.endswith(".json"): + try: + data = json.loads(source) + if isinstance(data, dict): + for key, value in data.items(): + entity = f"{path}::{key}" + symbols[entity] = Symbol( + entity, + "config", + 1, + len(source.splitlines()), + json.dumps(value, sort_keys=True), + value=repr(value), + ) + except ValueError: + unresolved.append(f"{path}: invalid JSON") + return symbols, imports, unresolved + + +def extract(base, changed): + old, oi, ou = state(base) + new, ni, nu = state(changed) + files = sorted( + p for p in base.keys() | changed.keys() if base.get(p) != changed.get(p) + ) + affected = sorted( + k + for k in old.keys() | new.keys() + if k not in old or k not in new or old[k].canonical != new[k].canonical + ) + # Canonical equivalence can hide a substantive syntactic edit (e.g. reversed None identity). + raw_old, raw_new = {}, {} + for p in files: + for index, snapshot in ((raw_old, base), (raw_new, changed)): + try: + tree = ast.parse(snapshot.get(p, "")) if p.endswith(".py") else None + if tree: + + def walk(ns, prefix=""): + for n in ns: + if isinstance( + n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ): + index[f"{p}::{prefix}{n.name}"] = dump(n) + walk(n.body, prefix + n.name + ".") + + walk(tree.body) + except SyntaxError: + pass + affected = sorted( + set(affected) + | {k for k in raw_old.keys() & raw_new.keys() if raw_old[k] != raw_new[k]} + ) + operations, ast_ops, contracts, reads, writes, calls = [], [], {}, set(), {}, [] + for k in affected: + a, b = old.get(k), new.get(k) + kind = "create" if a is None else "delete" if b is None else "modify" + operations.append({"entity": k, "operation": kind}) + for s in (a, b): + if s: + reads.update(s.reads) + calls.extend(s.calls) + # Only actual changed write effects, not every pre-existing write in a changed function. + aw, bw = a.writes if a else {}, b.writes if b else {} + for ref in aw.keys() | bw.keys(): + if aw.get(ref) != bw.get(ref): + writes.setdefault(ref, []).extend(bw.get(ref, [""])) + if (b or a).kind in {"state", "config", "dependency"}: + writes[k] = [b.value if b else ""] + if a and b: + fields = {} + if a.signature != b.signature: + fields["signature"] = {"before": a.signature, "after": b.signature} + if a.returns != b.returns: + fields["return_type"] = {"before": a.returns, "after": b.returns} + if a.value != b.value: + fields["value"] = {"before": a.value, "after": b.value} + if fields: + contracts[k] = fields + matcher = difflib.SequenceMatcher( + a=a.statements, b=b.statements, autojunk=False + ) + for op, i, j, x, y in matcher.get_opcodes(): + if op != "equal": + ast_ops.append( + { + "entity": k, + "operation": op, + "before": a.statements[i:j], + "after": b.statements[x:y], + } + ) + # Exact-body rename candidates (names excluded), retained as evidence, not guessed by spelling. + for op in list(operations): + if op["operation"] != "delete": + continue + a = old[op["entity"]] + matches = [ + p["entity"] + for p in operations + if p["operation"] == "create" + and new[p["entity"]].kind == a.kind + and a.statements + and a.statements == new[p["entity"]].statements + and a.signature == new[p["entity"]].signature + ] + if len(matches) == 1: + op["operation"] = "rename" + op["to"] = matches[0] + lines = {} + for p in files: + touched = set() + for op, i, j, x, y in difflib.SequenceMatcher( + a=base.get(p, "").splitlines(), + b=changed.get(p, "").splitlines(), + autojunk=False, + ).get_opcodes(): + if op != "equal": + touched.update(range(i + 1, j + 1) if j > i else [i + 0.5]) + lines[p] = sorted(touched) + imports_old = {v for p in files for v in oi.get(p, [])} + imports_new = {v for p in files for v in ni.get(p, [])} + deps_old = {k for k in old if k.startswith("dependency:")} | imports_old + deps_new = {k for k in new if k.startswith("dependency:")} | imports_new + return ProgramDelta( + files, + affected, + operations, + sorted(reads), + writes, + {k: old[k].signature for k in affected if k in old}, + {k: new[k].signature for k in affected if k in new}, + {k: old[k].returns for k in affected if k in old}, + {k: new[k].returns for k in affected if k in new}, + sorted(deps_new - deps_old), + sorted(deps_old - deps_new), + contracts, + sorted(imports_new), + calls, + ast_ops, + lines, + {k: old[k] for k in affected if k in old}, + {k: new[k] for k in affected if k in new}, + sorted(set(ou + nu)), + ) diff --git a/prototypes/structured-delta/experiments/delta/interaction.py b/prototypes/structured-delta/experiments/delta/interaction.py new file mode 100644 index 0000000..1fbc65c --- /dev/null +++ b/prototypes/structured-delta/experiments/delta/interaction.py @@ -0,0 +1,190 @@ +"""Frozen rule ladder. Overlap establishes scope; concrete effects decide conflict. + +Scores are ordinal rule strengths, never calibrated probabilities. +""" + +from .model import Prediction + +LEVELS = ( + "File", + "File + Symbol", + "File + Symbol + Dependency", + "File + Symbol + Dependency + Contract", + "Full Structured Delta", +) + + +def related(a, b): + return ( + a == b + or a.startswith(b + ".") + or b.startswith(a + ".") + or a.startswith(b + "[") + or b.startswith(a + "[") + ) + + +def detect(a, b, level="Full Structured Delta"): + depth = LEVELS.index(level) + files = sorted(set(a.files) & set(b.files)) + scope = sorted({x for x in a.symbols for y in b.symbols if related(x, y)}) + deps = sorted( + {x for x in a.symbols for y in b.reads + b.imports if related(x, y)} + | {x for x in b.symbols for y in a.reads + a.imports if related(x, y)} + ) + interference = sorted( + {x for x in a.writes for y in list(b.writes) + b.reads if related(x, y)} + | {x for x in b.writes for y in a.reads if related(x, y)} + ) + signals = { + "scope_overlap": scope, + "dependency_overlap": deps if depth >= 2 else [], + "read_write_interference": interference if depth == 4 else [], + "contract_incompatibility": [], + "redundancy": False, + "unresolved": sorted(set(a.unresolved + b.unresolved)), + } + + def result(label, entities, reason, score=0): + return Prediction( + label, list(dict.fromkeys(entities))[:3], score, [reason], signals + ) + + if depth == 0: + return result( + "conflicting" if files else "independent", + ["file:" + p for p in files], + "File intersection only", + 0.5 if files else 0, + ) + if depth == 1: + return result( + "conflicting" if scope else "independent", + scope, + "Symbol intersection only", + 0.5 if scope else 0, + ) + if depth == 2: + return result( + "conflicting" if scope or deps else "independent", + scope + deps, + "Symbol or dependency intersection only", + 0.5 if scope or deps else 0, + ) + if depth == 4 and a.symbols and set(a.symbols) == set(b.symbols): + if ( + set(a.after) == set(b.after) + and all(a.after[k].canonical == b.after[k].canonical for k in a.after) + and set(a.before) == set(b.before) + and all(a.before[k].canonical == b.before[k].canonical for k in a.before) + ): + signals["redundancy"] = True + return result( + "redundant", + scope or a.symbols, + "Equal normalized AST transformations", + 0.02, + ) + for source, target in ((a, b), (b, a)): + for op in source.operations: + k = op["entity"] + if op["operation"] in {"delete", "rename"} and any( + related(k, t) for t in target.symbols + target.reads + target.imports + ): + signals["contract_incompatibility"].append(k) + return result( + "conflicting", + [k], + f"{op['operation']} intersects changed implementation or surviving consumer", + 0.98, + ) + for k, contract in source.contracts_changed.items(): + consumers = [c for c in target.calls if related(c["target"], k)] + referenced = any(related(k, t) for t in target.reads + target.imports) + if "signature" in contract and consumers: + sig = contract["signature"]["after"] + for call in consumers: + if call["dynamic"]: + continue + keywords = set(call["keywords"]) + params = sig.get("parameters", []) + required = params[: sig.get("required", 0)] + bad = ( + sig.get("maximum") is not None and call["argc"] > sig["maximum"] + ) + bad |= any(p not in keywords for p in required[call["argc"] :]) + bad |= any( + p not in keywords for p in sig.get("required_kwonly", []) + ) + bad |= bool( + keywords + - set(params[sig.get("posonly", 0) :]) + - set(sig.get("kwonly", [])) + ) and not sig.get("kwargs") + if bad: + signals["contract_incompatibility"].append(k) + return result( + "conflicting", + [k], + "Changed signature rejects an observed call", + 0.96, + ) + if referenced and ("return_type" in contract or "value" in contract): + # Conservative: this prototype cannot prove adaptation at the use site. + signals["contract_incompatibility"].append(k) + return result( + "conflicting", + [k], + "Consumer references a changed return/configuration/dependency contract", + 0.85, + ) + if depth == 4: + if interference: + return result( + "conflicting", + interference, + "Changed writes intersect concurrent reads or writes", + 0.82, + ) + for k in scope: + if k in a.after and k in b.after and k in a.before and k in b.before: + aa, bb, old = a.after[k], b.after[k], a.before[k] + if ( + aa.return_exprs != old.return_exprs + and bb.return_exprs != old.return_exprs + and aa.return_exprs != bb.return_exprs + ): + return result( + "conflicting", + [k], + "Different concurrent replacements of return behavior", + 0.8, + ) + if ( + aa.signature != old.signature + and bb.signature != old.signature + and aa.signature != bb.signature + ): + return result( + "conflicting", + [k], + "Different concurrent function signatures", + 0.85, + ) + # Different creations at the same qualified symbol collide even without returns. + for k in set(a.after) & set(b.after) - set(a.before) - set(b.before): + if a.after[k].canonical != b.after[k].canonical: + return result( + "conflicting", + [k], + "Different definitions create the same symbol", + 0.9, + ) + if scope or deps: + return result( + "compatible", + scope + deps, + "Shared scope/dependency without a recognized incompatible effect", + 0.15, + ) + return result("independent", [], "No resolved interaction", 0) diff --git a/prototypes/structured-delta/experiments/delta/model.py b/prototypes/structured-delta/experiments/delta/model.py new file mode 100644 index 0000000..5261b60 --- /dev/null +++ b/prototypes/structured-delta/experiments/delta/model.py @@ -0,0 +1,61 @@ +"""Serializable observations inferred exclusively from source snapshots.""" + +from dataclasses import dataclass, field, asdict +from typing import Any + +LABELS = ("independent", "compatible", "redundant", "conflicting") + + +@dataclass +class Symbol: + entity: str + kind: str + start: int + end: int + canonical: str + signature: dict = field(default_factory=dict) + returns: list[str] = field(default_factory=list) + return_exprs: list[str] = field(default_factory=list) + reads: list[str] = field(default_factory=list) + writes: dict[str, list[str]] = field(default_factory=dict) + calls: list[dict] = field(default_factory=list) + statements: list[str] = field(default_factory=list) + value: str | None = None + + +@dataclass +class ProgramDelta: + files: list[str] + symbols: list[str] + operations: list[dict] + reads: list[str] + writes: dict[str, list[str]] + signatures_before: dict + signatures_after: dict + types_before: dict + types_after: dict + dependencies_added: list[str] + dependencies_removed: list[str] + contracts_changed: dict + imports: list[str] + calls: list[dict] + ast_operations: list[dict] + lines: dict[str, list[int]] + before: dict[str, Symbol] + after: dict[str, Symbol] + unresolved: list[str] + + def to_dict(self): + return asdict(self) + + +@dataclass +class Prediction: + label: str + entities: list[str] + score: float + reasons: list[str] + signals: dict[str, Any] = field(default_factory=dict) + + def to_dict(self): + return asdict(self) diff --git a/prototypes/structured-delta/experiments/evaluation/__init__.py b/prototypes/structured-delta/experiments/evaluation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prototypes/structured-delta/experiments/evaluation/evaluate.py b/prototypes/structured-delta/experiments/evaluation/evaluate.py new file mode 100644 index 0000000..6a5b81f --- /dev/null +++ b/prototypes/structured-delta/experiments/evaluation/evaluate.py @@ -0,0 +1,156 @@ +import argparse +import json +import platform +import time +from collections import Counter +from pathlib import Path +from ..delta.extractor import extract +from ..delta.interaction import detect, LEVELS +from ..baselines import file_overlap, ast_overlap, git_merge, embedding, llm_intent +from .runtime import snapshot, commutativity +from .metrics import score, grouped +from .freeze import verify + + +def run(root, benchmark, out, embedding_command=None, llm_command=None): + frozen = verify(root) + if out.exists(): + raise FileExistsError(f"Refusing to replace results: {out}") + out.mkdir(parents=True) + all_rows = {} + records = [] + for path in sorted(benchmark.glob("case[0-9]*")): + base, a, b = (snapshot(path / k) for k in ("base", "a", "b")) + started = time.perf_counter() + da, db = extract(base, a), extract(base, b) + extracted_ms = (time.perf_counter() - started) * 1000 + started = time.perf_counter() + preds = { + "B1 File": file_overlap.predict(da, db), + "B2 Line": ast_overlap.predict(da, db, "line"), + "B2 AST/Symbol": ast_overlap.predict(da, db), + } + preds.update({level: detect(da, db, level) for level in LEVELS}) + detection_ms = (time.perf_counter() - started) * 1000 + patch_a, patch_b = ((path / f"patch_{k}.diff").read_text() for k in ("A", "B")) + optional = {} + for key, command, fn in ( + ( + "B4 Embedding", + embedding_command, + lambda cmd: embedding.predict(cmd, patch_a, patch_b, da, db), + ), + ( + "B5 LLM descriptions", + llm_command, + lambda cmd: llm_intent.predict(cmd, patch_a, patch_b), + ), + ): + if command: + try: + preds[key] = fn(command) + optional[key] = {"status": "ok"} + except Exception as error: + optional[key] = {"status": "error", "error": str(error)} + else: + optional[key] = {"status": "not_configured"} + # Gold, requirements and witness data are read only after predictions. + gold = json.loads((path / "gold.json").read_text()) + checks = {k: (path / "checks" / f"{k}.py").read_text() for k in ("a", "b")} + commute = commutativity(base, a, b, patch_a, patch_b, checks) + preds["B3 Git merge"] = git_merge.predict(commute["three_way"]["ab"]) + rec = { + "id": path.name, + "gold": gold, + "delta_a": da.to_dict(), + "delta_b": db.to_dict(), + "predictions": {k: v.to_dict() for k, v in preds.items()}, + "optional": optional, + "timing": { + "extract_ms": extracted_ms, + "deterministic_all_rules_ms": detection_ms, + }, + "commutativity": commute, + } + (out / f"{path.name}.json").write_text( + json.dumps(rec, ensure_ascii=False, indent=2) + "\n" + ) + records.append(rec) + for method, prediction in preds.items(): + all_rows.setdefault(method, []).append( + { + "id": path.name, + "gold": gold["label"], + "entities": gold["entities"], + "family": gold["family"], + "cross_file": gold["cross_file"], + "prediction": prediction.to_dict(), + } + ) + print(path.name, flush=True) + # Unavailable optional results remain missing; partial subsets are explicitly marked. + results = { + method: { + "aggregate": score(rows), + "by_family": grouped(rows), + "coverage": {"scored": len(rows), "total": len(records)}, + } + for method, rows in all_rows.items() + } + errors = { + method: { + "false_positives": [ + r + for r in rows + if r["gold"] != "conflicting" + and r["prediction"]["label"] == "conflicting" + ], + "false_negatives": [ + r + for r in rows + if r["gold"] == "conflicting" + and r["prediction"]["label"] != "conflicting" + ], + "all_misclassified": [ + r for r in rows if r["gold"] != r["prediction"]["label"] + ], + } + for method, rows in all_rows.items() + } + summary = { + "environment": { + "python": platform.python_version(), + "platform": platform.platform(), + }, + "freeze_created_at": frozen["created_at"], + "cases": len(records), + "labels": dict(Counter(r["gold"]["label"] for r in records)), + "families": len({r["gold"]["family"] for r in records}), + "cross_file_conflicts": sum( + r["gold"]["label"] == "conflicting" and r["gold"]["cross_file"] + for r in records + ), + "methods": results, + "optional": { + key: dict(Counter(r["optional"][key]["status"] for r in records)) + for key in ("B4 Embedding", "B5 LLM descriptions") + }, + "errors": errors, + } + (out / "summary.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2) + "\n" + ) + print(json.dumps({k: v["aggregate"] for k, v in results.items()}, indent=2)) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--root", type=Path, default=Path(".")) + p.add_argument( + "--benchmark", type=Path, default=Path("experiments/benchmark/cases") + ) + p.add_argument("--out", type=Path, default=Path("experiments/results/run-001")) + p.add_argument("--embedding-command") + p.add_argument("--llm-command") + a = p.parse_args() + run(a.root, a.benchmark, a.out, a.embedding_command, a.llm_command) diff --git a/prototypes/structured-delta/experiments/evaluation/freeze.py b/prototypes/structured-delta/experiments/evaluation/freeze.py new file mode 100644 index 0000000..53d69ea --- /dev/null +++ b/prototypes/structured-delta/experiments/evaluation/freeze.py @@ -0,0 +1,55 @@ +"""Freeze all protocol, code and benchmark bytes before formal evaluation.""" + +import argparse +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path + + +def hashes(root): + paths = [root / "PROTOCOL.md"] + sorted((root / "experiments").rglob("*")) + return { + p.relative_to(root).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest() + for p in paths + if p.is_file() + and "__pycache__" not in p.parts + and "results" not in p.parts + and p.suffix in {".py", ".md", ".json", ".diff", ".txt"} + } + + +def verify(root): + frozen = json.loads((root / "FREEZE.json").read_text()) + current = hashes(root) + if frozen["sha256"] != current: + changed = sorted( + k + for k in frozen["sha256"].keys() | current.keys() + if frozen["sha256"].get(k) != current.get(k) + ) + raise RuntimeError( + "Frozen inputs changed; create a new protocol version, retain old runs: " + + str(changed) + ) + return frozen + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--root", type=Path, default=Path(".")) + args = p.parse_args() + dest = args.root / "FREEZE.json" + if dest.exists(): + raise FileExistsError("Refusing to replace existing freeze") + dest.write_text( + json.dumps( + { + "created_at": datetime.now(timezone.utc).isoformat(), + "sha256": hashes(args.root), + }, + indent=2, + ) + + "\n" + ) + print(dest) diff --git a/prototypes/structured-delta/experiments/evaluation/metrics.py b/prototypes/structured-delta/experiments/evaluation/metrics.py new file mode 100644 index 0000000..10e4aad --- /dev/null +++ b/prototypes/structured-delta/experiments/evaluation/metrics.py @@ -0,0 +1,67 @@ +"""Metrics have explicit denominators; null means undefined, never zero-filled.""" + +from ..delta.model import LABELS + + +def ratio(n, d): + return n / d if d else None + + +def score(rows): + confusion = {a: {b: 0 for b in LABELS} for a in LABELS} + for r in rows: + confusion[r["gold"]][r["prediction"]["label"]] += 1 + f1 = [] + per_class = {} + for label in LABELS: + tp = confusion[label][label] + fp = sum(confusion[g][label] for g in LABELS if g != label) + fn = sum(confusion[label][p] for p in LABELS if p != label) + v = 2 * tp / (2 * tp + fp + fn) if 2 * tp + fp + fn else 0.0 + f1.append(v) + per_class[label] = { + "precision": ratio(tp, tp + fp), + "recall": ratio(tp, tp + fn), + "f1": v, + "support": tp + fn, + } + compat = [r for r in rows if r["gold"] == "compatible"] + cross = [r for r in rows if r["gold"] == "conflicting" and r["cross_file"]] + local = [r for r in rows if r["entities"]] + top1 = sum( + bool(r["prediction"]["entities"]) + and r["prediction"]["entities"][0] in r["entities"] + for r in local + ) + recall3 = sum( + bool(set(r["prediction"]["entities"][:3]) & set(r["entities"])) for r in local + ) + cp = per_class["conflicting"] + return { + "n": len(rows), + "macro_f1": sum(f1) / 4, + "conflict_precision": cp["precision"], + "conflict_recall": cp["recall"], + "compatible_false_positive_rate": ratio( + sum(r["prediction"]["label"] == "conflicting" for r in compat), len(compat) + ), + "cross_file_conflict_recall": ratio( + sum(r["prediction"]["label"] == "conflicting" for r in cross), len(cross) + ), + "localization_top1": ratio(top1, len(local)), + "localization_recall3": ratio(recall3, len(local)), + "denominators": { + "compatible": len(compat), + "cross_file_conflicting": len(cross), + "localization": len(local), + }, + "per_class": per_class, + "confusion": confusion, + } + + +def grouped(rows): + return { + family: score([r for r in rows if r["family"] == family]) + for family in sorted({r["family"] for r in rows}) + } diff --git a/prototypes/structured-delta/experiments/evaluation/report.py b/prototypes/structured-delta/experiments/evaluation/report.py new file mode 100644 index 0000000..8b45171 --- /dev/null +++ b/prototypes/structured-delta/experiments/evaluation/report.py @@ -0,0 +1,296 @@ +"""Render a self-contained audit report from persisted results (no prediction).""" + +import argparse +import json +from pathlib import Path + + +def pct(v): + return "—" if v is None else f"{100 * v:.1f}%" + + +def conflict_f1(m): + p, r = m["conflict_precision"], m["conflict_recall"] + return 2 * p * r / (p + r) if p is not None and r is not None and p + r else 0 + + +def render(static, online, out): + s = json.loads((static / "summary.json").read_text()) + o = json.loads((online / "summary.json").read_text()) + recs = [json.loads(p.read_text()) for p in sorted(static.glob("case*.json"))] + full = s["methods"]["Full Structured Delta"]["aggregate"] + baselines = { + k: v["aggregate"] + for k, v in s["methods"].items() + if k.startswith(("B1", "B2", "B3")) + } + best = max(baselines, key=lambda k: conflict_f1(baselines[k])) + supported = ( + conflict_f1(full) > conflict_f1(baselines[best]) + and full["compatible_false_positive_rate"] + < baselines[best]["compatible_false_positive_rate"] + and full["cross_file_conflict_recall"] + > baselines[best]["cross_file_conflict_recall"] + ) + text = [ + "# 结构化程序状态 delta：实验报告", + "", + "2026-10-07。从共同 base 的实际代码差异提取结构化变化；没有使用代理自述、gold、测试反馈或参考实现作为检测输入。", + "", + f"**结论：{'满足' if supported else '未满足'}协议预先规定的有限支持条件。** " + + ( + "Full 在本轮构造样本上的冲突 F1、兼容误报率和跨文件召回优于最强可用简单基线。" + if supported + else "本轮不能支持结构化方法优于简单基线的有限命题。" + ), + "这不是总体假设的证明：样本和规则由同一实现者共同设计，没有独立留出集；B4/B5 未配置外部模型，无法检验“优于 embedding/LLM”的部分。", + "", + f"样本：{s['cases']} 个 Python 微型仓库、{s['families']} 个命名场景族；标签分布 `{s['labels']}`。" + f"跨文件冲突 {s['cross_file_conflicts']} 个，指两个补丁修改文件集合完全不相交。定位分母 {full['denominators']['localization']} 个，排除 independent。", + "", + f"冻结时刻：`{s['freeze_created_at']}`。环境：Python {s['environment']['python']}；Git 2.43.0。" + "实现/样本/协议哈希见 FREEZE.json；逐案例证据见 results/run-001（如为复跑则见本次指定目录）。", + "", + "## 可用方法比较", + "", + "B1/B2/B3 是二分类启发式，只输出 independent/conflicting；四分类 macro F1 天然受限。" + "不能仅凭 macro F1 判断胜负，故同时列出冲突 F1、精确率/召回率、兼容误报和跨文件召回。" + "定位为严格实体匹配，文件名不能替代函数/配置实体。", + "", + "| 方法 | Macro F1 | 冲突 P | 冲突 R | 冲突 F1 | 兼容 FP | 跨文件 R | Top-1 | R@3 |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + methods = [ + "B1 File", + "B2 Line", + "B2 AST/Symbol", + "B3 Git merge", + "B4 Embedding", + "B5 LLM descriptions", + "Full Structured Delta", + ] + + def row(k): + if k not in s["methods"]: + return f"| {k} | 未运行 | — | — | — | — | — | — | — |" + m = s["methods"][k]["aggregate"] + return ( + "| " + + k + + " | " + + " | ".join( + pct(v) + for v in ( + m["macro_f1"], + m["conflict_precision"], + m["conflict_recall"], + conflict_f1(m), + m["compatible_false_positive_rate"], + m["cross_file_conflict_recall"], + m["localization_top1"], + m["localization_recall3"], + ) + ) + + " |" + ) + + text.extend(row(k) for k in methods) + text += [ + "", + f"最强简单基线按冲突 F1 为 {best}。B4/B5 状态：`{s['optional']}`。" + "可选调用失败时保留错误和覆盖率，不用成功子集冒充全量。", + "", + "## 消融", + "", + "| 特征/规则束 | Macro F1 | 冲突 P | 冲突 R | 冲突 F1 | 兼容 FP | 跨文件 R | Top-1 | R@3 |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + levels = [ + "File", + "File + Symbol", + "File + Symbol + Dependency", + "File + Symbol + Dependency + Contract", + "Full Structured Delta", + ] + text.extend(row(k) for k in levels) + text += [ + "", + "这些是特征和判定规则一起变化的阶梯消融：Contract 阶段才开始把无已知不兼容的相关变化判为 compatible；Full 同时加入读写、返回表达式和规范化重复识别。因此不能把差值严格解释为单一结构组件的因果收益。", + "", + "| 相邻阶段 | 冲突 F1 变化（百分点） | Macro F1 变化（百分点） |", + "|---|---:|---:|", + ] + for prev, nxt in zip(levels, levels[1:]): + m, n = s["methods"][prev]["aggregate"], s["methods"][nxt]["aggregate"] + text.append( + f"| {nxt} | {(conflict_f1(n) - conflict_f1(m)) * 100:+.1f} | {(n['macro_f1'] - m['macro_f1']) * 100:+.1f} |" + ) + text += [ + "", + "## 按场景类型的结果", + "", + "下表给出每个家族的 Full 结果；所有方法的逐族七项指标及混淆矩阵均在 summary.json。" + "小家族只有 1–3 例，不能将单例的 0%/100% 当成稳定概率。", + "", + "| 类型 | n | 四分类正确数 | 冲突命中/真冲突 | 非冲突误报数 | 实体 Top-1 |", + "|---|---:|---:|---:|---:|---:|", + ] + for family, m in s["methods"]["Full Structured Delta"]["by_family"].items(): + records = [r for r in recs if r["gold"]["family"] == family] + correct = sum( + r["gold"]["label"] == r["predictions"]["Full Structured Delta"]["label"] + for r in records + ) + positives = [r for r in records if r["gold"]["label"] == "conflicting"] + tp = sum( + r["predictions"]["Full Structured Delta"]["label"] == "conflicting" + for r in positives + ) + fp = sum( + r["gold"]["label"] != "conflicting" + and r["predictions"]["Full Structured Delta"]["label"] == "conflicting" + for r in records + ) + text.append( + f"| {family} | {m['n']} | {correct} | {tp}/{len(positives)} | {fp} | {pct(m['localization_top1'])} |" + ) + text += [ + "", + "## 错误分析", + "", + "| 案例 | 真值 → 预测 | 类型 | 检测理由/实体 |", + "|---|---|---|---|", + ] + for r in s["errors"]["Full Structured Delta"]["all_misclassified"]: + pred = r["prediction"] + text.append( + f"| {r['id']} | {r['gold']} → {pred['label']} | {r['family']} | " + + ("; ".join(pred["reasons"]) + "; " + ", ".join(pred["entities"])).replace( + "|", "/" + ) + + " |" + ) + fps = s["errors"]["Full Structured Delta"]["false_positives"] + fns = s["errors"]["Full Structured Delta"]["false_negatives"] + text += [ + "", + f"Full 冲突误报 {len(fps)} 个：`{[r['id'] for r in fps]}`；漏报 {len(fns)} 个：`{[r['id'] for r in fns]}`。", + "误报风险来自保守的契约变化⇒失效、共享读写⇒冲突，以及未证明等价时把不同返回表达式视为互斥。" + "漏报风险来自动态查找、外部状态、JSON 读取和分发解析缺失。按实际错误行判断本轮触发了哪些限制，不能把这些局限用增加同模板样本掩盖。", + "更多限制：前后 reads/calls 的并集可能保留已移除的假设；未知类型和源代码注解不构成完整类型证明；manifest 尚未关联安装包到 import；同符号 compatible 默认也可能漏掉路径和执行顺序冲突。", + "", + "## 交换顺序和实际检查", + "", + "| 真值 | n | 两个 strict 顺序都适用 | strict 两序都通过联合测试 | 两个 three-way 都干净 | three-way 两序都通过联合测试 | 干净产物不同 |", + "|---|---:|---:|---:|---:|---:|---:|", + ] + for label in ("independent", "compatible", "redundant", "conflicting"): + group = [r for r in recs if r["gold"]["label"] == label] + + def count(fn): + return sum(fn(r["commutativity"]) for r in group) + + text.append( + f"| {label} | {len(group)} | {count(lambda c: c['strict_patch']['both_apply'])} | " + f"{count(lambda c: all(c['strict_patch'][k]['tests'] is not None and c['strict_patch'][k]['tests']['passed'] for k in ('ab', 'ba')))} | " + f"{count(lambda c: c['three_way']['both_clean'])} | " + f"{count(lambda c: all(c['three_way'][k]['tests'] is not None and c['three_way'][k]['tests']['passed'] for k in ('ab', 'ba')))} | " + f"{count(lambda c: c['three_way']['states_differ'] is True)} |" + ) + text += [ + "", + "适用/合并失败时测试结果与产物比较为 null，不是失败测试或相同产物。" + "冲突标记树、每个顺序的源码哈希、测试 stderr 和返回码均保留。three-way 是共同祖先文本合并，不是语义修复。" + "联合测试并非完备 oracle；兼容案例的 witness 证明存在满足双方要求的组合，不代表 Git 已产生该组合。", + "", + "## 代表性案例", + "", + ] + ids = [ + "case004", + "case011", + "case012", + "case025", + "case034", + "case037", + "case041", + "case045", + "case047", + ] + for cid in ids: + r = next(v for v in recs if v["id"] == cid) + g = r["gold"] + p = r["predictions"]["Full Structured Delta"] + text += [ + f"- **{cid} — {g['title']}**：真值 {g['label']}；B1={r['predictions']['B1 File']['label']}，Git={r['predictions']['B3 Git merge']['label']}，Full={p['label']}。{g['explanation']}" + ] + text += [ + "", + "高文本相似既可能是兼容（case011/012），也可能是重复（case025）或冲突（case044）；跨文件契约变化（case034/037）即使补丁词面不同也会破坏消费者。" + "这些是相似度不能定义语义冲突的机制反例；B4 未运行，不能声称某个 embedding 模型在这些案例实际失败。", + "", + "## 最小在线模拟", + "", + f"{o['cases']} 例、{o['events']} 个补丁事件。冲突检出 {o['conflicts_detected']}/{o['conflicting_cases']}；" + f"非冲突误干预 {o['false_interventions']}/{o['nonconflicting_cases']}，严格“早于首次失败”的误干预 {o['false_early_interventions']}。" + f"冲突案例有效 lead ticks：`{o['lead_ticks_conflicts']}`，均值 `{o['mean_lead_ticks_conflicts']}`；" + f"单事件提取＋检测中位 {o['median_detection_ms']:.3f} ms（单次共享主机测量）。", + "", + "| 案例 | 真值 | 首次告警 | 首次 merge 失败 | 首次行为测试失败 | lead ticks | semantic lead | 误干预 |", + "|---|---|---:|---:|---:|---:|---:|---|", + ] + for r in o["rows"]: + text.append( + "| " + + r["id"] + + " | " + + r["label"] + + " | " + + " | ".join( + "—" if r[k] is None else str(r[k]) + for k in ( + "first_detection_tick", + "first_merge_failure_tick", + "first_test_failure_tick", + "lead_ticks", + "semantic_lead_ticks", + "false_intervention", + ) + ) + + " |" + ) + text += [ + "", + "时间单位为事件序号；正 lead 才表示提前。缺失检测/失败时保留 null。" + "轨迹只有“准备 helper、首文件修改、完成/清理”三个预设阶段，不能证明真实代理工作中能提前预警。" + "要求在功能变化完整时立即激活，避免把清理阶段当作虚假的检测领先。静态标签也不能充分标注每个中间状态的临时冲突。", + "", + "## 对五个研究问题的回答", + "", + f"1. 结构化表示是否提高检测：本轮 Full 冲突 F1={pct(conflict_f1(full))}，最强简单基线={pct(conflict_f1(baselines[best]))}；有限支持判据={supported}，无外部泛化结论。", + "2. 哪些组件最有用：看消融相邻差值；依赖边负责跨文件候选，契约规则区分部分相关但兼容的修改，Full 的重复识别和读写规则也会引入误报。混合规则束不能做纯组件因果归因。", + f"3. 同作用域兼容误报：Full 兼容误报率 {pct(full['compatible_false_positive_rate'])}；仍应逐例看哪些误报由同作用域、契约或状态引发。", + f"4. 跨文件冲突：Full 召回 {pct(full['cross_file_conflict_recall'])}（分母 {full['denominators']['cross_file_conflicting']}）；动态关系仍是明显盲点。", + f"5. 实体定位：Top-1 {pct(full['localization_top1'])}、R@3 {pct(full['localization_recall3'])}；独立于类别判定评分，不能把定位正确当成冲突判断正确。", + "", + "## 唯一优先的下一项实验", + "", + "冻结当前实现，独立收集并盲标至少 100 个来自未见真实仓库的并发修改对，用可执行的双方要求裁决，重点覆盖向后兼容契约与动态跨文件依赖；B4/B5 在独立校准集固定模型和阈值后同轮运行。" + "该迁移盲测优先于扩展同模板案例、接入路由或构建完整 AgentGit。", + "", + "实现文件与复现命令见 README.md / FILES.md。没有提交、推送、插件安装、模型调用或自动修复。", + "", + ] + out.write_text("\n".join(text)) + print(out) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--static", type=Path, default=Path("experiments/results/run-001")) + p.add_argument( + "--online", type=Path, default=Path("experiments/results/online-001") + ) + p.add_argument("--out", type=Path, default=Path("REPORT.md")) + a = p.parse_args() + render(a.static, a.online, a.out) diff --git a/prototypes/structured-delta/experiments/evaluation/runtime.py b/prototypes/structured-delta/experiments/evaluation/runtime.py new file mode 100644 index 0000000..bae2f81 --- /dev/null +++ b/prototypes/structured-delta/experiments/evaluation/runtime.py @@ -0,0 +1,221 @@ +"""Real Git application/merges and subprocess checks; never used by extractor.""" + +import hashlib +import json +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path + + +def snapshot(path): + return { + p.relative_to(path).as_posix(): p.read_text() + for p in sorted(path.rglob("*")) + if p.is_file() and "__pycache__" not in p.parts and ".git" not in p.parts + } + + +def digest(state): + return hashlib.sha256(json.dumps(state, sort_keys=True).encode()).hexdigest() + + +def write(root, state): + for p, s in state.items(): + dest = root / p + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(s) + + +def git(root, *args, input=None, check=True): + env = { + **os.environ, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_AUTHOR_NAME": "Delta Research", + "GIT_AUTHOR_EMAIL": "delta@example.invalid", + "GIT_COMMITTER_NAME": "Delta Research", + "GIT_COMMITTER_EMAIL": "delta@example.invalid", + "GIT_AUTHOR_DATE": "2026-10-07T00:00:00Z", + "GIT_COMMITTER_DATE": "2026-10-07T00:00:00Z", + } + # Ignore inherited repository selection, avoiding accidental parent /tmp/.git. + for key in ( + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_COMMON_DIR", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + ): + env.pop(key, None) + result = subprocess.run( + ["git", "-C", str(root), *args], + input=input, + capture_output=True, + text=True, + env=env, + timeout=20, + ) + if check and result.returncode: + raise RuntimeError( + f"Git {args[0]} failed ({result.returncode}): {result.stderr}" + ) + return result + + +def check_state(state, checks): + started = time.perf_counter() + with tempfile.TemporaryDirectory(prefix="delta-check-") as temp: + root = Path(temp) + write(root, state) + results = [] + for name, code in [ + ( + "syntax", + 'import ast\nfrom pathlib import Path\nfor p in Path(".").rglob("*.py"): ast.parse(p.read_text(),filename=str(p))', + ) + ] + list(checks.items()): + try: + # -I plus explicit cwd in sys.path: no installed project/PYTHONPATH leakage. + result = subprocess.run( + [ + sys.executable, + "-I", + "-B", + "-c", + 'import sys; sys.path.insert(0, ".")\n' + code, + ], + cwd=root, + capture_output=True, + text=True, + timeout=5, + ) + results.append( + { + "check": name, + "passed": result.returncode == 0, + "exit_code": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + } + ) + except subprocess.TimeoutExpired as exc: + results.append( + { + "check": name, + "passed": False, + "exit_code": None, + "error": "timeout", + "stdout": str(exc.stdout), + "stderr": str(exc.stderr), + } + ) + return { + "passed": all(r["passed"] for r in results), + "checks": results, + "elapsed_ms": (time.perf_counter() - started) * 1000, + } + + +def strict_order(base, patches, checks): + with tempfile.TemporaryDirectory(prefix="delta-apply-") as temp: + root = Path(temp) + git(root, "init", "-q") + write(root, base) + steps = [] + for patch in patches: + r = git(root, "apply", "--whitespace=nowarn", "-", input=patch, check=False) + steps.append( + { + "applied": r.returncode == 0, + "stderr": r.stderr, + "exit_code": r.returncode, + } + ) + if r.returncode: + return { + "applied": False, + "steps": steps, + "state_hash": None, + "tests": None, + } + state = snapshot(root) + return { + "applied": True, + "steps": steps, + "state_hash": digest(state), + "tests": check_state(state, checks), + } + + +def merged_pair(base, a, b, checks=None): + """Git merge-tree in both parent orders, with fixed config and actual tree states.""" + with tempfile.TemporaryDirectory(prefix="delta-git-") as temp: + root = Path(temp) + git(root, "init", "-q") + + def commit(state, message): + old = snapshot(root) + for p in old.keys() - state.keys(): + (root / p).unlink() + write(root, state) + git(root, "add", "-A") + git(root, "commit", "--allow-empty", "-qm", message) + return git(root, "rev-parse", "HEAD").stdout.strip() + + base_id = commit(base, "base") + a_id = commit(a, "A") + git(root, "checkout", "-q", base_id) + b_id = commit(b, "B") + out = {} + for key, left, right in (("ab", a_id, b_id), ("ba", b_id, a_id)): + r = git(root, "merge-tree", "--write-tree", left, right, check=False) + if r.returncode not in (0, 1): + raise RuntimeError("merge-tree infrastructure error: " + r.stderr) + rows = r.stdout.splitlines() + tree = rows[0] + conflict_paths = sorted( + { + line.split("\t", 1)[1] + for line in rows[1:] + if "\t" in line + and line.split("\t")[0].split()[-1:] in (["1"], ["2"], ["3"]) + } + ) + # git show via each tracked tree path avoids filesystem archive extraction. + names = git(root, "ls-tree", "-r", "--name-only", tree).stdout.splitlines() + state = {p: git(root, "show", f"{tree}:{p}").stdout for p in names} + clean = r.returncode == 0 + out[key] = { + "clean": clean, + "conflict_paths": conflict_paths, + "state_hash": digest(state), + "tests": check_state(state, checks or {}) if clean else None, + "raw": r.stdout, + "state": state, + } + out["both_clean"] = out["ab"]["clean"] and out["ba"]["clean"] + out["states_differ"] = ( + out["ab"]["state_hash"] != out["ba"]["state_hash"] + if out["both_clean"] + else None + ) + return out + + +def commutativity(base, a, b, patch_a, patch_b, checks): + merge = merged_pair(base, a, b, checks) + strict = { + "ab": strict_order(base, [patch_a, patch_b], checks), + "ba": strict_order(base, [patch_b, patch_a], checks), + } + strict["both_apply"] = strict["ab"]["applied"] and strict["ba"]["applied"] + strict["states_differ"] = ( + strict["ab"]["state_hash"] != strict["ba"]["state_hash"] + if strict["both_apply"] + else None + ) + return {"strict_patch": strict, "three_way": merge} diff --git a/prototypes/structured-delta/experiments/evaluation/validate.py b/prototypes/structured-delta/experiments/evaluation/validate.py new file mode 100644 index 0000000..a01bce9 --- /dev/null +++ b/prototypes/structured-delta/experiments/evaluation/validate.py @@ -0,0 +1,66 @@ +"""Validate fixtures without exposing checks/witnesses to prediction modules.""" + +import argparse +import json +from pathlib import Path +from .runtime import snapshot, check_state, strict_order, digest + + +def validate(root, out): + records = [] + for path in sorted(root.glob("case[0-9]*")): + base, a, b = (snapshot(path / k) for k in ("base", "a", "b")) + gold = json.loads((path / "gold.json").read_text()) + checks = { + k: (path / "checks" / f"{k}.py").read_text() for k in ("base", "a", "b") + } + r = { + "id": path.name, + "base": check_state(base, {"base": checks["base"]}), + "a": check_state(a, {"a": checks["a"]}), + "b": check_state(b, {"b": checks["b"]}), + } + for key, target in (("a", a), ("b", b)): + applied = strict_order( + base, [(path / f"patch_{key.upper()}.diff").read_text()], {} + ) + r["patch_" + key] = { + "applied": applied["applied"], + "matches_snapshot": applied["state_hash"] == digest(target), + "details": applied if not applied["applied"] else None, + } + if gold["witness_available"]: + r["witness"] = check_state( + snapshot(path / "witness"), {"a": checks["a"], "b": checks["b"]} + ) + records.append(r) + print(path.name, "valid" if valid(r) else "INVALID", flush=True) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text( + json.dumps( + {"valid": all(valid(r) for r in records), "cases": records}, indent=2 + ) + + "\n" + ) + if not all(valid(r) for r in records): + raise SystemExit(1) + + +def valid(r): + return ( + all(r[k]["passed"] for k in ("base", "a", "b")) + and all(r["patch_" + k]["matches_snapshot"] for k in ("a", "b")) + and r.get("witness", {"passed": True})["passed"] + ) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument( + "--benchmark", type=Path, default=Path("experiments/benchmark/cases") + ) + p.add_argument( + "--out", type=Path, default=Path("experiments/results/validation.json") + ) + a = p.parse_args() + validate(a.benchmark, a.out) diff --git a/prototypes/structured-delta/experiments/online/__init__.py b/prototypes/structured-delta/experiments/online/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prototypes/structured-delta/experiments/online/sequences.py b/prototypes/structured-delta/experiments/online/sequences.py new file mode 100644 index 0000000..df07817 --- /dev/null +++ b/prototypes/structured-delta/experiments/online/sequences.py @@ -0,0 +1,24 @@ +"""Deterministic synthetic progress schedule, not real agent trajectories.""" + + +def stages(base, final, branch): + helper = f"_progress_{branch}.py" + draft = dict(base) + draft[helper] = "def ready():\n return True\n" + partial = dict(draft) + changed = sorted( + p for p in base.keys() | final.keys() if base.get(p) != final.get(p) + ) + if changed: + p = changed[0] + if p in final: + partial[p] = final[p] + else: + partial.pop(p, None) + return [draft, partial, dict(final)] + + +def functional_complete(snapshot, final): + return { + k: v for k, v in snapshot.items() if not k.startswith("_progress_") + } == final diff --git a/prototypes/structured-delta/experiments/online/simulator.py b/prototypes/structured-delta/experiments/online/simulator.py new file mode 100644 index 0000000..654931b --- /dev/null +++ b/prototypes/structured-delta/experiments/online/simulator.py @@ -0,0 +1,139 @@ +"""Recompute base-to-active deltas after every incremental patch event. + +Patch-event indices are the time unit. Runtime ms is separately recorded. +A merge failure and a semantic test failure are separate clocks. +""" + +import argparse +import json +import statistics +import time +from pathlib import Path +from ..delta.extractor import extract +from ..delta.interaction import detect +from ..evaluation.runtime import snapshot, merged_pair, strict_order, digest +from ..evaluation.freeze import verify +from .sequences import functional_complete + + +def simulate(root, benchmark, out): + verify(root) + if out.exists(): + raise FileExistsError(out) + out.mkdir(parents=True) + records = [] + for path in sorted(benchmark.glob("case[0-9]*")): + if not (path / "online" / "schedule.json").exists(): + continue + base = snapshot(path / "base") + final = {k: snapshot(path / k) for k in ("a", "b")} + active = {"a": dict(base), "b": dict(base)} + checks = {k: (path / "checks" / f"{k}.py").read_text() for k in ("a", "b")} + events = [] + for tick, event in enumerate( + json.loads((path / "online" / "schedule.json").read_text()), 1 + ): + branch = event["branch"] + target = snapshot(path / "online" / event["state"]) + inc = (path / "online" / event["patch"]).read_text() + application = strict_order(active[branch], [inc], {}) + if not application["applied"] or application["state_hash"] != digest( + target + ): + raise RuntimeError(f"Broken incremental sequence {path.name}:{event}") + active[branch] = target + start = time.perf_counter() + pred = detect(extract(base, active["a"]), extract(base, active["b"])) + elapsed = (time.perf_counter() - start) * 1000 + # Run branch requirements as soon as that branch's functional change is + # fully present, not after arbitrary cleanup (avoids artificial lead). + active_checks = { + k: v + for k, v in checks.items() + if functional_complete(active[k], final[k]) + } + merge = merged_pair(base, active["a"], active["b"], active_checks) + events.append( + { + "tick": tick, + "event": event, + "prediction": pred.to_dict(), + "detection_ms": elapsed, + "requirements_active": list(active_checks), + "merge_clean": merge["both_clean"], + "test_failure": any( + merge[k]["clean"] and not merge[k]["tests"]["passed"] + for k in ("ab", "ba") + ), + "merge": merge, + } + ) + # Gold is used only to score intervention correctness, not for detection. + gold = json.loads((path / "gold.json").read_text()) + + def first(predicate): + return next((e["tick"] for e in events if predicate(e)), None) + + detected = first(lambda e: e["prediction"]["label"] == "conflicting") + merge_failed = first(lambda e: not e["merge_clean"]) + test_failed = first(lambda e: e["test_failure"]) + failures = [v for v in (merge_failed, test_failed) if v is not None] + failed = min(failures) if failures else None + rec = { + "id": path.name, + "label": gold["label"], + "events": events, + "first_detection_tick": detected, + "first_merge_failure_tick": merge_failed, + "first_test_failure_tick": test_failed, + "first_failure_tick": failed, + "lead_ticks": failed - detected + if failed is not None and detected is not None + else None, + "semantic_lead_ticks": test_failed - detected + if test_failed is not None and detected is not None + else None, + "false_intervention": detected is not None + and gold["label"] != "conflicting", + "false_before_failure": detected is not None + and gold["label"] != "conflicting" + and (failed is None or detected < failed), + } + (out / f"{path.name}.json").write_text( + json.dumps(rec, ensure_ascii=False, indent=2) + "\n" + ) + records.append(rec) + print(path.name, flush=True) + conflict = [r for r in records if r["label"] == "conflicting"] + nonconf = [r for r in records if r["label"] != "conflicting"] + leads = [r["lead_ticks"] for r in conflict if r["lead_ticks"] is not None] + summary = { + "cases": len(records), + "events": sum(len(r["events"]) for r in records), + "conflicting_cases": len(conflict), + "conflicts_detected": sum( + r["first_detection_tick"] is not None for r in conflict + ), + "nonconflicting_cases": len(nonconf), + "false_interventions": sum(r["false_intervention"] for r in nonconf), + "false_early_interventions": sum(r["false_before_failure"] for r in nonconf), + "lead_ticks_conflicts": leads, + "mean_lead_ticks_conflicts": statistics.mean(leads) if leads else None, + "median_detection_ms": statistics.median( + e["detection_ms"] for r in records for e in r["events"] + ), + "rows": [{k: v for k, v in r.items() if k != "events"} for r in records], + } + (out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--root", type=Path, default=Path(".")) + p.add_argument( + "--benchmark", type=Path, default=Path("experiments/benchmark/cases") + ) + p.add_argument("--out", type=Path, default=Path("experiments/results/online-001")) + a = p.parse_args() + simulate(a.root, a.benchmark, a.out) diff --git a/prototypes/structured-delta/experiments/tests/__init__.py b/prototypes/structured-delta/experiments/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prototypes/structured-delta/experiments/tests/test_prototype.py b/prototypes/structured-delta/experiments/tests/test_prototype.py new file mode 100644 index 0000000..22c7f1e --- /dev/null +++ b/prototypes/structured-delta/experiments/tests/test_prototype.py @@ -0,0 +1,172 @@ +import unittest +from experiments.delta.extractor import extract +from experiments.delta.interaction import detect +from experiments.delta.model import Prediction +from experiments.evaluation.metrics import score +from experiments.evaluation.runtime import merged_pair, strict_order, digest +from experiments.benchmark.build import patch +from experiments.baselines import embedding, llm_intent +from experiments.online.sequences import stages, functional_complete + + +class ExtractorTests(unittest.TestCase): + def test_alias_and_relative_import(self): + base = { + "pkg/__init__.py": "", + "pkg/api.py": "def f(x): return x\n", + "pkg/c.py": "from .api import f as fetch\ndef run(): return fetch(1)\n", + } + a = {**base, "pkg/api.py": "def f(x,y): return x+y\n"} + b = { + **base, + "pkg/c.py": "from .api import f as fetch\ndef run(): return fetch(2)\n", + } + result = detect(extract(base, a), extract(base, b)) + self.assertEqual(result.label, "conflicting") + self.assertEqual(result.entities, ["pkg/api.py::f"]) + + def test_none_eq_is_not_identity(self): + base = {"m.py": "def f(x): return x\n"} + a = {"m.py": "def f(x):\n if x is None: return 0\n return x\n"} + b = {"m.py": "def f(x):\n if x == None: return 0\n return x\n"} + self.assertNotEqual( + detect(extract(base, a), extract(base, b)).label, "redundant" + ) + + def test_constant_fields_do_not_alias(self): + base = {"m.py": 'D={"a":0,"b":0}\ndef a(): D["a"]=1\ndef b(): D["b"]=1\n'} + a = {"m.py": base["m.py"].replace('D["a"]=1', 'D["a"]=2')} + b = {"m.py": base["m.py"].replace('D["b"]=1', 'D["b"]=3')} + self.assertEqual( + detect(extract(base, a), extract(base, b)).label, "independent" + ) + + def test_removed_call_records_conservative_reads(self): + base = { + "api.py": "def get(): return 1\n", + "c.py": "from api import get\ndef f(): return get()\n", + } + changed = {**base, "c.py": "from api import get\ndef f(): return 2\n"} + self.assertIn("api.py::get", extract(base, changed).reads) + + def test_parse_error_recorded_not_silently_valid(self): + d = extract({"x.py": "def f(): return 1\n"}, {"x.py": "def f(:\n"}) + self.assertTrue(d.unresolved) + + def test_rename_has_evidence(self): + d = extract({"x.py": "def a(): return 1\n"}, {"x.py": "def b(): return 1\n"}) + self.assertIn( + {"entity": "x.py::a", "operation": "rename", "to": "x.py::b"}, d.operations + ) + + def test_deletion_redundancy_and_order_symmetry(self): + base = {"x.py": "def f(): return 1\n"} + a = {"x.py": "# gone\n"} + d = extract(base, a) + self.assertEqual(detect(d, d).label, "redundant") + e = extract(base, {"x.py": "def f(): return 2\n"}) + self.assertEqual(detect(d, e).label, detect(e, d).label) + + +class MetricsTests(unittest.TestCase): + def test_denominators_and_exact_entity_localization(self): + rows = [ + { + "gold": "compatible", + "family": "a", + "cross_file": False, + "entities": ["m.py::f"], + "prediction": Prediction("conflicting", ["file:m.py"], 1, []).to_dict(), + }, + { + "gold": "conflicting", + "family": "b", + "cross_file": True, + "entities": ["m.py::g"], + "prediction": Prediction("conflicting", ["m.py::g"], 1, []).to_dict(), + }, + { + "gold": "independent", + "family": "a", + "cross_file": True, + "entities": [], + "prediction": Prediction("independent", [], 0, []).to_dict(), + }, + ] + result = score(rows) + self.assertEqual(result["conflict_precision"], 0.5) + self.assertEqual(result["compatible_false_positive_rate"], 1) + self.assertEqual(result["cross_file_conflict_recall"], 1) + self.assertEqual(result["localization_top1"], 0.5) + self.assertIsNone(score([])["conflict_precision"]) + + +class GitTests(unittest.TestCase): + def test_clean_cross_file_merge_can_fail_semantically(self): + base = { + "api.py": "def f(): return 1\n", + "c.py": "from api import f\ndef g(): return f()\n", + } + a = {**base, "api.py": 'def f(): return "one"\n'} + b = {**base, "c.py": "from api import f\ndef g(): return f()+1\n"} + result = merged_pair( + base, a, b, {"requirement": "from c import g\nassert g()==2"} + ) + self.assertTrue(result["both_clean"]) + self.assertFalse(result["ab"]["tests"]["passed"]) + self.assertFalse(result["states_differ"]) + + def test_patch_creation_deletion_exactness(self): + base = {"old.py": "X=1\n"} + final = {"new.py": "X=2\n"} + result = strict_order(base, [patch(base, final)], {}) + self.assertTrue(result["applied"]) + self.assertEqual(result["state_hash"], digest(final)) + + def test_text_overlap_is_not_semantic_truth(self): + base = {"m.py": "def f(x): return x\n"} + a = {"m.py": 'def f(x):\n """identity"""\n return x\n'} + b = {"m.py": "def f(x):\n if x is None: return 0\n return x\n"} + self.assertFalse(merged_pair(base, a, b)["both_clean"]) + self.assertEqual(detect(extract(base, a), extract(base, b)).label, "compatible") + + +class InterfaceTests(unittest.TestCase): + def test_optional_interfaces_without_network(self): + from unittest.mock import patch as mock + + d = extract({"m.py": "X=1\n"}, {"m.py": "X=2\n"}) + with mock.object( + embedding, + "invoke", + return_value={ + "vectors": [[1.0, 0.0], [0.0, 1.0]], + "model": "fixture", + "usage": None, + }, + ): + self.assertEqual( + embedding.predict("unused", "a", "b", d, d).label, "independent" + ) + with mock.object( + llm_intent, + "invoke", + side_effect=[ + {"description": "A"}, + {"description": "B"}, + {"label": "compatible", "entities": ["m.py::X"]}, + ], + ): + self.assertEqual(llm_intent.predict("unused", "a", "b").label, "compatible") + + def test_stage_completion_is_not_delayed_to_cleanup(self): + base = {"m.py": "X=1\n"} + final = {"m.py": "X=2\n"} + seq = stages(base, final, "a") + self.assertFalse(functional_complete(seq[0], final)) + self.assertTrue(functional_complete(seq[1], final)) + self.assertTrue(functional_complete(seq[2], final)) + + +if __name__ == "__main__": + unittest.main() diff --git a/prototypes/structured-delta/pyproject.toml b/prototypes/structured-delta/pyproject.toml new file mode 100644 index 0000000..5406c53 --- /dev/null +++ b/prototypes/structured-delta/pyproject.toml @@ -0,0 +1,7 @@ +[tool.ruff] +target-version = "py311" +line-length = 88 +exclude = ["artifacts", "experiments/benchmark/cases", "experiments/results"] + +[tool.ruff.lint] +select = ["E9", "F"] diff --git a/prototypes/structured-delta/tools/run_pilot.py b/prototypes/structured-delta/tools/run_pilot.py new file mode 100644 index 0000000..447cff7 --- /dev/null +++ b/prototypes/structured-delta/tools/run_pilot.py @@ -0,0 +1,57 @@ +"""Reproduce the pilot in a fresh ignored workspace without overwriting a run.""" + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +SOURCE_ROOT = Path(__file__).resolve().parents[1] + + +def run_module(workspace: Path, module: str, *arguments: str) -> None: + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + subprocess.run( + [sys.executable, "-B", "-m", module, *arguments], + cwd=workspace, + env=environment, + check=True, + ) + + +def reproduce(output: Path) -> None: + output = output.resolve() + if output == SOURCE_ROOT or SOURCE_ROOT.is_relative_to(output): + raise ValueError("Output must not contain the published source directory") + output.mkdir(parents=True, exist_ok=False) + shutil.copytree( + SOURCE_ROOT / "experiments", + output / "experiments", + ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "cases", "results"), + ) + shutil.copyfile(SOURCE_ROOT / "PROTOCOL.md", output / "PROTOCOL.md") + run_module(output, "experiments.benchmark.build") + run_module(output, "experiments.evaluation.validate") + run_module(output, "experiments.evaluation.freeze") + run_module(output, "experiments.evaluation.evaluate") + run_module(output, "experiments.online.simulator") + run_module( + output, + "experiments.evaluation.report", + "--static", + "experiments/results/run-001", + "--online", + "experiments/results/online-001", + "--out", + "REPORT.md", + ) + print(f"Report: {output / 'REPORT.md'}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, default=SOURCE_ROOT / "artifacts/pilot-001") + reproduce(parser.parse_args().out) diff --git a/prototypes/structured-delta/tools/verify_source.py b/prototypes/structured-delta/tools/verify_source.py new file mode 100644 index 0000000..7403248 --- /dev/null +++ b/prototypes/structured-delta/tools/verify_source.py @@ -0,0 +1,40 @@ +"""Check published source hashes and AST provenance without experimental data.""" + +import ast +import hashlib +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def digest(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def verify(root: Path = ROOT) -> None: + manifest = json.loads((root / "SOURCE_PROVENANCE.json").read_text()) + paths = { + path.relative_to(root).as_posix() + for path in (root / "experiments").rglob("*.py") + if "__pycache__" not in path.parts + and "cases" not in path.parts + and "results" not in path.parts + } + if paths != set(manifest["modules"]): + raise ValueError("Published module inventory differs from provenance") + for path, expected in manifest["modules"].items(): + content = (root / path).read_bytes() + tree = ast.dump(ast.parse(content), include_attributes=False).encode() + if digest(content) != expected["published_sha256"]: + raise ValueError(f"Published source changed: {path}") + if digest(tree) != expected["program_ast_sha256"]: + raise ValueError(f"Program AST changed: {path}") + if digest((root / "PROTOCOL.md").read_bytes()) != manifest["protocol_sha256"]: + raise ValueError("Historical protocol changed") + print(f"Verified {len(paths)} published modules and historical protocol") + + +if __name__ == "__main__": + verify() From f7cf3a2060279e1220fead5dbc143f608fd05570 Mon Sep 17 00:00:00 2001 From: iiChole <1622202832@qq.com> Date: Wed, 7 Oct 2026 21:18:46 +0800 Subject: [PATCH 2/2] fix: make source verification portable across Python versions --- .github/workflows/structured-delta.yml | 2 + prototypes/structured-delta/README.md | 3 ++ .../structured-delta/SOURCE_PROVENANCE.json | 3 +- .../tools/test_verify_source.py | 49 +++++++++++++++++++ .../structured-delta/tools/verify_source.py | 11 +++-- 5 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 prototypes/structured-delta/tools/test_verify_source.py diff --git a/.github/workflows/structured-delta.yml b/.github/workflows/structured-delta.yml index 96801d6..002ed1d 100644 --- a/.github/workflows/structured-delta.yml +++ b/.github/workflows/structured-delta.yml @@ -19,6 +19,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 strategy: + fail-fast: false matrix: python: ["3.11", "3.14"] defaults: @@ -32,5 +33,6 @@ jobs: - run: python -m pip install ruff==0.16.10 - run: ruff check . && ruff format --check . - run: python tools/verify_source.py + - run: python -m unittest discover -s tools -p 'test_*.py' -v - run: python -m unittest discover -s experiments/tests -v - run: python tools/run_pilot.py --out artifacts/ci-pilot diff --git a/prototypes/structured-delta/README.md b/prototypes/structured-delta/README.md index 1175d15..a8cf1d5 100644 --- a/prototypes/structured-delta/README.md +++ b/prototypes/structured-delta/README.md @@ -61,6 +61,9 @@ results, not evidence of real-repository generalization or superiority to an LLM digests. Formatting and removal of seven unused imports are the only changes to the published implementation. Historical source, frozen inputs and results are preserved locally. [PROTOCOL.md](PROTOCOL.md) is the unmodified original protocol. +Verification checks exact source bytes and parses syntax on each supported Python +version. The historical AST digests were generated with Python 3.14 and document +the same-interpreter comparison; `ast.dump()` is not a cross-version format. ## Optional model baselines diff --git a/prototypes/structured-delta/SOURCE_PROVENANCE.json b/prototypes/structured-delta/SOURCE_PROVENANCE.json index dbdaabc..1f48661 100644 --- a/prototypes/structured-delta/SOURCE_PROVENANCE.json +++ b/prototypes/structured-delta/SOURCE_PROVENANCE.json @@ -159,5 +159,6 @@ "tempfile" ] } - } + }, + "ast_digest_python": "3.14" } diff --git a/prototypes/structured-delta/tools/test_verify_source.py b/prototypes/structured-delta/tools/test_verify_source.py new file mode 100644 index 0000000..d315131 --- /dev/null +++ b/prototypes/structured-delta/tools/test_verify_source.py @@ -0,0 +1,49 @@ +"""Regression coverage for cross-version verification and exact integrity.""" + +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from verify_source import digest, verify + + +class SourceVerificationTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.module = self.root / "experiments/__init__.py" + self.module.parent.mkdir() + self.module.write_bytes(b'"""Source fixture."""\n') + protocol = b"Test protocol\n" + (self.root / "PROTOCOL.md").write_bytes(protocol) + manifest = { + "protocol_sha256": digest(protocol), + "modules": { + "experiments/__init__.py": { + "published_sha256": digest(self.module.read_bytes()), + "program_ast_sha256": digest(b"Historical interpreter AST dump"), + } + }, + } + (self.root / "SOURCE_PROVENANCE.json").write_text(json.dumps(manifest)) + + def test_historical_ast_format_does_not_reject_identical_source(self): + with patch("ast.dump", side_effect=AssertionError("Not cross-version stable")): + verify(self.root) + + def test_even_semantically_identical_source_edits_are_rejected(self): + self.module.write_bytes(self.module.read_bytes() + b"\n") + with self.assertRaisesRegex(ValueError, "Published source changed"): + verify(self.root) + + def test_unrecorded_modules_are_rejected(self): + (self.module.parent / "extra.py").write_text("VALUE = 1\n") + with self.assertRaisesRegex(ValueError, "module inventory"): + verify(self.root) + + +if __name__ == "__main__": + unittest.main() diff --git a/prototypes/structured-delta/tools/verify_source.py b/prototypes/structured-delta/tools/verify_source.py index 7403248..cbbbb74 100644 --- a/prototypes/structured-delta/tools/verify_source.py +++ b/prototypes/structured-delta/tools/verify_source.py @@ -1,4 +1,9 @@ -"""Check published source hashes and AST provenance without experimental data.""" +"""Check exact source bytes and syntax without experimental data. + +Historical AST digests document the original same-interpreter comparison. +They are not portable across Python AST schema and ast.dump format changes. +Exact byte hashes enforce source integrity on every supported interpreter. +""" import ast import hashlib @@ -26,11 +31,9 @@ def verify(root: Path = ROOT) -> None: raise ValueError("Published module inventory differs from provenance") for path, expected in manifest["modules"].items(): content = (root / path).read_bytes() - tree = ast.dump(ast.parse(content), include_attributes=False).encode() if digest(content) != expected["published_sha256"]: raise ValueError(f"Published source changed: {path}") - if digest(tree) != expected["program_ast_sha256"]: - raise ValueError(f"Program AST changed: {path}") + ast.parse(content, filename=path) if digest((root / "PROTOCOL.md").read_bytes()) != manifest["protocol_sha256"]: raise ValueError("Historical protocol changed") print(f"Verified {len(paths)} published modules and historical protocol")