diff --git a/.github/workflows/structured-delta.yml b/.github/workflows/structured-delta.yml new file mode 100644 index 0000000..002ed1d --- /dev/null +++ b/.github/workflows/structured-delta.yml @@ -0,0 +1,38 @@ +name: Structured Delta prototype + +on: + push: + paths: + - "prototypes/structured-delta/**" + - ".github/workflows/structured-delta.yml" + pull_request: + paths: + - "prototypes/structured-delta/**" + - ".github/workflows/structured-delta.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python: ["3.11", "3.14"] + defaults: + run: + working-directory: prototypes/structured-delta + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - run: python -m pip install ruff==0.16.10 + - run: ruff check . && ruff format --check . + - run: python tools/verify_source.py + - run: python -m unittest discover -s tools -p 'test_*.py' -v + - run: python -m unittest discover -s experiments/tests -v + - run: python tools/run_pilot.py --out artifacts/ci-pilot diff --git a/README.md b/README.md index d29ef90..8aa9ffd 100644 --- a/README.md +++ b/README.md @@ -524,3 +524,10 @@ spellings of either would mean a hook that never finds its own daemon, and a dae finds a reason not to start a second one. MIT licensed. + +## Structured Delta research prototype + +A standalone Python implementation and reproduction guide are available in +[prototypes/structured-delta](prototypes/structured-delta/README.md). +See [scope and evidence](docs/STRUCTURED-DELTA.md); this prototype is separate +from the product runtime, and generated experimental artifacts stay local. diff --git a/README.zh-cn.md b/README.zh-cn.md index 40dd701..81b6375 100644 --- a/README.zh-cn.md +++ b/README.zh-cn.md @@ -467,3 +467,10 @@ npm run typecheck 而 daemon 永远找不到一个"不必再起一个"的理由。 MIT 许可。 + +## 结构化 Delta 研究原型 + +独立 Python 实现、测试和复现入口见 +[prototypes/structured-delta](prototypes/structured-delta/README.md)。 +[方案范围与证据](docs/STRUCTURED-DELTA.md)说明现有结果及限制； +原型尚未接入产品运行流程，生成的实验数据与结果保留在本地。 diff --git a/docs/STRUCTURED-DELTA.md b/docs/STRUCTURED-DELTA.md new file mode 100644 index 0000000..36cf24e --- /dev/null +++ b/docs/STRUCTURED-DELTA.md @@ -0,0 +1,32 @@ +# Structured program-state delta prototype + +The standalone implementation is in +[prototypes/structured-delta](../prototypes/structured-delta/README.md). +It extracts changes from common-base Python snapshots and uses deterministic +dependency, contract and effect rules to classify concurrent interactions. +It is not integrated into the product's CLI, hooks, routing or acceptance flow. + +The published code includes regression tests and an isolated pilot reproduction +runner. Source has consistent formatting and unused imports removed; AST +provenance is recorded. Original frozen source and experimental evidence remain +unaltered in local research storage. Generated fixtures, results and real-repository +reviewer materials are not tracked in Git. + +The 48-case authored pilot achieved conflict F1 74.3%, with 13 true detections, +6 false positives and 3 missed conflicts. It detected 8/11 cross-file conflicts. +The online pilot showed no positive warning lead. Embedding and LLM baselines +were not run. These findings support further independent evaluation, not a +real-world superiority claim. + +Real-repository blind-transfer preparation selected 70 eligible pairs, below its +100-pair target. Packaging and independent labeling are incomplete; no blind-test +scores exist. The next step is independent annotation and adjudication with +executable branch requirements before detector scoring. + +To validate the published implementation, run from its directory: + +```bash +python3 tools/verify_source.py +python3 -m unittest discover -s experiments/tests -v +python3 tools/run_pilot.py --out artifacts/pilot-001 +``` diff --git a/prototypes/structured-delta/.gitignore b/prototypes/structured-delta/.gitignore new file mode 100644 index 0000000..c944428 --- /dev/null +++ b/prototypes/structured-delta/.gitignore @@ -0,0 +1,13 @@ +__pycache__/ +*.py[cod] +.ruff_cache/ +.venv/ +artifacts/ +experiments/benchmark/cases/ +experiments/results/ +FREEZE.json +blind-transfer-*/private/ +blind-transfer-*/reviewer/ +.env +.env.* +!.env.example diff --git a/prototypes/structured-delta/PROTOCOL.md b/prototypes/structured-delta/PROTOCOL.md new file mode 100644 index 0000000..df85e8b --- /dev/null +++ b/prototypes/structured-delta/PROTOCOL.md @@ -0,0 +1,165 @@ +# Structured program-state delta pilot, protocol v1 + +Date: 2026-10-07. This protocol and all code/case bytes are SHA-256 frozen in +FREEZE.json before the first scored run. Development unit tests and fixture +validation precede freezing; no scored results have been used to choose rules. +Do not edit frozen inputs to improve run-001. Retain failed runs and executor +issues; subsequent changes require a separately versioned protocol/directory. + +## Question and scope + +Can deterministic structured deltas improve detection and localization of +concurrent semantic interactions relative to scope/text heuristics? Only a +Python research prototype: no AgentGit runtime, routing, repair, persistent +memory, hooks, MCP, or live-agent orchestration. Dependencies are standard +Python >=3.11 and Git >=2.38 (merge-tree --write-tree). + +This is a jointly authored, transparent DEVELOPMENT benchmark. Case author and +rule implementer are the same; it is not blind or held-out. 48 microrepositories, +10 independent / 14 compatible / 8 redundant / 16 conflicting, with 36 named +families (names organize analysis, not proof of independent sampling). The +initial sample covers the requested ten situations and includes unsupported +features. Scaling the same templates does not create independent evidence. + +## Inputs, labels and ground truth + +Each directory holds base/, patch_A.diff, patch_B.diff, materialized a/ and b/, +gold.json with interaction label/entity/explanation, and checks/{base,a,b}.py. +Nonconflicting cases include witness/ showing a state that satisfies both sets +of branch requirements. Redundant witnesses use A's state. Gold denotes: + +- independent: no shared behavioral requirement or resolved interaction entity; +- compatible: related scopes/effects can preserve both branch requirements; +- redundant: equivalent change over the case's declared input domain; +- conflicting: the unchanged concurrent transformations break at least one + requirement, or require contradictory contracts for the same entity. + +Textual overlap alone does not decide the label. Checks are finite witnesses, +not proofs of universal equivalence. Base validation checks syntax; branch +checks establish local required behavior. All A and B states must pass their +own checks; every provided joint witness must pass both. Patch application +must reproduce the supplied branch snapshots exactly. No benchmark execution +failure is silently dropped. All 48 cases remain in scoring. + +Detector API accepts only base and changed source snapshots. It has no gold, +requirements, test results, witness, case ID, family, or edit description input. +Gold and checks are accessed after static predictions; Git baseline receives +only Git clean/conflict status, never test outcomes. Tests are isolated Python +subprocesses. This is not a repair experiment. There are no hidden tests and no +claim of hidden-quality generalization. + +## Methods and frozen choices + +B1 file overlap; B2 line overlap and qualified AST symbol overlap (reported +separately); B3 Git merge-tree conflict. These binary heuristics emit conflicting +on overlap/conflict and independent otherwise. They cannot emit compatible or +redundant. Report four-class macro F1 as requested, but do not infer superiority +from macro F1 alone: binary precision/recall and compatible false positives are +necessary comparisons. B2 AST means enclosing changed AST definitions, not a +full tree-edit-distance algorithm. Line positions use common-base lines and +half-integer insertion anchors. + +B4: opt-in external diff embedding via JSON command, cosine >=0.80 predicts +conflicting; no threshold selection on this benchmark, no fake local embedding. +No configured command means NOT RUN, not a score of zero. B5: opt-in command +first describes each actual diff, then judges only the descriptions, returning +one of four labels and up to three entities. This is a description bottleneck, +not a full-source LLM baseline. It is not actual live agents' self-reported +intent. Raw descriptions, judgments, reported usage, model ID and errors are +retained. Missing usage is unknown. Partial runs report coverage and are not +ranked against full-coverage methods. No external command is configured in v1. + +Structured delta: Python AST definitions, create/delete/body-equal rename, +statement edit operations, resolved local/relative imports and aliases, calls, +function signatures/annotations, literal return types/dict keys, module globals +and constant-key read/write effects, requirements.txt entries and flat JSON. +Identity-test normalization proves only a very narrow form of redundancy; it +does not equate == with is or perform arbitrary algebraic equivalence. + +Rule priority: canonical transformation equality => redundant; deletion/rename +against changed implementations or consumers => conflict; incompatible observed +call signature or changed return/value contract consumed by another patch => +conflict; (Full only) changed read/write interference, divergent return behavior +or signature replacements and duplicate definitions => conflict; resolved shared +scope/dependency otherwise => compatible; otherwise independent. Scores are +ordinal rule strengths, not probabilities or cosine values. + +Conservative contract rules may reject safe schema/config changes. Calls/reads +include before and after observations; obsolete assumptions can therefore +remain. No alias heap analysis, CFG/path sensitivity, general type inference, +transitive effect analysis, dynamic lookup/JSON consumer resolution or version +solver. Unresolved observations are retained; forced four-class output is not +a safety guarantee. Constant-key analysis is shallow. Source annotations are +observations, not enforced runtime types. Package manifest extraction does not +resolve installed distributions to module imports. + +Ablation ladder: File; File+Symbol; File+Symbol+Dependency; +File+Symbol+Dependency+Contract; Full Structured Delta. First three use binary +intersection rules; Contract uses incompatibility and compatible fallback; Full +adds actual write effects, divergent returns and canonical redundancy. These +are rule/feature bundle ablations, not a trained model with one fixed decoder. +Effects cannot be interpreted as isolated causal contributions of features. + +## Metrics and falsification criterion + +Four-class macro F1 (all four classes, undefined class F1=0), conflict precision +and recall, compatible->conflicting rate, cross-file conflict recall (changed +file sets are disjoint), entity Top-1 and Recall@3. Localization is exact entity +identity, independent of class correctness, denominator all non-independent +cases with gold entities. File-only predictions do not earn symbol localization +credit. A file-level localization score is not substituted for the entity metric. +One gold entity per case in v1, so Recall@3 is a hit rate. Undefined denominators +are null. Report counts, per-class confusion, every family, all FP/FN, not only +aggregate. Report binary conflict F1 alongside mandatory metrics. + +Bounded positive support requires Full conflict F1 to exceed the strongest +available B1/B2/B3 conflict F1, reduce compatible false positives, and improve +cross-file conflict recall. Failure of those criteria does not support even the +bounded claim. Meeting them supports only these authored cases, not general +semantic correctness, comparison to unavailable B4/B5, or real-world superiority. +No p-value based generalization claim is planned for this constructed sample. + +## Commutativity + +Strict attempt: git apply A then B and B then A, with atomic patch application, +recording both apply statuses, both branch requirement results if applicable, +and exact full-source SHA-256 state equality when both orders apply. A failed +order has tests=null and resulting-state comparison=null, never a pass. + +Separately record true Git three-way merge-tree in both parent orders and the +same checks and state hashes. This is a base-aware textual reconciliation, +not function composition or automatic repair. Preserve conflict-marked trees +for inspection. Text application or three-way noncommutativity is an interaction +signal only: compatible edits may overlap, and commuting cross-file edits may +break runtime tests. Never derive gold or structured predictions from it. + +## Online protocol + +12 preregistered cases, 6 interleaved events each: A1 B1 A2 B2 A3 B3. A1/B1 add +an unused helper; A2/B2 apply the first changed file; A3/B3 complete the remaining +files and remove helper. These are synthetic progress scaffolds, not natural +agent trajectories. All incremental diffs are stored and checked against their +snapshots. Active delta is always base->current branch, not only latest diff. +After every event recompute prediction and trial merge both active branches. +Requirements activate as soon as a branch is functionally complete (ignoring +progress helper), not merely at A3/B3, preventing artificial positive lead. +Always check syntax. Unfinished branches have no extra behavior assertions. + +Record first predicted conflict, first merge failure, first clean-merge test +failure, failure minus detection in event ticks (positive=earlier detection, +negative=late); null if unavailable. Report semantic-test lead separately and +real detector wall time in ms. False intervention = ever warning on final +nonconflicting gold; false early intervention additionally precedes first failure +or occurs when no failure is observed. Both definitions are reported so textual +failures do not legitimize warnings on compatible edits. Final labels are a +limited proxy for transient ground truth. Online run starts only after static +pairwise experiment succeeds. No repair or context-routing decisions occur. + +## Most important next experiment + +Freeze the detector, then independently collect and annotate >=100 real +concurrent edit pairs from repositories excluded from development, with executable +branch/joint requirements and independent adjudication. Include compatible +contract changes and dynamic cross-file dependencies; lock model providers and +thresholds on a separate calibration set for B4/B5. This one blinded transfer +test is more informative than expanding authored templates or adding routing. diff --git a/prototypes/structured-delta/README.md b/prototypes/structured-delta/README.md new file mode 100644 index 0000000..a8cf1d5 --- /dev/null +++ b/prototypes/structured-delta/README.md @@ -0,0 +1,101 @@ +# Structured program-state delta prototype + +This standalone Python prototype detects interactions between two changes from a +common source snapshot. It uses AST symbols, import/call dependencies, signatures, +limited return contracts and state read/write observations to classify changes as +`independent`, `compatible`, `redundant` or `conflicting` and locate affected entities. +It is separate from AgentGit's product runtime and has no automatic repair or +live-agent integration. + +Requirements: Python **>=3.11** and Git **>=2.38**. Runtime dependencies are entirely +in the Python standard library; no API keys or external services are needed. + +## Use the detector + +Run from this directory: + +```python +from experiments.delta.extractor import extract +from experiments.delta.interaction import detect + +base = {"api.py": "def fetch(): return 1\n", "client.py": "VALUE = 1\n"} +a = {**base, "api.py": "def load(): return 1\n"} +b = {**base, "client.py": "from api import fetch\ndef run(): return fetch() + 1\n"} + +prediction = detect(extract(base, a), extract(base, b)) +print(prediction.to_dict()) +# label: conflicting; entity: api.py::fetch +``` + +Inputs are dictionaries of repository-relative paths to source text. Detection +accepts source snapshots only, not ground truth, tests or edit descriptions. +Rule scores are ordinal strengths, not probabilities. Unsupported dynamic +lookups and runtime configuration relationships may remain unresolved. + +## Validate and reproduce the development pilot + +```bash +cd prototypes/structured-delta +python3 tools/verify_source.py +python3 -m unittest discover -s experiments/tests -v +python3 tools/run_pilot.py --out artifacts/pilot-001 +``` + +The runner creates a fresh, ignored directory, copies the published source and +original protocol, generates 48 fixtures, validates branch requirements and +joint witnesses, freezes inputs, runs static and online evaluations, then writes +`REPORT.md`. Existing output directories are rejected. Full case files, patches, +checks, logs, freezes and results stay in that directory; they are not tracked. +This is a reproduction using the cleaned source, not a rewrite of the historical +frozen run. The original protocol's "36 named families" is a narrative counting +error: the generator produces **35** families. + +The historical development pilot had conflict F1 **74.3%**, conflict precision +**68.4%**, recall **81.2%**, compatible false positives **3/14**, cross-file conflict +recall **8/11**, and entity Top-1 **34/38**. Its authored cases and rules shared an +author and were not a blind holdout. The online pilot detected 5/6 conflicts with +zero positive lead and 2/6 false interventions. These are limited development +results, not evidence of real-repository generalization or superiority to an LLM. + +`SOURCE_PROVENANCE.json` records original and published source hashes and AST +digests. Formatting and removal of seven unused imports are the only changes to +the published implementation. Historical source, frozen inputs and results are +preserved locally. [PROTOCOL.md](PROTOCOL.md) is the unmodified original protocol. +Verification checks exact source bytes and parses syntax on each supported Python +version. The historical AST digests were generated with Python 3.14 and document +the same-interpreter comparison; `ast.dump()` is not a cross-version format. + +## Optional model baselines + +The embedding and LLM description adapters in `experiments/baselines/` only invoke +an explicitly supplied JSON provider command. They do not discover credentials or +call a service automatically. Without a provider they report `not_configured`. +Provider commands receive JSON on stdin and return JSON on stdout; their module +docstrings describe the formats. Model IDs, available usage, errors and coverage +are retained. The pilot runner does not enable model providers. + +## Layout and blind-transfer status + +| Path | Purpose | +| --- | --- | +| `experiments/delta/` | Structured observations and deterministic interaction rules | +| `experiments/baselines/` | File, line, symbol, Git and optional model comparisons | +| `experiments/benchmark/` | Development fixture generator | +| `experiments/evaluation/` | Validation, freezing, metrics and reporting | +| `experiments/online/` | Synthetic incremental simulation | +| `experiments/tests/` | Detector, metric and Git regression tests | +| `tools/` | Source provenance verification and isolated reproduction | +| `artifacts/` | Locally generated data and results; ignored by Git | + +A separate real-repository annotation preparation selected **70** eligible pairs +under its preregistered sampling limits, below the target of 100. Packaging was +interrupted and independent annotation has not started. There are no blind-test +scores. Repository caches, source archives, provenance mappings and reviewer +materials are local research artifacts and are excluded from this publication. + +For style checks, use Ruff 0.16.10: + +```bash +ruff check . +ruff format --check . +``` diff --git a/prototypes/structured-delta/SOURCE_PROVENANCE.json b/prototypes/structured-delta/SOURCE_PROVENANCE.json new file mode 100644 index 0000000..1f48661 --- /dev/null +++ b/prototypes/structured-delta/SOURCE_PROVENANCE.json @@ -0,0 +1,164 @@ +{ + "source_date": "2026-10-07", + "original_freeze_sha256": "05c32a8d8dbbefbe5408b7a70c25ff4abdf40ac023a2c2335d0a4569a16d0970", + "protocol_sha256": "54cad569cf2242073884738a5ce229db24cfd48abaed73a1adfb6e5bd363a528", + "changes": "Ruff formatting and seven unused imports removed; all remaining ASTs identical to frozen originals.", + "modules": { + "experiments/__init__.py": { + "original_sha256": "303e4a013112bc04899edb2d66680d4856e30b79258f3cc28333e99e973e3b78", + "published_sha256": "303e4a013112bc04899edb2d66680d4856e30b79258f3cc28333e99e973e3b78", + "program_ast_sha256": "08febec1203888913739133cf412415dde251f771db1982c0b03cb1d76da1a48", + "removed_unused_imports": [] + }, + "experiments/baselines/__init__.py": { + "original_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "published_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "program_ast_sha256": "ad2e13b69c4fc1fda46413f740f426fc1793cfc6d6da8d226ba619f1aef48be7", + "removed_unused_imports": [] + }, + "experiments/baselines/ast_overlap.py": { + "original_sha256": "3eb04806389f0f4443fc09bda741d60ae924afd87d659dc7eb7cd9ff6c20bc93", + "published_sha256": "6a156b104c5e05c8122fa8c786cad789d9634abac1bf2d746caece1b4026a4d4", + "program_ast_sha256": "0b7e2c55a217974d596aa9ff232db6c1b057c717a4b05a10a53550680869e64b", + "removed_unused_imports": [] + }, + "experiments/baselines/embedding.py": { + "original_sha256": "b55e9cfb9bb17e22fcd6029f55fb8ba314f46754abd8aa87e899a944ea5f6d90", + "published_sha256": "b21a809416fa0a6e43f63200c68d25884140c45912ef0794999b3fc9ffb49444", + "program_ast_sha256": "4a94ba0546613ca159fc970635bb9c4a7fff4f34b1ed851d5dcbdb6f396f7ebe", + "removed_unused_imports": [] + }, + "experiments/baselines/file_overlap.py": { + "original_sha256": "943457daba77dd5780eaf6a3b3221fafed16f205859c24669827df1815b9a8ef", + "published_sha256": "5cf29d56660981a00207e4884f7a3e8e7f5ed3c787615f2099b7a950ad94e790", + "program_ast_sha256": "dc4f1e8ecdb2e7122b2ff748efbaee50dd46da64dc49762f1d47f1b8dc320dd5", + "removed_unused_imports": [] + }, + "experiments/baselines/git_merge.py": { + "original_sha256": "9890f74b9c015ea12653b86c6deb2389f47371dbd72c83ed304ddc4f3cebeac5", + "published_sha256": "e2233dce3b295fad4fd135fb35021fde1d660d11471e643eb3dd6f3ed0dce3ce", + "program_ast_sha256": "c2cb7f664b29339c2b701dd91aed1a248e4726b4f3e4c08f505af8d97c10a6a9", + "removed_unused_imports": [] + }, + "experiments/baselines/llm_intent.py": { + "original_sha256": "77cb7a058683aa5e381fef8214dc1c14253a90e916ccaa9eff93c896214e4a5b", + "published_sha256": "e43dc098d8da1c29d0b2fe8b6c76b48601a917605a17e531f4d64d8934d08054", + "program_ast_sha256": "e4a4f3d4acbb4f6f6ac6fe80031f8e3695844e40d695b32b25a39391af0de19e", + "removed_unused_imports": [] + }, + "experiments/benchmark/build.py": { + "original_sha256": "472a22aab9a9aa33011fd62791ab94e3242f110657ad178f93f7a57471501410", + "published_sha256": "ad197e5ef0f3e5b8a6d557dcb1e00dec707f6a449dc7763fd192cef1813c3718", + "program_ast_sha256": "8ba61efbe04df540bfbe802c32ad6ad3f4ce2395ee49be2f9caabb2848e24cfc", + "removed_unused_imports": [] + }, + "experiments/delta/__init__.py": { + "original_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "published_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "program_ast_sha256": "ad2e13b69c4fc1fda46413f740f426fc1793cfc6d6da8d226ba619f1aef48be7", + "removed_unused_imports": [] + }, + "experiments/delta/extractor.py": { + "original_sha256": "5d5f06bdda32007ba6f34257ffb368bef6b5125a2fa52916aeb2f70d4308b2fb", + "published_sha256": "ed271ab1b05dbfbb7a26b2c80f2dd8603abf67b6079553de74fa237dfe00898a", + "program_ast_sha256": "02a62047c8d0719b4fc90f68b697e5beb7b24363ba8d438ee4de8b71a60bd609", + "removed_unused_imports": [] + }, + "experiments/delta/interaction.py": { + "original_sha256": "34ffdd983956a7e00ed0e52a684094c0bac8499ede08cf111ab36a86bdd8ce4f", + "published_sha256": "010b90920b702c7fbf03936290d060abe2ff08cdce24d53e3f5a803c11db1754", + "program_ast_sha256": "3def3e6dc452130156c5f2f60d79b0419c1c9cec7d35ac654d618013a56943fe", + "removed_unused_imports": [] + }, + "experiments/delta/model.py": { + "original_sha256": "7463425ea8bbf61e9a39968802a511991b91e43a7cb8eaf1492f625ae80da7a0", + "published_sha256": "5462a45f1951fac2820e2bf410a61a946da464c4a73d63a80b6c89a7a05a2f3b", + "program_ast_sha256": "1e70d37045f7587f0ce64713a04f1594c72a2b3fb8838264248f4461fefa5ccc", + "removed_unused_imports": [] + }, + "experiments/evaluation/__init__.py": { + "original_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "published_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "program_ast_sha256": "ad2e13b69c4fc1fda46413f740f426fc1793cfc6d6da8d226ba619f1aef48be7", + "removed_unused_imports": [] + }, + "experiments/evaluation/evaluate.py": { + "original_sha256": "a396889a515954261702718335d642621ae0b2165b84ad36263eaef2f2cbf666", + "published_sha256": "2a0fa4d4f2840f158ed99907b0e23e6a8ba2c67e9097637c7c1a2005562a6ad8", + "program_ast_sha256": "a630c9fc72907145eaadc7a105a7b597c50313c4c957eb6587f266a81da1b349", + "removed_unused_imports": [] + }, + "experiments/evaluation/freeze.py": { + "original_sha256": "969f7987af75373662d6b42fa4f12e4e05c523ad3f115aa2dcc13753f2e37807", + "published_sha256": "4df8b069f78ad80c4f8dcc9b24208dbf87bbeced6cf08dee456e6b3190df4bf9", + "program_ast_sha256": "afd3ac3fc1dd44c70ed5ec26e77d809b807cd993e5614767eb17e3892a5f2915", + "removed_unused_imports": [] + }, + "experiments/evaluation/metrics.py": { + "original_sha256": "acdf0062997ee8a1584e8402b741d8598ec9812a1dc8581888f144c79d024401", + "published_sha256": "5c452bc86751a0ba493cf98d52c7187fbfcc7ca028f9b420ca165ce3fc02a4a6", + "program_ast_sha256": "759a410c56eb9f18f1dd7a1ee6dd7bafe6253ff7787c52af52543162720e08b2", + "removed_unused_imports": [] + }, + "experiments/evaluation/report.py": { + "original_sha256": "35c54e5999c957ad63d9f39632718b13621889a6410bf09ebece269732c880b8", + "published_sha256": "707b52d43190a4f4dbc9dc14d34f5f0251204cb473fc8eff9ffa4d0f5222130c", + "program_ast_sha256": "2f0dc50f9c0d28b19d5f439043f21f083cd43fc23d1c754b6aa74a8d453b1ca9", + "removed_unused_imports": [ + "Counter" + ] + }, + "experiments/evaluation/runtime.py": { + "original_sha256": "11afc99694ac12b49696731dcb3218e898599a405218c83f53a3646befe9ac41", + "published_sha256": "e0fe979c533c120d0008ed6a250a78dca3839eab050fcfdb7fded4a4a4a66d77", + "program_ast_sha256": "1be4c2dfeca897ee8f272a4bb287fdbb56ff4f491589c7680131ac8d9ce3784f", + "removed_unused_imports": [ + "io", + "tarfile" + ] + }, + "experiments/evaluation/validate.py": { + "original_sha256": "88d08dd2316c2473bff4e3549c9dec2f33e04b3f743077eccc3ab1759ac58174", + "published_sha256": "cca64706b27c61ef770c5f999e7adab0450594da40987cf9794d5a78efe0e30f", + "program_ast_sha256": "63c626297e04963dc88ed91c3752fc8313456e0e83fd9cd129a382fc16160a48", + "removed_unused_imports": [] + }, + "experiments/online/__init__.py": { + "original_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "published_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "program_ast_sha256": "ad2e13b69c4fc1fda46413f740f426fc1793cfc6d6da8d226ba619f1aef48be7", + "removed_unused_imports": [] + }, + "experiments/online/sequences.py": { + "original_sha256": "fad006138e17606d9022f6c28d26c0c5aa5a0d71176b9bac8863d4a63b3083c5", + "published_sha256": "50742fb4b5f04991dca4ca14c634d2e10ef59bdf5a8a2c3c11394245aa0064a8", + "program_ast_sha256": "6a3f42759362bd013048093a9ba82918006f58c3e91f22bbb0af54b726b2a41b", + "removed_unused_imports": [] + }, + "experiments/online/simulator.py": { + "original_sha256": "1b2efa1d5738dda82ccfbdeee8574fc8477f53d5a9d8c2e7124e5084b71adbbc", + "published_sha256": "c654804584b4f8f152d08278c39127c7365970b9b6ab4b93125af2c655319a8e", + "program_ast_sha256": "2b450ec1cd9eb46906c44d8198e8c9d5ca66a90a104ea5c07926949c83a8a86e", + "removed_unused_imports": [ + "patch" + ] + }, + "experiments/tests/__init__.py": { + "original_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "published_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "program_ast_sha256": "ad2e13b69c4fc1fda46413f740f426fc1793cfc6d6da8d226ba619f1aef48be7", + "removed_unused_imports": [] + }, + "experiments/tests/test_prototype.py": { + "original_sha256": "e3ccded56a270097564000319388727e5e063bfe25841b0b7beb09d0172e07c6", + "published_sha256": "16b2fad62f0362c5b38f82cc38794c8ee13acd36f9ddaa116b2d94a4f299245b", + "program_ast_sha256": "55e891e729a60f09681ebc7abf87a2466754df43d41617d8faa44f6bedc24124", + "removed_unused_imports": [ + "Path", + "json", + "tempfile" + ] + } + }, + "ast_digest_python": "3.14" +} diff --git a/prototypes/structured-delta/experiments/__init__.py b/prototypes/structured-delta/experiments/__init__.py new file mode 100644 index 0000000..ba7f492 --- /dev/null +++ b/prototypes/structured-delta/experiments/__init__.py @@ -0,0 +1 @@ +"""Standalone structured program-delta experiment; no AgentGit runtime integration.""" diff --git a/prototypes/structured-delta/experiments/baselines/__init__.py b/prototypes/structured-delta/experiments/baselines/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prototypes/structured-delta/experiments/baselines/ast_overlap.py b/prototypes/structured-delta/experiments/baselines/ast_overlap.py new file mode 100644 index 0000000..c0f669c --- /dev/null +++ b/prototypes/structured-delta/experiments/baselines/ast_overlap.py @@ -0,0 +1,18 @@ +from ..delta.model import Prediction + + +def predict(a, b, mode="symbol"): + if mode == "line": + entities = [ + f"file:{p}" + for p in sorted(set(a.files) & set(b.files)) + if set(a.lines[p]) & set(b.lines[p]) + ] + else: + entities = sorted(set(a.symbols) & set(b.symbols)) + return Prediction( + "conflicting" if entities else "independent", + entities[:3], + float(bool(entities)), + [mode + " overlap"], + ) diff --git a/prototypes/structured-delta/experiments/baselines/embedding.py b/prototypes/structured-delta/experiments/baselines/embedding.py new file mode 100644 index 0000000..fa614f5 --- /dev/null +++ b/prototypes/structured-delta/experiments/baselines/embedding.py @@ -0,0 +1,50 @@ +"""Optional external embedding baseline; never replaced with a fake embedding. + +Provider command receives JSON on stdin: {task: 'embed', texts:[diffA,diffB]}. +Return {vectors:[[...],[...]], model: str, usage: object|null} on stdout. +The provider owns auth/network. No automatic model discovery or invocation. +""" + +import json +import math +import shlex +import subprocess +from ..delta.model import Prediction + + +def invoke(command, payload): + process = subprocess.run( + shlex.split(command), + input=json.dumps(payload), + text=True, + capture_output=True, + timeout=180, + check=True, + ) + return json.loads(process.stdout) + + +def predict(command, diff_a, diff_b, a, b, threshold=0.80): + raw = invoke(command, {"task": "embed", "texts": [diff_a, diff_b]}) + x, y = raw["vectors"] + if ( + not x + or len(x) != len(y) + or not all(isinstance(v, (float, int)) and math.isfinite(v) for v in x + y) + ): + raise ValueError("Invalid embedding vectors") + norm = math.sqrt(sum(v * v for v in x) * sum(v * v for v in y)) + if not norm: + raise ValueError("Zero embedding norm") + sim = sum(u * v for u, v in zip(x, y)) / norm + # This is intentionally a similarity baseline, not our definition of conflict. + candidates = sorted(set(a.symbols) & set(b.symbols)) + [ + "file:" + p for p in sorted(set(a.files) & set(b.files)) + ] + return Prediction( + "conflicting" if sim >= threshold else "independent", + candidates[:3], + sim, + ["Fixed embedding cosine threshold"], + {"model": raw.get("model"), "usage": raw.get("usage"), "threshold": threshold}, + ) diff --git a/prototypes/structured-delta/experiments/baselines/file_overlap.py b/prototypes/structured-delta/experiments/baselines/file_overlap.py new file mode 100644 index 0000000..a9dbc9d --- /dev/null +++ b/prototypes/structured-delta/experiments/baselines/file_overlap.py @@ -0,0 +1,11 @@ +from ..delta.model import Prediction + + +def predict(a, b): + entities = ["file:" + p for p in sorted(set(a.files) & set(b.files))] + return Prediction( + "conflicting" if entities else "independent", + entities[:3], + float(bool(entities)), + ["File overlap"], + ) diff --git a/prototypes/structured-delta/experiments/baselines/git_merge.py b/prototypes/structured-delta/experiments/baselines/git_merge.py new file mode 100644 index 0000000..878ee65 --- /dev/null +++ b/prototypes/structured-delta/experiments/baselines/git_merge.py @@ -0,0 +1,11 @@ +from ..delta.model import Prediction + + +def predict(merge_result): + entities = ["file:" + p for p in merge_result["conflict_paths"]] + return Prediction( + "conflicting" if not merge_result["clean"] else "independent", + entities[:3], + float(not merge_result["clean"]), + ["Git merge-tree textual/structural merge"], + ) diff --git a/prototypes/structured-delta/experiments/baselines/llm_intent.py b/prototypes/structured-delta/experiments/baselines/llm_intent.py new file mode 100644 index 0000000..b885674 --- /dev/null +++ b/prototypes/structured-delta/experiments/baselines/llm_intent.py @@ -0,0 +1,44 @@ +"""Optional two-stage edit-description baseline, based on actual diffs, not gold. + +Provider command accepts task='describe' / task='judge', returning description +or label/entities, plus model and usage. Prompts and raw outputs are persisted. +""" + +from .embedding import invoke +from ..delta.model import Prediction, LABELS + + +def predict(command, diff_a, diff_b): + traces = [] + descriptions = [] + for diff in (diff_a, diff_b): + request = { + "task": "describe", + "instruction": "Describe the behavior of this code edit using only the diff. Identify affected entities as path::symbol or dependency:name. Do not assume unstated intent.", + "diff": diff, + } + response = invoke(command, request) + if not isinstance(response.get("description"), str): + raise ValueError("Missing edit description") + descriptions.append(response["description"]) + traces.append({"request": request, "response": response}) + request = { + "task": "judge", + "instruction": "Judge concurrent changes from a common base. Return label: independent, compatible, redundant, or conflicting; and up to 3 interaction entities. Same scope is not sufficient for conflict.", + "descriptions": descriptions, + } + response = invoke(command, request) + traces.append({"request": request, "response": response}) + if ( + response.get("label") not in LABELS + or not isinstance(response.get("entities"), list) + or not all(isinstance(e, str) for e in response["entities"]) + ): + raise ValueError("Invalid model judgment") + return Prediction( + response["label"], + response["entities"][:3], + 0, + ["LLM edit descriptions + judgment"], + {"traces": traces}, + ) diff --git a/prototypes/structured-delta/experiments/benchmark/build.py b/prototypes/structured-delta/experiments/benchmark/build.py new file mode 100644 index 0000000..826a136 --- /dev/null +++ b/prototypes/structured-delta/experiments/benchmark/build.py @@ -0,0 +1,968 @@ +"""48 authored micro-repositories. Extend with another add() call; no case cap. + +Gold and witness code live only in benchmark artifacts. Detectors do not import +this module. Cases are a development benchmark, NOT independently held-out data. +""" + +import argparse +import difflib +import json +import textwrap +from collections import Counter +from pathlib import Path + +CASES = [] + + +def clean(s): + return textwrap.dedent(s).lstrip("\n").rstrip() + "\n" + + +def add( + title, + label, + family, + base, + a, + b, + entity, + why, + check_a, + check_b, + check_base="", + witness=None, + online=False, +): + base = {p: clean(s) for p, s in base.items()} + + def version(changes): + out = dict(base) + for p, s in changes.items(): + if s is None: + out.pop(p, None) + else: + out[p] = clean(s) + return out + + aa, bb = version(a), version(b) + CASES.append( + dict( + id=f"case{len(CASES) + 1:03}", + title=title, + label=label, + family=family, + base=base, + a=aa, + b=bb, + entities=[] if entity is None else [entity], + explanation=why, + checks={ + "base": clean(check_base), + "a": clean(check_a), + "b": clean(check_b), + }, + witness=version(witness) + if witness is not None + else (aa if label == "redundant" else None), + online=online, + ) + ) + + +def cases(): + if CASES: + return CASES + # Independent: varied scopes, including same-file and shared dictionary roots. + add( + "Unrelated arithmetic modules", + "independent", + "unrelated_files", + {"left.py": "def inc(x): return x+1", "right.py": "def twice(x): return x*2"}, + {"left.py": "def inc(x): return x+2"}, + {"right.py": "def twice(x): return x*3"}, + None, + "Different modules have no calls or shared state.", + "from left import inc\nassert inc(1)==3", + "from right import twice\nassert twice(2)==6", + witness={ + "left.py": "def inc(x): return x+2", + "right.py": "def twice(x): return x*3", + }, + online=True, + ) + add( + "Separate functions in one file", + "independent", + "disjoint_symbols", + {"ops.py": "def left(): return 1\n\ndef right(): return 2"}, + {"ops.py": "def left(): return 10\n\ndef right(): return 2"}, + {"ops.py": "def left(): return 1\n\ndef right(): return 20"}, + None, + "No state or call edge between the functions.", + "from ops import left\nassert left()==10", + "from ops import right\nassert right()==20", + witness={"ops.py": "def left(): return 10\n\ndef right(): return 20"}, + ) + add( + "Separate classes", + "independent", + "disjoint_symbols", + { + "model.py": 'class Cat:\n def sound(self): return "mew"\n\nclass Dog:\n def sound(self): return "woof"' + }, + { + "model.py": 'class Cat:\n def sound(self): return "purr"\n\nclass Dog:\n def sound(self): return "woof"' + }, + { + "model.py": 'class Cat:\n def sound(self): return "mew"\n\nclass Dog:\n def sound(self): return "bark"' + }, + None, + "Neither class refers to the other.", + 'from model import Cat\nassert Cat().sound()=="purr"', + 'from model import Dog\nassert Dog().sound()=="bark"', + witness={ + "model.py": 'class Cat:\n def sound(self): return "purr"\n\nclass Dog:\n def sound(self): return "bark"' + }, + ) + add( + "Separate state keys", + "independent", + "field_sensitive", + { + "state.py": 'VALUES={"a":0,"b":0}\ndef set_a(): VALUES["a"]=1\ndef set_b(): VALUES["b"]=1' + }, + { + "state.py": 'VALUES={"a":0,"b":0}\ndef set_a(): VALUES["a"]=2\ndef set_b(): VALUES["b"]=1' + }, + { + "state.py": 'VALUES={"a":0,"b":0}\ndef set_a(): VALUES["a"]=1\ndef set_b(): VALUES["b"]=3' + }, + None, + "Distinct dictionary keys can be assigned independently.", + 'import state\nstate.set_a()\nassert state.VALUES["a"]==2', + 'import state\nstate.set_b()\nassert state.VALUES["b"]==3', + witness={ + "state.py": 'VALUES={"a":0,"b":0}\ndef set_a(): VALUES["a"]=2\ndef set_b(): VALUES["b"]=3' + }, + ) + add( + "Library import unrelated to formatter", + "independent", + "unrelated_files", + { + "calc.py": "def root(x): return x**0.5", + "fmt.py": "def format_name(x): return x", + }, + {"calc.py": "import math\ndef root(x): return math.sqrt(x)"}, + {"fmt.py": "def format_name(x): return x.upper()"}, + None, + "Import has no dependency on formatter.", + "from calc import root\nassert root(9)==3", + 'from fmt import format_name\nassert format_name("a")=="A"', + witness={ + "calc.py": "import math\ndef root(x): return math.sqrt(x)", + "fmt.py": "def format_name(x): return x.upper()", + }, + ) + add( + "Annotation and unrelated new endpoint", + "independent", + "unrelated_files", + {"a.py": "def f(x): return x", "b.py": "def g(): return 0"}, + {"a.py": "def f(x: int) -> int: return x"}, + {"b.py": "def g(): return 0\ndef h(): return 4"}, + None, + "Type annotation has no consumer in other patch.", + "from a import f\nassert f(3)==3", + "from b import h\nassert h()==4", + witness={ + "a.py": "def f(x: int) -> int: return x", + "b.py": "def g(): return 0\ndef h(): return 4", + }, + ) + add( + "Independent configuration keys", + "independent", + "disjoint_config", + {"config.py": 'TIMEOUT=10\nCOLOR="blue"'}, + {"config.py": 'TIMEOUT=20\nCOLOR="blue"'}, + {"config.py": 'TIMEOUT=10\nCOLOR="red"'}, + None, + "No consumer couples timeout and color.", + "from config import TIMEOUT\nassert TIMEOUT==20", + 'from config import COLOR\nassert COLOR=="red"', + witness={"config.py": 'TIMEOUT=20\nCOLOR="red"'}, + ) + add( + "Comment and behavior elsewhere", + "independent", + "nonsemantic", + {"a.py": "# identity\ndef f(x): return x", "b.py": "def g(): return 1"}, + {"a.py": "# Return input unchanged.\ndef f(x): return x"}, + {"b.py": "def g(): return 2"}, + None, + "Comment has no runtime effect.", + "from a import f\nassert f(2)==2", + "from b import g\nassert g()==2", + witness={ + "a.py": "# Return input unchanged.\ndef f(x): return x", + "b.py": "def g(): return 2", + }, + ) + add( + "Same spelling in different modules", + "independent", + "qualified_names", + {"a.py": "VERSION=1", "b.py": "VERSION=1"}, + {"a.py": "VERSION=1\ndef parse(x): return int(x)"}, + {"b.py": 'VERSION=1\ndef parse(x): return x.split(",")'}, + None, + "Qualified symbols differ despite identical names.", + 'from a import parse\nassert parse("2")==2', + 'from b import parse\nassert parse("a,b")==["a","b"]', + witness={ + "a.py": "VERSION=1\ndef parse(x): return int(x)", + "b.py": 'VERSION=1\ndef parse(x): return x.split(",")', + }, + ) + add( + "Delete unused file and edit another", + "independent", + "file_lifecycle", + {"old.py": "UNUSED=1", "live.py": "def value(): return 1"}, + {"old.py": None}, + {"live.py": "def value(): return 2"}, + None, + "Deleted file has no live imports.", + 'from pathlib import Path\nassert not Path("old.py").exists()', + "from live import value\nassert value()==2", + witness={"old.py": None, "live.py": "def value(): return 2"}, + ) + # Compatible: shared scope/edge, both requirements can coexist. + parse = {"parser.py": "import logging\ndef parse(x):\n return int(x)"} + validate = 'import logging\ndef parse(x):\n if x is None: raise ValueError("empty")\n return int(x)' + log = 'import logging\ndef parse(x):\n logging.info("parse")\n return int(x)' + ca = 'from parser import parse\ntry: parse(None)\nexcept ValueError: pass\nelse: raise AssertionError("validation missing")' + cb = 'from unittest.mock import patch\nfrom parser import parse\nwith patch("parser.logging.info") as log:\n assert parse("2")==2\n log.assert_called_once_with("parse")' + add( + "Validation plus logging", + "compatible", + "same_function_additive", + parse, + {"parser.py": validate}, + {"parser.py": log}, + "parser.py::parse", + "Input validation and successful-call logging can both be preserved.", + ca, + cb, + witness={ + "parser.py": 'import logging\ndef parse(x):\n if x is None: raise ValueError("empty")\n logging.info("parse")\n return int(x)' + }, + online=True, + ) + add( + "Overlapping one-line replacements", + "compatible", + "overlapping_lines", + {"parser.py": "import logging\ndef parse(x): return int(x)"}, + {"parser.py": validate}, + {"parser.py": log}, + "parser.py::parse", + "Both replace the same physical line but request independent added effects.", + ca, + cb, + witness={ + "parser.py": 'import logging\ndef parse(x):\n if x is None: raise ValueError("empty")\n logging.info("parse")\n return int(x)' + }, + ) + add( + "Docstring and implementation", + "compatible", + "same_function_additive", + {"ops.py": "def double(x):\n return x*2"}, + {"ops.py": 'def double(x):\n """Multiply by two."""\n return x*2'}, + {"ops.py": "def double(x):\n return x+x"}, + "ops.py::double", + "Documentation and equivalent arithmetic coexist.", + 'from ops import double\nassert double.__doc__=="Multiply by two."', + "from ops import double\nassert double(4)==8", + witness={ + "ops.py": 'def double(x):\n """Multiply by two."""\n return x+x' + }, + ) + add( + "Optional argument and logging", + "compatible", + "signature_compatible", + {"ops.py": "import logging\ndef scale(x):\n return x*2"}, + {"ops.py": "import logging\ndef scale(x, factor=2):\n return x*factor"}, + { + "ops.py": 'import logging\ndef scale(x):\n logging.info("scale")\n return x*2' + }, + "ops.py::scale", + "Default preserves old calls while logging adds an effect.", + "from ops import scale\nassert scale(2,3)==6", + 'from unittest.mock import patch\nfrom ops import scale\nwith patch("ops.logging.info") as p:\n assert scale(2)==4\n p.assert_called_once()', + witness={ + "ops.py": 'import logging\ndef scale(x, factor=2):\n logging.info("scale")\n return x*factor' + }, + ) + add( + "Producer optional parameter and old-shape caller", + "compatible", + "cross_file_compatible", + { + "api.py": "def get(x): return x", + "client.py": "from api import get\ndef run(): return get(1)", + }, + {"api.py": "def get(x, extra=0): return x+extra"}, + {"client.py": "from api import get\ndef run(): return get(2)"}, + "api.py::get", + "New parameter is optional; existing arity remains valid.", + "from api import get\nassert get(2,3)==5", + "from client import run\nassert run()==2", + witness={ + "api.py": "def get(x, extra=0): return x+extra", + "client.py": "from api import get\ndef run(): return get(2)", + }, + online=True, + ) + add( + "Add result key with tolerant consumer", + "compatible", + "contract_compatible", + { + "api.py": 'def get(): return {"id":1}', + "client.py": 'from api import get\ndef run(): return get()["id"]', + }, + {"api.py": 'def get(): return {"id":1,"name":"A"}'}, + {"client.py": 'from api import get\ndef run(): return get()["id"]+1'}, + "api.py::get", + "Consumer accesses retained key; added key is backward compatible.", + 'from api import get\nassert get()["name"]=="A"', + "from client import run\nassert run()==2", + witness={ + "api.py": 'def get(): return {"id":1,"name":"A"}', + "client.py": 'from api import get\ndef run(): return get()["id"]+1', + }, + online=True, + ) + add( + "Config change with tolerant consumer", + "compatible", + "config_compatible", + { + "config.py": "LIMIT=10", + "client.py": "from config import LIMIT\ndef valid(x): return x10: raise ValueError("large")\n return x*x' + }, + "ops.py::square", + "Disjoint rejected domains preserve the common valid domain.", + "from ops import square\ntry: square(-1)\nexcept ValueError: pass\nelse: raise AssertionError()", + "from ops import square\ntry: square(11)\nexcept ValueError: pass\nelse: raise AssertionError()", + witness={ + "ops.py": 'def square(x):\n if x<0: raise ValueError("negative")\n if x>10: raise ValueError("large")\n return x*x' + }, + ) + add( + "Equivalent producer optimization and new consumer", + "compatible", + "dependency_compatible", + { + "api.py": "def double(x: int): return x*2", + "client.py": "from api import double\ndef run(): return double(1)", + }, + {"api.py": "def double(x: int): return x+x"}, + {"client.py": "from api import double\ndef run(): return double(3)"}, + "api.py::double", + "Integer-domain optimization preserves the called behavior.", + "from api import double\nassert double(4)==8", + "from client import run\nassert run()==6", + witness={ + "api.py": "def double(x: int): return x+x", + "client.py": "from api import double\ndef run(): return double(3)", + }, + ) + add( + "Type annotation plus runtime guard", + "compatible", + "annotation_compatible", + {"ops.py": "def length(x):\n return len(x)"}, + {"ops.py": "def length(x: str) -> int:\n return len(x)"}, + { + "ops.py": 'def length(x):\n if x is None: raise ValueError("empty")\n return len(x)' + }, + "ops.py::length", + "Annotation and guard impose no contradictory runtime result.", + 'from ops import length\nassert length.__annotations__["x"] is str', + "from ops import length\ntry: length(None)\nexcept ValueError: pass\nelse: raise AssertionError()", + witness={ + "ops.py": 'def length(x: str) -> int:\n if x is None: raise ValueError("empty")\n return len(x)' + }, + ) + add( + "Compatible dependency patch release", + "compatible", + "dependency_compatible", + { + "requirements.txt": "tiny==1.0", + "tiny.py": "def encode(x): return str(x)", + "client.py": "import tiny\ndef run(): return tiny.encode(1)", + }, + {"requirements.txt": "tiny==1.1"}, + {"client.py": "import tiny\ndef run(): return tiny.encode(2)"}, + "dependency:tiny", + "Local package API remains unchanged; pinned version metadata alone is not breakage.", + 'from pathlib import Path\nassert "1.1" in Path("requirements.txt").read_text()', + 'from client import run\nassert run()=="2"', + witness={ + "requirements.txt": "tiny==1.1", + "client.py": "import tiny\ndef run(): return tiny.encode(2)", + }, + ) + add( + "New keyword-only optional parameter", + "compatible", + "signature_compatible", + { + "api.py": "def get(x): return x", + "client.py": "from api import get\ndef run(): return get(x=1)", + }, + {"api.py": "def get(x, *, trace=False): return x"}, + {"client.py": "from api import get\ndef run(): return get(x=2)"}, + "api.py::get", + "Caller uses an accepted keyword and does not need optional trace.", + "from api import get\nassert get(3,trace=True)==3", + "from client import run\nassert run()==2", + witness={ + "api.py": "def get(x, *, trace=False): return x", + "client.py": "from api import get\ndef run(): return get(x=2)", + }, + ) + # Redundancy: some canonicalized, some intentionally outside the rule subset. + add( + "Equivalent None identity guards", + "redundant", + "null_equivalence", + {"ops.py": "def value(x):\n return x"}, + {"ops.py": "def value(x):\n if x is None: return 0\n return x"}, + {"ops.py": "def value(x):\n if None is x: return 0\n return x"}, + "ops.py::value", + "Identity comparison operand reversal is equivalent.", + "from ops import value\nassert value(None)==0 and value(2)==2", + "from ops import value\nassert value(None)==0 and value(3)==3", + online=True, + ) + add( + "Same logging edit", + "redundant", + "exact_duplicate", + parse, + {"parser.py": log}, + {"parser.py": log}, + "parser.py::parse", + "Both patches add exactly the same single logging call.", + cb, + cb, + ) + add( + "Same default with different formatting", + "redundant", + "ast_equivalence", + {"ops.py": "def f(x): return x"}, + {"ops.py": "def f(x=1): return x"}, + {"ops.py": "def f( x = 1 ):\n return x"}, + "ops.py::f", + "Whitespace differs, signatures and bodies are identical.", + "from ops import f\nassert f()==1", + "from ops import f\nassert f()==1", + ) + add( + "Equivalent conditional branches", + "redundant", + "null_equivalence", + {"ops.py": "def f(x): return x"}, + { + "ops.py": "def f(x):\n if x is None:\n return 0\n else:\n return x" + }, + { + "ops.py": "def f(x):\n if x is not None:\n return x\n else:\n return 0" + }, + "ops.py::f", + "Inverting the identity test and swapping return arms is equivalent.", + "from ops import f\nassert f(None)==0 and f(4)==4", + "from ops import f\nassert f(None)==0 and f(7)==7", + ) + add( + "Equivalent integer addition", + "redundant", + "algebraic_equivalence", + {"ops.py": "def f(x: int): return x"}, + {"ops.py": "def f(x: int): return x+1"}, + {"ops.py": "def f(x: int): return 1+x"}, + "ops.py::f", + "On the declared plain-integer domain addition commutes.", + "from ops import f\nassert all(f(i)==i+1 for i in range(-10,11))", + "from ops import f\nassert f(8)==9", + ) + add( + "Equivalent local variable rename", + "redundant", + "alpha_equivalence", + {"ops.py": "def f(x): return x"}, + {"ops.py": "def f(x):\n result=x*2\n return result"}, + {"ops.py": "def f(x):\n answer=x*2\n return answer"}, + "ops.py::f", + "Only the local temporary name differs.", + "from ops import f\nassert f(3)==6", + "from ops import f\nassert f(4)==8", + ) + add( + "Equivalent string format", + "redundant", + "format_equivalence", + {"ops.py": "def greet(x: str): return x"}, + {"ops.py": 'def greet(x: str): return f"Hello {x}"'}, + {"ops.py": 'def greet(x: str): return "Hello {}".format(x)'}, + "ops.py::greet", + "For string inputs the two formatting forms have identical output.", + 'from ops import greet\nassert greet("A")=="Hello A"', + 'from ops import greet\nassert greet("B")=="Hello B"', + ) + add( + "Duplicate dictionary field addition", + "redundant", + "exact_duplicate", + {"api.py": 'def data(): return {"id":1}'}, + {"api.py": 'def data(): return {"id":1,"ok":True}'}, + {"api.py": 'def data(): return {"id":1,"ok":True}'}, + "api.py::data", + "Both produce the same extended return contract.", + 'from api import data\nassert data()["ok"] is True', + 'from api import data\nassert data()=={"id":1,"ok":True}', + ) + # Conflicting: 16 cases, cross-file dependencies plus expected hard failures. + add( + "Delete class versus extend class", + "conflicting", + "delete_extend", + {"model.py": "class Foo:\n def value(self): return 1"}, + {"model.py": "# Foo removed"}, + { + "model.py": "class Foo:\n def value(self): return 1\n def extra(self): return 2" + }, + "model.py::Foo", + "A requires Foo absent while B requires its new method.", + 'import model\nassert not hasattr(model,"Foo")', + "from model import Foo\nassert Foo().extra()==2", + online=True, + ) + add( + "API rename versus new old-name consumer", + "conflicting", + "rename_use", + {"api.py": "def fetch(): return 1", "client.py": "VALUE=1"}, + {"api.py": "def load(): return 1"}, + {"client.py": "from api import fetch\ndef run(): return fetch()+1"}, + "api.py::fetch", + "B imports an API removed by A.", + 'import api\nassert api.load()==1 and not hasattr(api,"fetch")', + "from client import run\nassert run()==2", + online=True, + ) + add( + "Different functions share mutable scalar", + "conflicting", + "shared_state", + { + "state.py": "COUNT=0\ndef reset():\n global COUNT\n COUNT=0\ndef step():\n global COUNT\n COUNT+=1\n return COUNT" + }, + { + "state.py": "COUNT=0\ndef reset():\n global COUNT\n COUNT=10\ndef step():\n global COUNT\n COUNT+=1\n return COUNT" + }, + { + "state.py": "COUNT=0\ndef reset():\n global COUNT\n COUNT=0\ndef step():\n global COUNT\n assert COUNT<5\n COUNT+=1\n return COUNT" + }, + "state.py::COUNT", + "Reset changes state range; B assumes reset establishes a count below five.", + "import state\nstate.reset()\nassert state.COUNT==10", + "import state\nstate.reset()\nassert state.step()==1", + online=True, + ) + add( + "Cross-file configuration unit", + "conflicting", + "shared_config", + { + "config.py": "TIMEOUT=5", + "client.py": "from config import TIMEOUT\ndef seconds(): return TIMEOUT", + }, + {"config.py": "TIMEOUT=5000"}, + {"client.py": "from config import TIMEOUT\ndef seconds(): return TIMEOUT*2"}, + "config.py::TIMEOUT", + "A stores milliseconds while B continues using seconds.", + "from config import TIMEOUT\nassert TIMEOUT==5000", + "from client import seconds\nassert seconds()==10", + online=True, + ) + add( + "Producer scalar to mapping", + "conflicting", + "return_contract", + { + "api.py": "def value(): return 3", + "client.py": "from api import value\ndef run(): return value()", + }, + {"api.py": 'def value(): return {"value":3}'}, + {"client.py": "from api import value\ndef run(): return value()+1"}, + "api.py::value", + "Numeric consumer cannot add one to mapping return.", + 'from api import value\nassert value()=={"value":3}', + "from client import run\nassert run()==4", + online=True, + ) + add( + "Producer key rename", + "conflicting", + "return_contract", + { + "api.py": 'def user(): return {"name":"A"}', + "client.py": 'from api import user\ndef label(): return user()["name"]', + }, + {"api.py": 'def user(): return {"display":"A"}'}, + { + "client.py": 'from api import user\ndef label(): return user()["name"].upper()' + }, + "api.py::user", + "Old key is absent in the new return mapping.", + 'from api import user\nassert "name" not in user() and user()["display"]=="A"', + 'from client import label\nassert label()=="A"', + ) + add( + "New required positional argument", + "conflicting", + "signature_break", + { + "api.py": "def scale(x): return x*2", + "client.py": "from api import scale\ndef run(): return scale(1)", + }, + {"api.py": "def scale(x, factor): return x*factor"}, + {"client.py": "from api import scale\ndef run(): return scale(3)"}, + "api.py::scale", + "The new consumer call lacks the newly required factor.", + "from api import scale\nassert scale(3,4)==12", + "from client import run\nassert run()==6", + ) + add( + "Rename keyword argument", + "conflicting", + "signature_break", + { + "api.py": "def get(name): return name", + "client.py": 'from api import get\ndef run(): return get(name="A")', + }, + {"api.py": "def get(key): return key"}, + {"client.py": 'from api import get\ndef run(): return get(name="B")'}, + "api.py::get", + "B passes a keyword no longer accepted by the producer.", + 'from api import get\nassert get(key="C")=="C"', + 'from client import run\nassert run()=="B"', + ) + add( + "Dependency removal invalidates import", + "conflicting", + "dependency_removal", + { + "requirements.txt": "tiny==1.0", + "tiny.py": "def encode(x): return str(x)", + "client.py": "VALUE=1", + }, + {"requirements.txt": "# no dependencies", "tiny.py": None}, + {"client.py": "import tiny\ndef run(): return tiny.encode(2)"}, + "tiny.py::encode", + "Bundled deterministic dependency fixture is removed; B adds a live import.", + 'from pathlib import Path\nassert not Path("tiny.py").exists()', + 'from client import run\nassert run()=="2"', + ) + add( + "Dependency upgrade changes local API", + "conflicting", + "dependency_upgrade", + { + "requirements.txt": "codec==1.0", + "codec.py": "def encode(x): return str(x)", + "client.py": "import codec\ndef run(): return codec.encode(1)", + }, + { + "requirements.txt": "codec==2.0", + "codec.py": "def encode(x): return str(x).encode()", + }, + {"client.py": 'import codec\ndef run(): return "value:"+codec.encode(2)'}, + "codec.py::encode", + "Vendored dependency fixture now returns bytes, while consumer concatenates str.", + 'import codec\nassert codec.encode(1)==b"1"', + 'from client import run\nassert run()=="value:2"', + ) + add( + "Different replacement values", + "conflicting", + "write_write", + {"config.py": 'MODE="base"'}, + {"config.py": 'MODE="fast"'}, + {"config.py": 'MODE="safe"'}, + "config.py::MODE", + "One scalar cannot equal both required values.", + 'from config import MODE\nassert MODE=="fast"', + 'from config import MODE\nassert MODE=="safe"', + ) + add( + "Incompatible return semantics", + "conflicting", + "return_value", + {"ops.py": "def fee(x): return x"}, + {"ops.py": "def fee(x): return x*2"}, + {"ops.py": "def fee(x): return x+2"}, + "ops.py::fee", + "At x=3 the two required outputs are six and five.", + "from ops import fee\nassert fee(3)==6", + "from ops import fee\nassert fee(3)==5", + ) + add( + "Dynamic dispatch after rename", + "conflicting", + "dynamic_dispatch", + { + "api.py": "def fetch(): return 1", + "client.py": 'import api\ndef run(): return getattr(api,"fetch")()', + }, + {"api.py": "def load(): return 1"}, + {"client.py": 'import api\ndef run(): return getattr(api,"fetch")()+1'}, + "api.py::fetch", + "String-based dynamic lookup still asks for removed API.", + 'import api\nassert api.load()==1 and not hasattr(api,"fetch")', + "from client import run\nassert run()==2", + online=True, + ) + add( + "Environment variable contract", + "conflicting", + "dynamic_configuration", + { + "producer.py": 'import os\ndef setup(): os.environ["APP_MODE"]="text"', + "client.py": 'import os\ndef run(): return os.environ.get("APP_MODE")', + }, + {"producer.py": 'import os\ndef setup(): os.environ["APP_MODE"]="binary"'}, + { + "client.py": 'import os\ndef run():\n assert os.environ.get("APP_MODE")=="text"\n return "ok"' + }, + "environment:APP_MODE", + "Producer changes environment protocol consumed in a different module.", + 'import os\nfrom producer import setup\nsetup()\nassert os.environ["APP_MODE"]=="binary"', + 'from producer import setup\nfrom client import run\nsetup()\nassert run()=="ok"', + ) + add( + "JSON configuration consumer", + "conflicting", + "dynamic_configuration", + { + "config.json": '{"mode":"text"}', + "client.py": 'import json\ndef read():\n return json.load(open("config.json"))["mode"]', + }, + {"config.json": '{"mode":"binary"}'}, + { + "client.py": 'import json\ndef read():\n mode=json.load(open("config.json"))["mode"]\n assert mode=="text"\n return mode.upper()' + }, + "config.json::mode", + "File-based dynamic dependency is not an import edge.", + 'import json\nassert json.load(open("config.json"))["mode"]=="binary"', + 'from client import read\nassert read()=="TEXT"', + ) + add( + "Shared string state protocol", + "conflicting", + "shared_state", + { + "state.py": 'MODE="text"\ndef setup():\n global MODE\n MODE="text"\ndef run(): return MODE' + }, + { + "state.py": 'MODE="text"\ndef setup():\n global MODE\n MODE="binary"\ndef run(): return MODE' + }, + { + "state.py": 'MODE="text"\ndef setup():\n global MODE\n MODE="text"\ndef run():\n assert MODE=="text"\n return MODE.upper()' + }, + "state.py::MODE", + "Changed writer violates another function's new state precondition.", + 'import state\nstate.setup()\nassert state.MODE=="binary"', + 'import state\nstate.setup()\nassert state.run()=="TEXT"', + ) + return CASES + + +def patch(base, target): + return "".join( + "".join( + difflib.unified_diff( + base.get(p, "").splitlines(True), + target.get(p, "").splitlines(True), + fromfile="a/" + p if p in base else "/dev/null", + tofile="b/" + p if p in target else "/dev/null", + ) + ) + for p in sorted(base.keys() | target.keys()) + if base.get(p) != target.get(p) + ) + + +def write_state(root, state): + for p, source in state.items(): + dest = root / p + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(source) + + +def build(root): + root.mkdir(parents=True, exist_ok=True) + for c in cases(): + path = root / c["id"] + if path.exists(): + raise FileExistsError(f"Refusing to overwrite benchmark: {path}") + path.mkdir() + for key in ("base", "a", "b"): + write_state(path / key, c[key]) + for key in ("a", "b"): + (path / f"patch_{key.upper()}.diff").write_text(patch(c["base"], c[key])) + (path / "checks").mkdir() + for key, source in c["checks"].items(): + (path / "checks" / f"{key}.py").write_text(source) + if c["witness"] is not None: + write_state(path / "witness", c["witness"]) + metadata = { + k: v + for k, v in c.items() + if k not in {"base", "a", "b", "checks", "witness"} + } + metadata["cross_file"] = not bool( + { + p + for p in c["base"].keys() | c["a"].keys() + if c["base"].get(p) != c["a"].get(p) + } + & { + p + for p in c["base"].keys() | c["b"].keys() + if c["base"].get(p) != c["b"].get(p) + } + ) + metadata["witness_available"] = c["witness"] is not None + (path / "gold.json").write_text( + json.dumps(metadata, ensure_ascii=False, indent=2) + "\n" + ) + if c["online"]: + # Three real source snapshots per branch: harmless helper, first changed + # file, final state. Helpers remain until final cleanup. Steps + # are syntactic and deterministic, not synthesized from gold labels. + from ..online.sequences import stages, functional_complete + + events = [] + for index, (sa, sb) in enumerate( + zip(stages(c["base"], c["a"], "a"), stages(c["base"], c["b"], "b")), 1 + ): + for branch, snapshot in (("a", sa), ("b", sb)): + write_state(path / "online" / f"{branch}{index}", snapshot) + prev = ( + c["base"] + if index == 1 + else (stages(c["base"], c[branch], branch)[index - 2]) + ) + dest = path / "online" / f"{branch}{index}.diff" + dest.write_text(patch(prev, snapshot)) + events.append( + { + "branch": branch, + "stage": index, + "state": f"{branch}{index}", + "patch": f"{branch}{index}.diff", + "requirements_active": functional_complete( + snapshot, c[branch] + ), + } + ) + (path / "online" / "schedule.json").write_text( + json.dumps(events, indent=2) + "\n" + ) + summary = { + "cases": len(CASES), + "labels": dict(Counter(c["label"] for c in CASES)), + "families": dict(Counter(c["family"] for c in CASES)), + "online_cases": sum(c["online"] for c in CASES), + } + (root / "statistics.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--out", type=Path, default=Path("experiments/benchmark/cases")) + build(parser.parse_args().out) diff --git a/prototypes/structured-delta/experiments/delta/__init__.py b/prototypes/structured-delta/experiments/delta/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prototypes/structured-delta/experiments/delta/extractor.py b/prototypes/structured-delta/experiments/delta/extractor.py new file mode 100644 index 0000000..dd87e46 --- /dev/null +++ b/prototypes/structured-delta/experiments/delta/extractor.py @@ -0,0 +1,449 @@ +"""Python AST + simple requirements/JSON analysis; no tests, labels or intent input. + +This is deliberately bounded analysis, not an equivalence prover or type checker. +""" + +import ast +import copy +import difflib +import json +from pathlib import PurePosixPath +from .model import ProgramDelta, Symbol + + +def dump(node): + return ast.dump(node, include_attributes=False) if node is not None else "" + + +class Canonical(ast.NodeTransformer): + def visit_Compare(self, node): + node = self.generic_visit(node) + if len(node.ops) == 1 and isinstance(node.ops[0], (ast.Is, ast.IsNot)): + if isinstance(node.left, ast.Constant) and node.left.value is None: + node.left, node.comparators[0] = node.comparators[0], node.left + return node + + def visit_If(self, node): + node = self.generic_visit(node) + # Invert only an explicit identity test, preserving Python == semantics. + if ( + len(node.body) == len(node.orelse) == 1 + and isinstance(node.body[0], ast.Return) + and isinstance(node.orelse[0], ast.Return) + and isinstance(node.test, ast.Compare) + and len(node.test.ops) == 1 + and isinstance(node.test.ops[0], ast.IsNot) + ): + node.test.ops[0] = ast.Is() + node.body, node.orelse = node.orelse, node.body + return node + + +def canonical(node): + return dump(Canonical().visit(copy.deepcopy(node))) + + +def module_path(name, files, current="", level=0): + if level: + parts = list(PurePosixPath(current).parent.parts) + parts = parts[: len(parts) - level + 1] + name = ".".join(parts + ([name] if name else [])) + path = name.replace(".", "/") + for candidate in (path + ".py", path + "/__init__.py"): + if candidate in files: + return candidate + return None + + +def literal_type(n): + if n is None: + return "None" + if isinstance(n, ast.Constant): + return type(n.value).__name__ + if isinstance(n, ast.Dict): + keys = [ + k.value + for k in n.keys + if isinstance(k, ast.Constant) and isinstance(k.value, str) + ] + return "dict:" + ",".join(sorted(keys)) + if isinstance(n, (ast.List, ast.ListComp)): + return "list" + if isinstance(n, ast.Tuple): + return "tuple:" + str(len(n.elts)) + if ( + isinstance(n, ast.Call) + and isinstance(n.func, ast.Name) + and n.func.id in {"str", "int", "float", "dict", "list", "bool", "set"} + ): + return n.func.id + return "unknown" + + +def state(snapshot): + symbols, imports, unresolved = {}, {}, [] + for path, source in sorted(snapshot.items()): + if not path.endswith(".py"): + continue + try: + tree = ast.parse(source, filename=path) + except SyntaxError as exc: + unresolved.append(f"{path}:{exc.lineno}: {exc.msg}") + continue + aliases, module_imports = {}, set() + globals_ = { + n.id + for stmt in tree.body + if isinstance(stmt, (ast.Assign, ast.AnnAssign)) + for n in ast.walk(stmt) + if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store) + } + defs = { + stmt.name + for stmt in tree.body + if isinstance(stmt, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)) + } + # Module-local aliases, including relative and package imports. + for n in ast.walk(tree): + if isinstance(n, ast.ImportFrom): + target = module_path(n.module or "", snapshot, path, n.level) + for alias in n.names: + entity = ( + f"{target}::{alias.name}" + if target + else f"dependency:{n.module}" + ) + aliases[alias.asname or alias.name] = entity + module_imports.add(entity) + elif isinstance(n, ast.Import): + for alias in n.names: + target = module_path(alias.name, snapshot) + aliases[alias.asname or alias.name.split(".")[0]] = ( + f"{target}::" + if target + else f"dependency:{alias.name.split('.')[0]}" + ) + module_imports.add( + aliases[alias.asname or alias.name.split(".")[0]] + ) + imports[path] = sorted(module_imports) + + def resolve(n, locals_): + if isinstance(n, ast.Name): + if n.id in locals_: + return None + if n.id in aliases: + return aliases[n.id] + if n.id in globals_ or n.id in defs: + return f"{path}::{n.id}" + if isinstance(n, ast.Attribute): + parent = resolve(n.value, locals_) + if parent: + return parent + ("" if parent.endswith("::") else ".") + n.attr + if isinstance(n, ast.Subscript): + parent = resolve(n.value, locals_) + if parent and isinstance(n.slice, ast.Constant): + return parent + "[" + repr(n.slice.value) + "]" + return None + + def collect(n, qual): + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + entity = f"{path}::{qual}{n.name}" + is_func = not isinstance(n, ast.ClassDef) + sig, returns, rexpr, reads, writes, calls = {}, [], [], set(), {}, [] + local_names = set() + if is_func: + args = n.args + declared_global = { + name + for k in ast.walk(n) + if isinstance(k, ast.Global) + for name in k.names + } + local_names = { + k.id + for k in ast.walk(n) + if isinstance(k, ast.Name) and isinstance(k.ctx, ast.Store) + } - declared_global + local_names |= { + k.arg for k in ast.walk(args) if isinstance(k, ast.arg) + } + positional = args.posonlyargs + args.args + sig = { + "parameters": [a.arg for a in positional], + "required": len(positional) - len(args.defaults), + "maximum": None if args.vararg else len(positional), + "posonly": len(args.posonlyargs), + "kwonly": [a.arg for a in args.kwonlyargs], + "required_kwonly": [ + a.arg + for a, d in zip(args.kwonlyargs, args.kw_defaults) + if d is None + ], + "kwargs": bool(args.kwarg), + "annotations": [ + ast.unparse(a.annotation) if a.annotation else "" + for a in positional + ], + "return": ast.unparse(n.returns) if n.returns else "", + } + parents = { + child: parent + for parent in ast.walk(n) + for child in ast.iter_child_nodes(parent) + } + for k in ast.walk(n): + if isinstance(k, ast.Return): + returns.append(literal_type(k.value)) + rexpr.append(canonical(k.value)) + if isinstance( + k, (ast.Name, ast.Attribute, ast.Subscript) + ) and isinstance(k.ctx, ast.Load): + ref = resolve(k, local_names) + parent = parents.get(k) + # Attribute/subscript access uses its most specific resolved entity. + # Avoid inventing interference between distinct constant keys. + covered = isinstance( + parent, (ast.Attribute, ast.Subscript) + ) and resolve(parent, local_names) + if ref and not covered: + reads.add(ref) + if isinstance(k, (ast.Assign, ast.AnnAssign, ast.AugAssign)): + targets = ( + k.targets if isinstance(k, ast.Assign) else [k.target] + ) + for t in targets: + ref = resolve(t, local_names) + if ref: + writes.setdefault(ref, []).append(canonical(k)) + if isinstance(k, ast.Call): + ref = resolve(k.func, local_names) + if ref: + calls.append( + { + "target": ref, + "argc": len(k.args), + "keywords": [v.arg for v in k.keywords], + "dynamic": any( + isinstance(a, ast.Starred) for a in k.args + ) + or any(v.arg is None for v in k.keywords), + "source": entity, + } + ) + # Common collection mutation methods are writes to the collection. + if isinstance( + k.func, ast.Attribute + ) and k.func.attr in { + "append", + "extend", + "update", + "pop", + "clear", + "add", + "remove", + "setdefault", + }: + parent = resolve(k.func.value, local_names) + if parent: + writes.setdefault(parent, []).append( + canonical(k) + ) + if isinstance(k.func, ast.Name) and k.func.id in { + "getattr", + "eval", + "exec", + "globals", + }: + unresolved.append(f"{entity}: dynamic call {k.func.id}") + symbols[entity] = Symbol( + entity, + "function" if is_func else "class", + n.lineno, + n.end_lineno, + canonical(n), + sig, + sorted(set(returns)), + rexpr, + sorted(reads), + writes, + calls, + [canonical(s) for s in n.body], + ) + for child in n.body: + if isinstance( + child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ): + collect(child, qual + n.name + ".") + elif isinstance(n, (ast.Assign, ast.AnnAssign)) and not qual: + for t in n.targets if isinstance(n, ast.Assign) else [n.target]: + if isinstance(t, ast.Name): + entity = f"{path}::{t.id}" + symbols[entity] = Symbol( + entity, + "state", + n.lineno, + n.end_lineno, + canonical(n), + returns=[literal_type(n.value)], + value=canonical(n.value), + ) + + for node in tree.body: + collect(node, "") + for path, source in snapshot.items(): + if path == "requirements.txt": + for row, line in enumerate(source.splitlines(), 1): + if line.strip() and not line.startswith("#"): + import re + + name = re.split(r"[<>=!~\[; ]", line.strip())[0].replace("-", "_") + key = "dependency:" + name + symbols[key] = Symbol( + key, "dependency", row, row, line.strip(), value=line.strip() + ) + elif path.endswith(".json"): + try: + data = json.loads(source) + if isinstance(data, dict): + for key, value in data.items(): + entity = f"{path}::{key}" + symbols[entity] = Symbol( + entity, + "config", + 1, + len(source.splitlines()), + json.dumps(value, sort_keys=True), + value=repr(value), + ) + except ValueError: + unresolved.append(f"{path}: invalid JSON") + return symbols, imports, unresolved + + +def extract(base, changed): + old, oi, ou = state(base) + new, ni, nu = state(changed) + files = sorted( + p for p in base.keys() | changed.keys() if base.get(p) != changed.get(p) + ) + affected = sorted( + k + for k in old.keys() | new.keys() + if k not in old or k not in new or old[k].canonical != new[k].canonical + ) + # Canonical equivalence can hide a substantive syntactic edit (e.g. reversed None identity). + raw_old, raw_new = {}, {} + for p in files: + for index, snapshot in ((raw_old, base), (raw_new, changed)): + try: + tree = ast.parse(snapshot.get(p, "")) if p.endswith(".py") else None + if tree: + + def walk(ns, prefix=""): + for n in ns: + if isinstance( + n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ): + index[f"{p}::{prefix}{n.name}"] = dump(n) + walk(n.body, prefix + n.name + ".") + + walk(tree.body) + except SyntaxError: + pass + affected = sorted( + set(affected) + | {k for k in raw_old.keys() & raw_new.keys() if raw_old[k] != raw_new[k]} + ) + operations, ast_ops, contracts, reads, writes, calls = [], [], {}, set(), {}, [] + for k in affected: + a, b = old.get(k), new.get(k) + kind = "create" if a is None else "delete" if b is None else "modify" + operations.append({"entity": k, "operation": kind}) + for s in (a, b): + if s: + reads.update(s.reads) + calls.extend(s.calls) + # Only actual changed write effects, not every pre-existing write in a changed function. + aw, bw = a.writes if a else {}, b.writes if b else {} + for ref in aw.keys() | bw.keys(): + if aw.get(ref) != bw.get(ref): + writes.setdefault(ref, []).extend(bw.get(ref, [""])) + if (b or a).kind in {"state", "config", "dependency"}: + writes[k] = [b.value if b else ""] + if a and b: + fields = {} + if a.signature != b.signature: + fields["signature"] = {"before": a.signature, "after": b.signature} + if a.returns != b.returns: + fields["return_type"] = {"before": a.returns, "after": b.returns} + if a.value != b.value: + fields["value"] = {"before": a.value, "after": b.value} + if fields: + contracts[k] = fields + matcher = difflib.SequenceMatcher( + a=a.statements, b=b.statements, autojunk=False + ) + for op, i, j, x, y in matcher.get_opcodes(): + if op != "equal": + ast_ops.append( + { + "entity": k, + "operation": op, + "before": a.statements[i:j], + "after": b.statements[x:y], + } + ) + # Exact-body rename candidates (names excluded), retained as evidence, not guessed by spelling. + for op in list(operations): + if op["operation"] != "delete": + continue + a = old[op["entity"]] + matches = [ + p["entity"] + for p in operations + if p["operation"] == "create" + and new[p["entity"]].kind == a.kind + and a.statements + and a.statements == new[p["entity"]].statements + and a.signature == new[p["entity"]].signature + ] + if len(matches) == 1: + op["operation"] = "rename" + op["to"] = matches[0] + lines = {} + for p in files: + touched = set() + for op, i, j, x, y in difflib.SequenceMatcher( + a=base.get(p, "").splitlines(), + b=changed.get(p, "").splitlines(), + autojunk=False, + ).get_opcodes(): + if op != "equal": + touched.update(range(i + 1, j + 1) if j > i else [i + 0.5]) + lines[p] = sorted(touched) + imports_old = {v for p in files for v in oi.get(p, [])} + imports_new = {v for p in files for v in ni.get(p, [])} + deps_old = {k for k in old if k.startswith("dependency:")} | imports_old + deps_new = {k for k in new if k.startswith("dependency:")} | imports_new + return ProgramDelta( + files, + affected, + operations, + sorted(reads), + writes, + {k: old[k].signature for k in affected if k in old}, + {k: new[k].signature for k in affected if k in new}, + {k: old[k].returns for k in affected if k in old}, + {k: new[k].returns for k in affected if k in new}, + sorted(deps_new - deps_old), + sorted(deps_old - deps_new), + contracts, + sorted(imports_new), + calls, + ast_ops, + lines, + {k: old[k] for k in affected if k in old}, + {k: new[k] for k in affected if k in new}, + sorted(set(ou + nu)), + ) diff --git a/prototypes/structured-delta/experiments/delta/interaction.py b/prototypes/structured-delta/experiments/delta/interaction.py new file mode 100644 index 0000000..1fbc65c --- /dev/null +++ b/prototypes/structured-delta/experiments/delta/interaction.py @@ -0,0 +1,190 @@ +"""Frozen rule ladder. Overlap establishes scope; concrete effects decide conflict. + +Scores are ordinal rule strengths, never calibrated probabilities. +""" + +from .model import Prediction + +LEVELS = ( + "File", + "File + Symbol", + "File + Symbol + Dependency", + "File + Symbol + Dependency + Contract", + "Full Structured Delta", +) + + +def related(a, b): + return ( + a == b + or a.startswith(b + ".") + or b.startswith(a + ".") + or a.startswith(b + "[") + or b.startswith(a + "[") + ) + + +def detect(a, b, level="Full Structured Delta"): + depth = LEVELS.index(level) + files = sorted(set(a.files) & set(b.files)) + scope = sorted({x for x in a.symbols for y in b.symbols if related(x, y)}) + deps = sorted( + {x for x in a.symbols for y in b.reads + b.imports if related(x, y)} + | {x for x in b.symbols for y in a.reads + a.imports if related(x, y)} + ) + interference = sorted( + {x for x in a.writes for y in list(b.writes) + b.reads if related(x, y)} + | {x for x in b.writes for y in a.reads if related(x, y)} + ) + signals = { + "scope_overlap": scope, + "dependency_overlap": deps if depth >= 2 else [], + "read_write_interference": interference if depth == 4 else [], + "contract_incompatibility": [], + "redundancy": False, + "unresolved": sorted(set(a.unresolved + b.unresolved)), + } + + def result(label, entities, reason, score=0): + return Prediction( + label, list(dict.fromkeys(entities))[:3], score, [reason], signals + ) + + if depth == 0: + return result( + "conflicting" if files else "independent", + ["file:" + p for p in files], + "File intersection only", + 0.5 if files else 0, + ) + if depth == 1: + return result( + "conflicting" if scope else "independent", + scope, + "Symbol intersection only", + 0.5 if scope else 0, + ) + if depth == 2: + return result( + "conflicting" if scope or deps else "independent", + scope + deps, + "Symbol or dependency intersection only", + 0.5 if scope or deps else 0, + ) + if depth == 4 and a.symbols and set(a.symbols) == set(b.symbols): + if ( + set(a.after) == set(b.after) + and all(a.after[k].canonical == b.after[k].canonical for k in a.after) + and set(a.before) == set(b.before) + and all(a.before[k].canonical == b.before[k].canonical for k in a.before) + ): + signals["redundancy"] = True + return result( + "redundant", + scope or a.symbols, + "Equal normalized AST transformations", + 0.02, + ) + for source, target in ((a, b), (b, a)): + for op in source.operations: + k = op["entity"] + if op["operation"] in {"delete", "rename"} and any( + related(k, t) for t in target.symbols + target.reads + target.imports + ): + signals["contract_incompatibility"].append(k) + return result( + "conflicting", + [k], + f"{op['operation']} intersects changed implementation or surviving consumer", + 0.98, + ) + for k, contract in source.contracts_changed.items(): + consumers = [c for c in target.calls if related(c["target"], k)] + referenced = any(related(k, t) for t in target.reads + target.imports) + if "signature" in contract and consumers: + sig = contract["signature"]["after"] + for call in consumers: + if call["dynamic"]: + continue + keywords = set(call["keywords"]) + params = sig.get("parameters", []) + required = params[: sig.get("required", 0)] + bad = ( + sig.get("maximum") is not None and call["argc"] > sig["maximum"] + ) + bad |= any(p not in keywords for p in required[call["argc"] :]) + bad |= any( + p not in keywords for p in sig.get("required_kwonly", []) + ) + bad |= bool( + keywords + - set(params[sig.get("posonly", 0) :]) + - set(sig.get("kwonly", [])) + ) and not sig.get("kwargs") + if bad: + signals["contract_incompatibility"].append(k) + return result( + "conflicting", + [k], + "Changed signature rejects an observed call", + 0.96, + ) + if referenced and ("return_type" in contract or "value" in contract): + # Conservative: this prototype cannot prove adaptation at the use site. + signals["contract_incompatibility"].append(k) + return result( + "conflicting", + [k], + "Consumer references a changed return/configuration/dependency contract", + 0.85, + ) + if depth == 4: + if interference: + return result( + "conflicting", + interference, + "Changed writes intersect concurrent reads or writes", + 0.82, + ) + for k in scope: + if k in a.after and k in b.after and k in a.before and k in b.before: + aa, bb, old = a.after[k], b.after[k], a.before[k] + if ( + aa.return_exprs != old.return_exprs + and bb.return_exprs != old.return_exprs + and aa.return_exprs != bb.return_exprs + ): + return result( + "conflicting", + [k], + "Different concurrent replacements of return behavior", + 0.8, + ) + if ( + aa.signature != old.signature + and bb.signature != old.signature + and aa.signature != bb.signature + ): + return result( + "conflicting", + [k], + "Different concurrent function signatures", + 0.85, + ) + # Different creations at the same qualified symbol collide even without returns. + for k in set(a.after) & set(b.after) - set(a.before) - set(b.before): + if a.after[k].canonical != b.after[k].canonical: + return result( + "conflicting", + [k], + "Different definitions create the same symbol", + 0.9, + ) + if scope or deps: + return result( + "compatible", + scope + deps, + "Shared scope/dependency without a recognized incompatible effect", + 0.15, + ) + return result("independent", [], "No resolved interaction", 0) diff --git a/prototypes/structured-delta/experiments/delta/model.py b/prototypes/structured-delta/experiments/delta/model.py new file mode 100644 index 0000000..5261b60 --- /dev/null +++ b/prototypes/structured-delta/experiments/delta/model.py @@ -0,0 +1,61 @@ +"""Serializable observations inferred exclusively from source snapshots.""" + +from dataclasses import dataclass, field, asdict +from typing import Any + +LABELS = ("independent", "compatible", "redundant", "conflicting") + + +@dataclass +class Symbol: + entity: str + kind: str + start: int + end: int + canonical: str + signature: dict = field(default_factory=dict) + returns: list[str] = field(default_factory=list) + return_exprs: list[str] = field(default_factory=list) + reads: list[str] = field(default_factory=list) + writes: dict[str, list[str]] = field(default_factory=dict) + calls: list[dict] = field(default_factory=list) + statements: list[str] = field(default_factory=list) + value: str | None = None + + +@dataclass +class ProgramDelta: + files: list[str] + symbols: list[str] + operations: list[dict] + reads: list[str] + writes: dict[str, list[str]] + signatures_before: dict + signatures_after: dict + types_before: dict + types_after: dict + dependencies_added: list[str] + dependencies_removed: list[str] + contracts_changed: dict + imports: list[str] + calls: list[dict] + ast_operations: list[dict] + lines: dict[str, list[int]] + before: dict[str, Symbol] + after: dict[str, Symbol] + unresolved: list[str] + + def to_dict(self): + return asdict(self) + + +@dataclass +class Prediction: + label: str + entities: list[str] + score: float + reasons: list[str] + signals: dict[str, Any] = field(default_factory=dict) + + def to_dict(self): + return asdict(self) diff --git a/prototypes/structured-delta/experiments/evaluation/__init__.py b/prototypes/structured-delta/experiments/evaluation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prototypes/structured-delta/experiments/evaluation/evaluate.py b/prototypes/structured-delta/experiments/evaluation/evaluate.py new file mode 100644 index 0000000..6a5b81f --- /dev/null +++ b/prototypes/structured-delta/experiments/evaluation/evaluate.py @@ -0,0 +1,156 @@ +import argparse +import json +import platform +import time +from collections import Counter +from pathlib import Path +from ..delta.extractor import extract +from ..delta.interaction import detect, LEVELS +from ..baselines import file_overlap, ast_overlap, git_merge, embedding, llm_intent +from .runtime import snapshot, commutativity +from .metrics import score, grouped +from .freeze import verify + + +def run(root, benchmark, out, embedding_command=None, llm_command=None): + frozen = verify(root) + if out.exists(): + raise FileExistsError(f"Refusing to replace results: {out}") + out.mkdir(parents=True) + all_rows = {} + records = [] + for path in sorted(benchmark.glob("case[0-9]*")): + base, a, b = (snapshot(path / k) for k in ("base", "a", "b")) + started = time.perf_counter() + da, db = extract(base, a), extract(base, b) + extracted_ms = (time.perf_counter() - started) * 1000 + started = time.perf_counter() + preds = { + "B1 File": file_overlap.predict(da, db), + "B2 Line": ast_overlap.predict(da, db, "line"), + "B2 AST/Symbol": ast_overlap.predict(da, db), + } + preds.update({level: detect(da, db, level) for level in LEVELS}) + detection_ms = (time.perf_counter() - started) * 1000 + patch_a, patch_b = ((path / f"patch_{k}.diff").read_text() for k in ("A", "B")) + optional = {} + for key, command, fn in ( + ( + "B4 Embedding", + embedding_command, + lambda cmd: embedding.predict(cmd, patch_a, patch_b, da, db), + ), + ( + "B5 LLM descriptions", + llm_command, + lambda cmd: llm_intent.predict(cmd, patch_a, patch_b), + ), + ): + if command: + try: + preds[key] = fn(command) + optional[key] = {"status": "ok"} + except Exception as error: + optional[key] = {"status": "error", "error": str(error)} + else: + optional[key] = {"status": "not_configured"} + # Gold, requirements and witness data are read only after predictions. + gold = json.loads((path / "gold.json").read_text()) + checks = {k: (path / "checks" / f"{k}.py").read_text() for k in ("a", "b")} + commute = commutativity(base, a, b, patch_a, patch_b, checks) + preds["B3 Git merge"] = git_merge.predict(commute["three_way"]["ab"]) + rec = { + "id": path.name, + "gold": gold, + "delta_a": da.to_dict(), + "delta_b": db.to_dict(), + "predictions": {k: v.to_dict() for k, v in preds.items()}, + "optional": optional, + "timing": { + "extract_ms": extracted_ms, + "deterministic_all_rules_ms": detection_ms, + }, + "commutativity": commute, + } + (out / f"{path.name}.json").write_text( + json.dumps(rec, ensure_ascii=False, indent=2) + "\n" + ) + records.append(rec) + for method, prediction in preds.items(): + all_rows.setdefault(method, []).append( + { + "id": path.name, + "gold": gold["label"], + "entities": gold["entities"], + "family": gold["family"], + "cross_file": gold["cross_file"], + "prediction": prediction.to_dict(), + } + ) + print(path.name, flush=True) + # Unavailable optional results remain missing; partial subsets are explicitly marked. + results = { + method: { + "aggregate": score(rows), + "by_family": grouped(rows), + "coverage": {"scored": len(rows), "total": len(records)}, + } + for method, rows in all_rows.items() + } + errors = { + method: { + "false_positives": [ + r + for r in rows + if r["gold"] != "conflicting" + and r["prediction"]["label"] == "conflicting" + ], + "false_negatives": [ + r + for r in rows + if r["gold"] == "conflicting" + and r["prediction"]["label"] != "conflicting" + ], + "all_misclassified": [ + r for r in rows if r["gold"] != r["prediction"]["label"] + ], + } + for method, rows in all_rows.items() + } + summary = { + "environment": { + "python": platform.python_version(), + "platform": platform.platform(), + }, + "freeze_created_at": frozen["created_at"], + "cases": len(records), + "labels": dict(Counter(r["gold"]["label"] for r in records)), + "families": len({r["gold"]["family"] for r in records}), + "cross_file_conflicts": sum( + r["gold"]["label"] == "conflicting" and r["gold"]["cross_file"] + for r in records + ), + "methods": results, + "optional": { + key: dict(Counter(r["optional"][key]["status"] for r in records)) + for key in ("B4 Embedding", "B5 LLM descriptions") + }, + "errors": errors, + } + (out / "summary.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2) + "\n" + ) + print(json.dumps({k: v["aggregate"] for k, v in results.items()}, indent=2)) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--root", type=Path, default=Path(".")) + p.add_argument( + "--benchmark", type=Path, default=Path("experiments/benchmark/cases") + ) + p.add_argument("--out", type=Path, default=Path("experiments/results/run-001")) + p.add_argument("--embedding-command") + p.add_argument("--llm-command") + a = p.parse_args() + run(a.root, a.benchmark, a.out, a.embedding_command, a.llm_command) diff --git a/prototypes/structured-delta/experiments/evaluation/freeze.py b/prototypes/structured-delta/experiments/evaluation/freeze.py new file mode 100644 index 0000000..53d69ea --- /dev/null +++ b/prototypes/structured-delta/experiments/evaluation/freeze.py @@ -0,0 +1,55 @@ +"""Freeze all protocol, code and benchmark bytes before formal evaluation.""" + +import argparse +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path + + +def hashes(root): + paths = [root / "PROTOCOL.md"] + sorted((root / "experiments").rglob("*")) + return { + p.relative_to(root).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest() + for p in paths + if p.is_file() + and "__pycache__" not in p.parts + and "results" not in p.parts + and p.suffix in {".py", ".md", ".json", ".diff", ".txt"} + } + + +def verify(root): + frozen = json.loads((root / "FREEZE.json").read_text()) + current = hashes(root) + if frozen["sha256"] != current: + changed = sorted( + k + for k in frozen["sha256"].keys() | current.keys() + if frozen["sha256"].get(k) != current.get(k) + ) + raise RuntimeError( + "Frozen inputs changed; create a new protocol version, retain old runs: " + + str(changed) + ) + return frozen + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--root", type=Path, default=Path(".")) + args = p.parse_args() + dest = args.root / "FREEZE.json" + if dest.exists(): + raise FileExistsError("Refusing to replace existing freeze") + dest.write_text( + json.dumps( + { + "created_at": datetime.now(timezone.utc).isoformat(), + "sha256": hashes(args.root), + }, + indent=2, + ) + + "\n" + ) + print(dest) diff --git a/prototypes/structured-delta/experiments/evaluation/metrics.py b/prototypes/structured-delta/experiments/evaluation/metrics.py new file mode 100644 index 0000000..10e4aad --- /dev/null +++ b/prototypes/structured-delta/experiments/evaluation/metrics.py @@ -0,0 +1,67 @@ +"""Metrics have explicit denominators; null means undefined, never zero-filled.""" + +from ..delta.model import LABELS + + +def ratio(n, d): + return n / d if d else None + + +def score(rows): + confusion = {a: {b: 0 for b in LABELS} for a in LABELS} + for r in rows: + confusion[r["gold"]][r["prediction"]["label"]] += 1 + f1 = [] + per_class = {} + for label in LABELS: + tp = confusion[label][label] + fp = sum(confusion[g][label] for g in LABELS if g != label) + fn = sum(confusion[label][p] for p in LABELS if p != label) + v = 2 * tp / (2 * tp + fp + fn) if 2 * tp + fp + fn else 0.0 + f1.append(v) + per_class[label] = { + "precision": ratio(tp, tp + fp), + "recall": ratio(tp, tp + fn), + "f1": v, + "support": tp + fn, + } + compat = [r for r in rows if r["gold"] == "compatible"] + cross = [r for r in rows if r["gold"] == "conflicting" and r["cross_file"]] + local = [r for r in rows if r["entities"]] + top1 = sum( + bool(r["prediction"]["entities"]) + and r["prediction"]["entities"][0] in r["entities"] + for r in local + ) + recall3 = sum( + bool(set(r["prediction"]["entities"][:3]) & set(r["entities"])) for r in local + ) + cp = per_class["conflicting"] + return { + "n": len(rows), + "macro_f1": sum(f1) / 4, + "conflict_precision": cp["precision"], + "conflict_recall": cp["recall"], + "compatible_false_positive_rate": ratio( + sum(r["prediction"]["label"] == "conflicting" for r in compat), len(compat) + ), + "cross_file_conflict_recall": ratio( + sum(r["prediction"]["label"] == "conflicting" for r in cross), len(cross) + ), + "localization_top1": ratio(top1, len(local)), + "localization_recall3": ratio(recall3, len(local)), + "denominators": { + "compatible": len(compat), + "cross_file_conflicting": len(cross), + "localization": len(local), + }, + "per_class": per_class, + "confusion": confusion, + } + + +def grouped(rows): + return { + family: score([r for r in rows if r["family"] == family]) + for family in sorted({r["family"] for r in rows}) + } diff --git a/prototypes/structured-delta/experiments/evaluation/report.py b/prototypes/structured-delta/experiments/evaluation/report.py new file mode 100644 index 0000000..8b45171 --- /dev/null +++ b/prototypes/structured-delta/experiments/evaluation/report.py @@ -0,0 +1,296 @@ +"""Render a self-contained audit report from persisted results (no prediction).""" + +import argparse +import json +from pathlib import Path + + +def pct(v): + return "—" if v is None else f"{100 * v:.1f}%" + + +def conflict_f1(m): + p, r = m["conflict_precision"], m["conflict_recall"] + return 2 * p * r / (p + r) if p is not None and r is not None and p + r else 0 + + +def render(static, online, out): + s = json.loads((static / "summary.json").read_text()) + o = json.loads((online / "summary.json").read_text()) + recs = [json.loads(p.read_text()) for p in sorted(static.glob("case*.json"))] + full = s["methods"]["Full Structured Delta"]["aggregate"] + baselines = { + k: v["aggregate"] + for k, v in s["methods"].items() + if k.startswith(("B1", "B2", "B3")) + } + best = max(baselines, key=lambda k: conflict_f1(baselines[k])) + supported = ( + conflict_f1(full) > conflict_f1(baselines[best]) + and full["compatible_false_positive_rate"] + < baselines[best]["compatible_false_positive_rate"] + and full["cross_file_conflict_recall"] + > baselines[best]["cross_file_conflict_recall"] + ) + text = [ + "# 结构化程序状态 delta：实验报告", + "", + "2026-10-07。从共同 base 的实际代码差异提取结构化变化；没有使用代理自述、gold、测试反馈或参考实现作为检测输入。", + "", + f"**结论：{'满足' if supported else '未满足'}协议预先规定的有限支持条件。** " + + ( + "Full 在本轮构造样本上的冲突 F1、兼容误报率和跨文件召回优于最强可用简单基线。" + if supported + else "本轮不能支持结构化方法优于简单基线的有限命题。" + ), + "这不是总体假设的证明：样本和规则由同一实现者共同设计，没有独立留出集；B4/B5 未配置外部模型，无法检验“优于 embedding/LLM”的部分。", + "", + f"样本：{s['cases']} 个 Python 微型仓库、{s['families']} 个命名场景族；标签分布 `{s['labels']}`。" + f"跨文件冲突 {s['cross_file_conflicts']} 个，指两个补丁修改文件集合完全不相交。定位分母 {full['denominators']['localization']} 个，排除 independent。", + "", + f"冻结时刻：`{s['freeze_created_at']}`。环境：Python {s['environment']['python']}；Git 2.43.0。" + "实现/样本/协议哈希见 FREEZE.json；逐案例证据见 results/run-001（如为复跑则见本次指定目录）。", + "", + "## 可用方法比较", + "", + "B1/B2/B3 是二分类启发式，只输出 independent/conflicting；四分类 macro F1 天然受限。" + "不能仅凭 macro F1 判断胜负，故同时列出冲突 F1、精确率/召回率、兼容误报和跨文件召回。" + "定位为严格实体匹配，文件名不能替代函数/配置实体。", + "", + "| 方法 | Macro F1 | 冲突 P | 冲突 R | 冲突 F1 | 兼容 FP | 跨文件 R | Top-1 | R@3 |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + methods = [ + "B1 File", + "B2 Line", + "B2 AST/Symbol", + "B3 Git merge", + "B4 Embedding", + "B5 LLM descriptions", + "Full Structured Delta", + ] + + def row(k): + if k not in s["methods"]: + return f"| {k} | 未运行 | — | — | — | — | — | — | — |" + m = s["methods"][k]["aggregate"] + return ( + "| " + + k + + " | " + + " | ".join( + pct(v) + for v in ( + m["macro_f1"], + m["conflict_precision"], + m["conflict_recall"], + conflict_f1(m), + m["compatible_false_positive_rate"], + m["cross_file_conflict_recall"], + m["localization_top1"], + m["localization_recall3"], + ) + ) + + " |" + ) + + text.extend(row(k) for k in methods) + text += [ + "", + f"最强简单基线按冲突 F1 为 {best}。B4/B5 状态：`{s['optional']}`。" + "可选调用失败时保留错误和覆盖率，不用成功子集冒充全量。", + "", + "## 消融", + "", + "| 特征/规则束 | Macro F1 | 冲突 P | 冲突 R | 冲突 F1 | 兼容 FP | 跨文件 R | Top-1 | R@3 |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + levels = [ + "File", + "File + Symbol", + "File + Symbol + Dependency", + "File + Symbol + Dependency + Contract", + "Full Structured Delta", + ] + text.extend(row(k) for k in levels) + text += [ + "", + "这些是特征和判定规则一起变化的阶梯消融：Contract 阶段才开始把无已知不兼容的相关变化判为 compatible；Full 同时加入读写、返回表达式和规范化重复识别。因此不能把差值严格解释为单一结构组件的因果收益。", + "", + "| 相邻阶段 | 冲突 F1 变化（百分点） | Macro F1 变化（百分点） |", + "|---|---:|---:|", + ] + for prev, nxt in zip(levels, levels[1:]): + m, n = s["methods"][prev]["aggregate"], s["methods"][nxt]["aggregate"] + text.append( + f"| {nxt} | {(conflict_f1(n) - conflict_f1(m)) * 100:+.1f} | {(n['macro_f1'] - m['macro_f1']) * 100:+.1f} |" + ) + text += [ + "", + "## 按场景类型的结果", + "", + "下表给出每个家族的 Full 结果；所有方法的逐族七项指标及混淆矩阵均在 summary.json。" + "小家族只有 1–3 例，不能将单例的 0%/100% 当成稳定概率。", + "", + "| 类型 | n | 四分类正确数 | 冲突命中/真冲突 | 非冲突误报数 | 实体 Top-1 |", + "|---|---:|---:|---:|---:|---:|", + ] + for family, m in s["methods"]["Full Structured Delta"]["by_family"].items(): + records = [r for r in recs if r["gold"]["family"] == family] + correct = sum( + r["gold"]["label"] == r["predictions"]["Full Structured Delta"]["label"] + for r in records + ) + positives = [r for r in records if r["gold"]["label"] == "conflicting"] + tp = sum( + r["predictions"]["Full Structured Delta"]["label"] == "conflicting" + for r in positives + ) + fp = sum( + r["gold"]["label"] != "conflicting" + and r["predictions"]["Full Structured Delta"]["label"] == "conflicting" + for r in records + ) + text.append( + f"| {family} | {m['n']} | {correct} | {tp}/{len(positives)} | {fp} | {pct(m['localization_top1'])} |" + ) + text += [ + "", + "## 错误分析", + "", + "| 案例 | 真值 → 预测 | 类型 | 检测理由/实体 |", + "|---|---|---|---|", + ] + for r in s["errors"]["Full Structured Delta"]["all_misclassified"]: + pred = r["prediction"] + text.append( + f"| {r['id']} | {r['gold']} → {pred['label']} | {r['family']} | " + + ("; ".join(pred["reasons"]) + "; " + ", ".join(pred["entities"])).replace( + "|", "/" + ) + + " |" + ) + fps = s["errors"]["Full Structured Delta"]["false_positives"] + fns = s["errors"]["Full Structured Delta"]["false_negatives"] + text += [ + "", + f"Full 冲突误报 {len(fps)} 个：`{[r['id'] for r in fps]}`；漏报 {len(fns)} 个：`{[r['id'] for r in fns]}`。", + "误报风险来自保守的契约变化⇒失效、共享读写⇒冲突，以及未证明等价时把不同返回表达式视为互斥。" + "漏报风险来自动态查找、外部状态、JSON 读取和分发解析缺失。按实际错误行判断本轮触发了哪些限制，不能把这些局限用增加同模板样本掩盖。", + "更多限制：前后 reads/calls 的并集可能保留已移除的假设；未知类型和源代码注解不构成完整类型证明；manifest 尚未关联安装包到 import；同符号 compatible 默认也可能漏掉路径和执行顺序冲突。", + "", + "## 交换顺序和实际检查", + "", + "| 真值 | n | 两个 strict 顺序都适用 | strict 两序都通过联合测试 | 两个 three-way 都干净 | three-way 两序都通过联合测试 | 干净产物不同 |", + "|---|---:|---:|---:|---:|---:|---:|", + ] + for label in ("independent", "compatible", "redundant", "conflicting"): + group = [r for r in recs if r["gold"]["label"] == label] + + def count(fn): + return sum(fn(r["commutativity"]) for r in group) + + text.append( + f"| {label} | {len(group)} | {count(lambda c: c['strict_patch']['both_apply'])} | " + f"{count(lambda c: all(c['strict_patch'][k]['tests'] is not None and c['strict_patch'][k]['tests']['passed'] for k in ('ab', 'ba')))} | " + f"{count(lambda c: c['three_way']['both_clean'])} | " + f"{count(lambda c: all(c['three_way'][k]['tests'] is not None and c['three_way'][k]['tests']['passed'] for k in ('ab', 'ba')))} | " + f"{count(lambda c: c['three_way']['states_differ'] is True)} |" + ) + text += [ + "", + "适用/合并失败时测试结果与产物比较为 null，不是失败测试或相同产物。" + "冲突标记树、每个顺序的源码哈希、测试 stderr 和返回码均保留。three-way 是共同祖先文本合并，不是语义修复。" + "联合测试并非完备 oracle；兼容案例的 witness 证明存在满足双方要求的组合，不代表 Git 已产生该组合。", + "", + "## 代表性案例", + "", + ] + ids = [ + "case004", + "case011", + "case012", + "case025", + "case034", + "case037", + "case041", + "case045", + "case047", + ] + for cid in ids: + r = next(v for v in recs if v["id"] == cid) + g = r["gold"] + p = r["predictions"]["Full Structured Delta"] + text += [ + f"- **{cid} — {g['title']}**：真值 {g['label']}；B1={r['predictions']['B1 File']['label']}，Git={r['predictions']['B3 Git merge']['label']}，Full={p['label']}。{g['explanation']}" + ] + text += [ + "", + "高文本相似既可能是兼容（case011/012），也可能是重复（case025）或冲突（case044）；跨文件契约变化（case034/037）即使补丁词面不同也会破坏消费者。" + "这些是相似度不能定义语义冲突的机制反例；B4 未运行，不能声称某个 embedding 模型在这些案例实际失败。", + "", + "## 最小在线模拟", + "", + f"{o['cases']} 例、{o['events']} 个补丁事件。冲突检出 {o['conflicts_detected']}/{o['conflicting_cases']}；" + f"非冲突误干预 {o['false_interventions']}/{o['nonconflicting_cases']}，严格“早于首次失败”的误干预 {o['false_early_interventions']}。" + f"冲突案例有效 lead ticks：`{o['lead_ticks_conflicts']}`，均值 `{o['mean_lead_ticks_conflicts']}`；" + f"单事件提取＋检测中位 {o['median_detection_ms']:.3f} ms（单次共享主机测量）。", + "", + "| 案例 | 真值 | 首次告警 | 首次 merge 失败 | 首次行为测试失败 | lead ticks | semantic lead | 误干预 |", + "|---|---|---:|---:|---:|---:|---:|---|", + ] + for r in o["rows"]: + text.append( + "| " + + r["id"] + + " | " + + r["label"] + + " | " + + " | ".join( + "—" if r[k] is None else str(r[k]) + for k in ( + "first_detection_tick", + "first_merge_failure_tick", + "first_test_failure_tick", + "lead_ticks", + "semantic_lead_ticks", + "false_intervention", + ) + ) + + " |" + ) + text += [ + "", + "时间单位为事件序号；正 lead 才表示提前。缺失检测/失败时保留 null。" + "轨迹只有“准备 helper、首文件修改、完成/清理”三个预设阶段，不能证明真实代理工作中能提前预警。" + "要求在功能变化完整时立即激活，避免把清理阶段当作虚假的检测领先。静态标签也不能充分标注每个中间状态的临时冲突。", + "", + "## 对五个研究问题的回答", + "", + f"1. 结构化表示是否提高检测：本轮 Full 冲突 F1={pct(conflict_f1(full))}，最强简单基线={pct(conflict_f1(baselines[best]))}；有限支持判据={supported}，无外部泛化结论。", + "2. 哪些组件最有用：看消融相邻差值；依赖边负责跨文件候选，契约规则区分部分相关但兼容的修改，Full 的重复识别和读写规则也会引入误报。混合规则束不能做纯组件因果归因。", + f"3. 同作用域兼容误报：Full 兼容误报率 {pct(full['compatible_false_positive_rate'])}；仍应逐例看哪些误报由同作用域、契约或状态引发。", + f"4. 跨文件冲突：Full 召回 {pct(full['cross_file_conflict_recall'])}（分母 {full['denominators']['cross_file_conflicting']}）；动态关系仍是明显盲点。", + f"5. 实体定位：Top-1 {pct(full['localization_top1'])}、R@3 {pct(full['localization_recall3'])}；独立于类别判定评分，不能把定位正确当成冲突判断正确。", + "", + "## 唯一优先的下一项实验", + "", + "冻结当前实现，独立收集并盲标至少 100 个来自未见真实仓库的并发修改对，用可执行的双方要求裁决，重点覆盖向后兼容契约与动态跨文件依赖；B4/B5 在独立校准集固定模型和阈值后同轮运行。" + "该迁移盲测优先于扩展同模板案例、接入路由或构建完整 AgentGit。", + "", + "实现文件与复现命令见 README.md / FILES.md。没有提交、推送、插件安装、模型调用或自动修复。", + "", + ] + out.write_text("\n".join(text)) + print(out) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--static", type=Path, default=Path("experiments/results/run-001")) + p.add_argument( + "--online", type=Path, default=Path("experiments/results/online-001") + ) + p.add_argument("--out", type=Path, default=Path("REPORT.md")) + a = p.parse_args() + render(a.static, a.online, a.out) diff --git a/prototypes/structured-delta/experiments/evaluation/runtime.py b/prototypes/structured-delta/experiments/evaluation/runtime.py new file mode 100644 index 0000000..bae2f81 --- /dev/null +++ b/prototypes/structured-delta/experiments/evaluation/runtime.py @@ -0,0 +1,221 @@ +"""Real Git application/merges and subprocess checks; never used by extractor.""" + +import hashlib +import json +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path + + +def snapshot(path): + return { + p.relative_to(path).as_posix(): p.read_text() + for p in sorted(path.rglob("*")) + if p.is_file() and "__pycache__" not in p.parts and ".git" not in p.parts + } + + +def digest(state): + return hashlib.sha256(json.dumps(state, sort_keys=True).encode()).hexdigest() + + +def write(root, state): + for p, s in state.items(): + dest = root / p + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(s) + + +def git(root, *args, input=None, check=True): + env = { + **os.environ, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_AUTHOR_NAME": "Delta Research", + "GIT_AUTHOR_EMAIL": "delta@example.invalid", + "GIT_COMMITTER_NAME": "Delta Research", + "GIT_COMMITTER_EMAIL": "delta@example.invalid", + "GIT_AUTHOR_DATE": "2026-10-07T00:00:00Z", + "GIT_COMMITTER_DATE": "2026-10-07T00:00:00Z", + } + # Ignore inherited repository selection, avoiding accidental parent /tmp/.git. + for key in ( + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_COMMON_DIR", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + ): + env.pop(key, None) + result = subprocess.run( + ["git", "-C", str(root), *args], + input=input, + capture_output=True, + text=True, + env=env, + timeout=20, + ) + if check and result.returncode: + raise RuntimeError( + f"Git {args[0]} failed ({result.returncode}): {result.stderr}" + ) + return result + + +def check_state(state, checks): + started = time.perf_counter() + with tempfile.TemporaryDirectory(prefix="delta-check-") as temp: + root = Path(temp) + write(root, state) + results = [] + for name, code in [ + ( + "syntax", + 'import ast\nfrom pathlib import Path\nfor p in Path(".").rglob("*.py"): ast.parse(p.read_text(),filename=str(p))', + ) + ] + list(checks.items()): + try: + # -I plus explicit cwd in sys.path: no installed project/PYTHONPATH leakage. + result = subprocess.run( + [ + sys.executable, + "-I", + "-B", + "-c", + 'import sys; sys.path.insert(0, ".")\n' + code, + ], + cwd=root, + capture_output=True, + text=True, + timeout=5, + ) + results.append( + { + "check": name, + "passed": result.returncode == 0, + "exit_code": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + } + ) + except subprocess.TimeoutExpired as exc: + results.append( + { + "check": name, + "passed": False, + "exit_code": None, + "error": "timeout", + "stdout": str(exc.stdout), + "stderr": str(exc.stderr), + } + ) + return { + "passed": all(r["passed"] for r in results), + "checks": results, + "elapsed_ms": (time.perf_counter() - started) * 1000, + } + + +def strict_order(base, patches, checks): + with tempfile.TemporaryDirectory(prefix="delta-apply-") as temp: + root = Path(temp) + git(root, "init", "-q") + write(root, base) + steps = [] + for patch in patches: + r = git(root, "apply", "--whitespace=nowarn", "-", input=patch, check=False) + steps.append( + { + "applied": r.returncode == 0, + "stderr": r.stderr, + "exit_code": r.returncode, + } + ) + if r.returncode: + return { + "applied": False, + "steps": steps, + "state_hash": None, + "tests": None, + } + state = snapshot(root) + return { + "applied": True, + "steps": steps, + "state_hash": digest(state), + "tests": check_state(state, checks), + } + + +def merged_pair(base, a, b, checks=None): + """Git merge-tree in both parent orders, with fixed config and actual tree states.""" + with tempfile.TemporaryDirectory(prefix="delta-git-") as temp: + root = Path(temp) + git(root, "init", "-q") + + def commit(state, message): + old = snapshot(root) + for p in old.keys() - state.keys(): + (root / p).unlink() + write(root, state) + git(root, "add", "-A") + git(root, "commit", "--allow-empty", "-qm", message) + return git(root, "rev-parse", "HEAD").stdout.strip() + + base_id = commit(base, "base") + a_id = commit(a, "A") + git(root, "checkout", "-q", base_id) + b_id = commit(b, "B") + out = {} + for key, left, right in (("ab", a_id, b_id), ("ba", b_id, a_id)): + r = git(root, "merge-tree", "--write-tree", left, right, check=False) + if r.returncode not in (0, 1): + raise RuntimeError("merge-tree infrastructure error: " + r.stderr) + rows = r.stdout.splitlines() + tree = rows[0] + conflict_paths = sorted( + { + line.split("\t", 1)[1] + for line in rows[1:] + if "\t" in line + and line.split("\t")[0].split()[-1:] in (["1"], ["2"], ["3"]) + } + ) + # git show via each tracked tree path avoids filesystem archive extraction. + names = git(root, "ls-tree", "-r", "--name-only", tree).stdout.splitlines() + state = {p: git(root, "show", f"{tree}:{p}").stdout for p in names} + clean = r.returncode == 0 + out[key] = { + "clean": clean, + "conflict_paths": conflict_paths, + "state_hash": digest(state), + "tests": check_state(state, checks or {}) if clean else None, + "raw": r.stdout, + "state": state, + } + out["both_clean"] = out["ab"]["clean"] and out["ba"]["clean"] + out["states_differ"] = ( + out["ab"]["state_hash"] != out["ba"]["state_hash"] + if out["both_clean"] + else None + ) + return out + + +def commutativity(base, a, b, patch_a, patch_b, checks): + merge = merged_pair(base, a, b, checks) + strict = { + "ab": strict_order(base, [patch_a, patch_b], checks), + "ba": strict_order(base, [patch_b, patch_a], checks), + } + strict["both_apply"] = strict["ab"]["applied"] and strict["ba"]["applied"] + strict["states_differ"] = ( + strict["ab"]["state_hash"] != strict["ba"]["state_hash"] + if strict["both_apply"] + else None + ) + return {"strict_patch": strict, "three_way": merge} diff --git a/prototypes/structured-delta/experiments/evaluation/validate.py b/prototypes/structured-delta/experiments/evaluation/validate.py new file mode 100644 index 0000000..a01bce9 --- /dev/null +++ b/prototypes/structured-delta/experiments/evaluation/validate.py @@ -0,0 +1,66 @@ +"""Validate fixtures without exposing checks/witnesses to prediction modules.""" + +import argparse +import json +from pathlib import Path +from .runtime import snapshot, check_state, strict_order, digest + + +def validate(root, out): + records = [] + for path in sorted(root.glob("case[0-9]*")): + base, a, b = (snapshot(path / k) for k in ("base", "a", "b")) + gold = json.loads((path / "gold.json").read_text()) + checks = { + k: (path / "checks" / f"{k}.py").read_text() for k in ("base", "a", "b") + } + r = { + "id": path.name, + "base": check_state(base, {"base": checks["base"]}), + "a": check_state(a, {"a": checks["a"]}), + "b": check_state(b, {"b": checks["b"]}), + } + for key, target in (("a", a), ("b", b)): + applied = strict_order( + base, [(path / f"patch_{key.upper()}.diff").read_text()], {} + ) + r["patch_" + key] = { + "applied": applied["applied"], + "matches_snapshot": applied["state_hash"] == digest(target), + "details": applied if not applied["applied"] else None, + } + if gold["witness_available"]: + r["witness"] = check_state( + snapshot(path / "witness"), {"a": checks["a"], "b": checks["b"]} + ) + records.append(r) + print(path.name, "valid" if valid(r) else "INVALID", flush=True) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text( + json.dumps( + {"valid": all(valid(r) for r in records), "cases": records}, indent=2 + ) + + "\n" + ) + if not all(valid(r) for r in records): + raise SystemExit(1) + + +def valid(r): + return ( + all(r[k]["passed"] for k in ("base", "a", "b")) + and all(r["patch_" + k]["matches_snapshot"] for k in ("a", "b")) + and r.get("witness", {"passed": True})["passed"] + ) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument( + "--benchmark", type=Path, default=Path("experiments/benchmark/cases") + ) + p.add_argument( + "--out", type=Path, default=Path("experiments/results/validation.json") + ) + a = p.parse_args() + validate(a.benchmark, a.out) diff --git a/prototypes/structured-delta/experiments/online/__init__.py b/prototypes/structured-delta/experiments/online/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prototypes/structured-delta/experiments/online/sequences.py b/prototypes/structured-delta/experiments/online/sequences.py new file mode 100644 index 0000000..df07817 --- /dev/null +++ b/prototypes/structured-delta/experiments/online/sequences.py @@ -0,0 +1,24 @@ +"""Deterministic synthetic progress schedule, not real agent trajectories.""" + + +def stages(base, final, branch): + helper = f"_progress_{branch}.py" + draft = dict(base) + draft[helper] = "def ready():\n return True\n" + partial = dict(draft) + changed = sorted( + p for p in base.keys() | final.keys() if base.get(p) != final.get(p) + ) + if changed: + p = changed[0] + if p in final: + partial[p] = final[p] + else: + partial.pop(p, None) + return [draft, partial, dict(final)] + + +def functional_complete(snapshot, final): + return { + k: v for k, v in snapshot.items() if not k.startswith("_progress_") + } == final diff --git a/prototypes/structured-delta/experiments/online/simulator.py b/prototypes/structured-delta/experiments/online/simulator.py new file mode 100644 index 0000000..654931b --- /dev/null +++ b/prototypes/structured-delta/experiments/online/simulator.py @@ -0,0 +1,139 @@ +"""Recompute base-to-active deltas after every incremental patch event. + +Patch-event indices are the time unit. Runtime ms is separately recorded. +A merge failure and a semantic test failure are separate clocks. +""" + +import argparse +import json +import statistics +import time +from pathlib import Path +from ..delta.extractor import extract +from ..delta.interaction import detect +from ..evaluation.runtime import snapshot, merged_pair, strict_order, digest +from ..evaluation.freeze import verify +from .sequences import functional_complete + + +def simulate(root, benchmark, out): + verify(root) + if out.exists(): + raise FileExistsError(out) + out.mkdir(parents=True) + records = [] + for path in sorted(benchmark.glob("case[0-9]*")): + if not (path / "online" / "schedule.json").exists(): + continue + base = snapshot(path / "base") + final = {k: snapshot(path / k) for k in ("a", "b")} + active = {"a": dict(base), "b": dict(base)} + checks = {k: (path / "checks" / f"{k}.py").read_text() for k in ("a", "b")} + events = [] + for tick, event in enumerate( + json.loads((path / "online" / "schedule.json").read_text()), 1 + ): + branch = event["branch"] + target = snapshot(path / "online" / event["state"]) + inc = (path / "online" / event["patch"]).read_text() + application = strict_order(active[branch], [inc], {}) + if not application["applied"] or application["state_hash"] != digest( + target + ): + raise RuntimeError(f"Broken incremental sequence {path.name}:{event}") + active[branch] = target + start = time.perf_counter() + pred = detect(extract(base, active["a"]), extract(base, active["b"])) + elapsed = (time.perf_counter() - start) * 1000 + # Run branch requirements as soon as that branch's functional change is + # fully present, not after arbitrary cleanup (avoids artificial lead). + active_checks = { + k: v + for k, v in checks.items() + if functional_complete(active[k], final[k]) + } + merge = merged_pair(base, active["a"], active["b"], active_checks) + events.append( + { + "tick": tick, + "event": event, + "prediction": pred.to_dict(), + "detection_ms": elapsed, + "requirements_active": list(active_checks), + "merge_clean": merge["both_clean"], + "test_failure": any( + merge[k]["clean"] and not merge[k]["tests"]["passed"] + for k in ("ab", "ba") + ), + "merge": merge, + } + ) + # Gold is used only to score intervention correctness, not for detection. + gold = json.loads((path / "gold.json").read_text()) + + def first(predicate): + return next((e["tick"] for e in events if predicate(e)), None) + + detected = first(lambda e: e["prediction"]["label"] == "conflicting") + merge_failed = first(lambda e: not e["merge_clean"]) + test_failed = first(lambda e: e["test_failure"]) + failures = [v for v in (merge_failed, test_failed) if v is not None] + failed = min(failures) if failures else None + rec = { + "id": path.name, + "label": gold["label"], + "events": events, + "first_detection_tick": detected, + "first_merge_failure_tick": merge_failed, + "first_test_failure_tick": test_failed, + "first_failure_tick": failed, + "lead_ticks": failed - detected + if failed is not None and detected is not None + else None, + "semantic_lead_ticks": test_failed - detected + if test_failed is not None and detected is not None + else None, + "false_intervention": detected is not None + and gold["label"] != "conflicting", + "false_before_failure": detected is not None + and gold["label"] != "conflicting" + and (failed is None or detected < failed), + } + (out / f"{path.name}.json").write_text( + json.dumps(rec, ensure_ascii=False, indent=2) + "\n" + ) + records.append(rec) + print(path.name, flush=True) + conflict = [r for r in records if r["label"] == "conflicting"] + nonconf = [r for r in records if r["label"] != "conflicting"] + leads = [r["lead_ticks"] for r in conflict if r["lead_ticks"] is not None] + summary = { + "cases": len(records), + "events": sum(len(r["events"]) for r in records), + "conflicting_cases": len(conflict), + "conflicts_detected": sum( + r["first_detection_tick"] is not None for r in conflict + ), + "nonconflicting_cases": len(nonconf), + "false_interventions": sum(r["false_intervention"] for r in nonconf), + "false_early_interventions": sum(r["false_before_failure"] for r in nonconf), + "lead_ticks_conflicts": leads, + "mean_lead_ticks_conflicts": statistics.mean(leads) if leads else None, + "median_detection_ms": statistics.median( + e["detection_ms"] for r in records for e in r["events"] + ), + "rows": [{k: v for k, v in r.items() if k != "events"} for r in records], + } + (out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--root", type=Path, default=Path(".")) + p.add_argument( + "--benchmark", type=Path, default=Path("experiments/benchmark/cases") + ) + p.add_argument("--out", type=Path, default=Path("experiments/results/online-001")) + a = p.parse_args() + simulate(a.root, a.benchmark, a.out) diff --git a/prototypes/structured-delta/experiments/tests/__init__.py b/prototypes/structured-delta/experiments/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prototypes/structured-delta/experiments/tests/test_prototype.py b/prototypes/structured-delta/experiments/tests/test_prototype.py new file mode 100644 index 0000000..22c7f1e --- /dev/null +++ b/prototypes/structured-delta/experiments/tests/test_prototype.py @@ -0,0 +1,172 @@ +import unittest +from experiments.delta.extractor import extract +from experiments.delta.interaction import detect +from experiments.delta.model import Prediction +from experiments.evaluation.metrics import score +from experiments.evaluation.runtime import merged_pair, strict_order, digest +from experiments.benchmark.build import patch +from experiments.baselines import embedding, llm_intent +from experiments.online.sequences import stages, functional_complete + + +class ExtractorTests(unittest.TestCase): + def test_alias_and_relative_import(self): + base = { + "pkg/__init__.py": "", + "pkg/api.py": "def f(x): return x\n", + "pkg/c.py": "from .api import f as fetch\ndef run(): return fetch(1)\n", + } + a = {**base, "pkg/api.py": "def f(x,y): return x+y\n"} + b = { + **base, + "pkg/c.py": "from .api import f as fetch\ndef run(): return fetch(2)\n", + } + result = detect(extract(base, a), extract(base, b)) + self.assertEqual(result.label, "conflicting") + self.assertEqual(result.entities, ["pkg/api.py::f"]) + + def test_none_eq_is_not_identity(self): + base = {"m.py": "def f(x): return x\n"} + a = {"m.py": "def f(x):\n if x is None: return 0\n return x\n"} + b = {"m.py": "def f(x):\n if x == None: return 0\n return x\n"} + self.assertNotEqual( + detect(extract(base, a), extract(base, b)).label, "redundant" + ) + + def test_constant_fields_do_not_alias(self): + base = {"m.py": 'D={"a":0,"b":0}\ndef a(): D["a"]=1\ndef b(): D["b"]=1\n'} + a = {"m.py": base["m.py"].replace('D["a"]=1', 'D["a"]=2')} + b = {"m.py": base["m.py"].replace('D["b"]=1', 'D["b"]=3')} + self.assertEqual( + detect(extract(base, a), extract(base, b)).label, "independent" + ) + + def test_removed_call_records_conservative_reads(self): + base = { + "api.py": "def get(): return 1\n", + "c.py": "from api import get\ndef f(): return get()\n", + } + changed = {**base, "c.py": "from api import get\ndef f(): return 2\n"} + self.assertIn("api.py::get", extract(base, changed).reads) + + def test_parse_error_recorded_not_silently_valid(self): + d = extract({"x.py": "def f(): return 1\n"}, {"x.py": "def f(:\n"}) + self.assertTrue(d.unresolved) + + def test_rename_has_evidence(self): + d = extract({"x.py": "def a(): return 1\n"}, {"x.py": "def b(): return 1\n"}) + self.assertIn( + {"entity": "x.py::a", "operation": "rename", "to": "x.py::b"}, d.operations + ) + + def test_deletion_redundancy_and_order_symmetry(self): + base = {"x.py": "def f(): return 1\n"} + a = {"x.py": "# gone\n"} + d = extract(base, a) + self.assertEqual(detect(d, d).label, "redundant") + e = extract(base, {"x.py": "def f(): return 2\n"}) + self.assertEqual(detect(d, e).label, detect(e, d).label) + + +class MetricsTests(unittest.TestCase): + def test_denominators_and_exact_entity_localization(self): + rows = [ + { + "gold": "compatible", + "family": "a", + "cross_file": False, + "entities": ["m.py::f"], + "prediction": Prediction("conflicting", ["file:m.py"], 1, []).to_dict(), + }, + { + "gold": "conflicting", + "family": "b", + "cross_file": True, + "entities": ["m.py::g"], + "prediction": Prediction("conflicting", ["m.py::g"], 1, []).to_dict(), + }, + { + "gold": "independent", + "family": "a", + "cross_file": True, + "entities": [], + "prediction": Prediction("independent", [], 0, []).to_dict(), + }, + ] + result = score(rows) + self.assertEqual(result["conflict_precision"], 0.5) + self.assertEqual(result["compatible_false_positive_rate"], 1) + self.assertEqual(result["cross_file_conflict_recall"], 1) + self.assertEqual(result["localization_top1"], 0.5) + self.assertIsNone(score([])["conflict_precision"]) + + +class GitTests(unittest.TestCase): + def test_clean_cross_file_merge_can_fail_semantically(self): + base = { + "api.py": "def f(): return 1\n", + "c.py": "from api import f\ndef g(): return f()\n", + } + a = {**base, "api.py": 'def f(): return "one"\n'} + b = {**base, "c.py": "from api import f\ndef g(): return f()+1\n"} + result = merged_pair( + base, a, b, {"requirement": "from c import g\nassert g()==2"} + ) + self.assertTrue(result["both_clean"]) + self.assertFalse(result["ab"]["tests"]["passed"]) + self.assertFalse(result["states_differ"]) + + def test_patch_creation_deletion_exactness(self): + base = {"old.py": "X=1\n"} + final = {"new.py": "X=2\n"} + result = strict_order(base, [patch(base, final)], {}) + self.assertTrue(result["applied"]) + self.assertEqual(result["state_hash"], digest(final)) + + def test_text_overlap_is_not_semantic_truth(self): + base = {"m.py": "def f(x): return x\n"} + a = {"m.py": 'def f(x):\n """identity"""\n return x\n'} + b = {"m.py": "def f(x):\n if x is None: return 0\n return x\n"} + self.assertFalse(merged_pair(base, a, b)["both_clean"]) + self.assertEqual(detect(extract(base, a), extract(base, b)).label, "compatible") + + +class InterfaceTests(unittest.TestCase): + def test_optional_interfaces_without_network(self): + from unittest.mock import patch as mock + + d = extract({"m.py": "X=1\n"}, {"m.py": "X=2\n"}) + with mock.object( + embedding, + "invoke", + return_value={ + "vectors": [[1.0, 0.0], [0.0, 1.0]], + "model": "fixture", + "usage": None, + }, + ): + self.assertEqual( + embedding.predict("unused", "a", "b", d, d).label, "independent" + ) + with mock.object( + llm_intent, + "invoke", + side_effect=[ + {"description": "A"}, + {"description": "B"}, + {"label": "compatible", "entities": ["m.py::X"]}, + ], + ): + self.assertEqual(llm_intent.predict("unused", "a", "b").label, "compatible") + + def test_stage_completion_is_not_delayed_to_cleanup(self): + base = {"m.py": "X=1\n"} + final = {"m.py": "X=2\n"} + seq = stages(base, final, "a") + self.assertFalse(functional_complete(seq[0], final)) + self.assertTrue(functional_complete(seq[1], final)) + self.assertTrue(functional_complete(seq[2], final)) + + +if __name__ == "__main__": + unittest.main() diff --git a/prototypes/structured-delta/pyproject.toml b/prototypes/structured-delta/pyproject.toml new file mode 100644 index 0000000..5406c53 --- /dev/null +++ b/prototypes/structured-delta/pyproject.toml @@ -0,0 +1,7 @@ +[tool.ruff] +target-version = "py311" +line-length = 88 +exclude = ["artifacts", "experiments/benchmark/cases", "experiments/results"] + +[tool.ruff.lint] +select = ["E9", "F"] diff --git a/prototypes/structured-delta/tools/run_pilot.py b/prototypes/structured-delta/tools/run_pilot.py new file mode 100644 index 0000000..447cff7 --- /dev/null +++ b/prototypes/structured-delta/tools/run_pilot.py @@ -0,0 +1,57 @@ +"""Reproduce the pilot in a fresh ignored workspace without overwriting a run.""" + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +SOURCE_ROOT = Path(__file__).resolve().parents[1] + + +def run_module(workspace: Path, module: str, *arguments: str) -> None: + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + subprocess.run( + [sys.executable, "-B", "-m", module, *arguments], + cwd=workspace, + env=environment, + check=True, + ) + + +def reproduce(output: Path) -> None: + output = output.resolve() + if output == SOURCE_ROOT or SOURCE_ROOT.is_relative_to(output): + raise ValueError("Output must not contain the published source directory") + output.mkdir(parents=True, exist_ok=False) + shutil.copytree( + SOURCE_ROOT / "experiments", + output / "experiments", + ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "cases", "results"), + ) + shutil.copyfile(SOURCE_ROOT / "PROTOCOL.md", output / "PROTOCOL.md") + run_module(output, "experiments.benchmark.build") + run_module(output, "experiments.evaluation.validate") + run_module(output, "experiments.evaluation.freeze") + run_module(output, "experiments.evaluation.evaluate") + run_module(output, "experiments.online.simulator") + run_module( + output, + "experiments.evaluation.report", + "--static", + "experiments/results/run-001", + "--online", + "experiments/results/online-001", + "--out", + "REPORT.md", + ) + print(f"Report: {output / 'REPORT.md'}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, default=SOURCE_ROOT / "artifacts/pilot-001") + reproduce(parser.parse_args().out) diff --git a/prototypes/structured-delta/tools/test_verify_source.py b/prototypes/structured-delta/tools/test_verify_source.py new file mode 100644 index 0000000..d315131 --- /dev/null +++ b/prototypes/structured-delta/tools/test_verify_source.py @@ -0,0 +1,49 @@ +"""Regression coverage for cross-version verification and exact integrity.""" + +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from verify_source import digest, verify + + +class SourceVerificationTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.module = self.root / "experiments/__init__.py" + self.module.parent.mkdir() + self.module.write_bytes(b'"""Source fixture."""\n') + protocol = b"Test protocol\n" + (self.root / "PROTOCOL.md").write_bytes(protocol) + manifest = { + "protocol_sha256": digest(protocol), + "modules": { + "experiments/__init__.py": { + "published_sha256": digest(self.module.read_bytes()), + "program_ast_sha256": digest(b"Historical interpreter AST dump"), + } + }, + } + (self.root / "SOURCE_PROVENANCE.json").write_text(json.dumps(manifest)) + + def test_historical_ast_format_does_not_reject_identical_source(self): + with patch("ast.dump", side_effect=AssertionError("Not cross-version stable")): + verify(self.root) + + def test_even_semantically_identical_source_edits_are_rejected(self): + self.module.write_bytes(self.module.read_bytes() + b"\n") + with self.assertRaisesRegex(ValueError, "Published source changed"): + verify(self.root) + + def test_unrecorded_modules_are_rejected(self): + (self.module.parent / "extra.py").write_text("VALUE = 1\n") + with self.assertRaisesRegex(ValueError, "module inventory"): + verify(self.root) + + +if __name__ == "__main__": + unittest.main() diff --git a/prototypes/structured-delta/tools/verify_source.py b/prototypes/structured-delta/tools/verify_source.py new file mode 100644 index 0000000..cbbbb74 --- /dev/null +++ b/prototypes/structured-delta/tools/verify_source.py @@ -0,0 +1,43 @@ +"""Check exact source bytes and syntax without experimental data. + +Historical AST digests document the original same-interpreter comparison. +They are not portable across Python AST schema and ast.dump format changes. +Exact byte hashes enforce source integrity on every supported interpreter. +""" + +import ast +import hashlib +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def digest(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def verify(root: Path = ROOT) -> None: + manifest = json.loads((root / "SOURCE_PROVENANCE.json").read_text()) + paths = { + path.relative_to(root).as_posix() + for path in (root / "experiments").rglob("*.py") + if "__pycache__" not in path.parts + and "cases" not in path.parts + and "results" not in path.parts + } + if paths != set(manifest["modules"]): + raise ValueError("Published module inventory differs from provenance") + for path, expected in manifest["modules"].items(): + content = (root / path).read_bytes() + if digest(content) != expected["published_sha256"]: + raise ValueError(f"Published source changed: {path}") + ast.parse(content, filename=path) + if digest((root / "PROTOCOL.md").read_bytes()) != manifest["protocol_sha256"]: + raise ValueError("Historical protocol changed") + print(f"Verified {len(paths)} published modules and historical protocol") + + +if __name__ == "__main__": + verify()