From 9862ab6c2b78a69655fd6a86a335c7a21bc5b15c Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Sat, 29 Aug 2026 05:37:17 +0200 Subject: [PATCH 1/2] Both method selectors in one pass: method_ref as a typed run source, method_id on the tooling routes, the crate routes added MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The addressing-methods Phase 4 leg for pipelex-sdk (design: workspace wip/addressing-methods/design.md), mirroring @pipelex/sdk v0.16.0: - method_ref is a typed keyword run source on execute/start/start_and_wait (layer 2 — the runner resolves it), pairing with nothing: client-side guards mirror the server's 422s against inline mthds_contents and method_id, while inline+method_id keeps its documented linkage exception and pipe_code beside method_ref stays legal. Provenance comes back typed: PipelexRunResultStart / PipelexExecuteResult carry method_provenance {address, tag, commit_sha}. - validate takes method_ref= / method_id= keyword selectors under the strict tooling XOR (a selector body carries no mthds_contents key at all; mthds_sources is inline-only); resolve/codegen are added (pipelex_sdk/crate_models.py) with the typed method_id pass-through and the three-way XOR at construction; build_inputs gains the shared files-XOR-method_ref closure and refuses method_id with a teaching error. - The reserved-extra guard spans both layers now (method_ref joins method_id), and a method_ref-carrying build_inputs/resolve/codegen gets a fetch-sized 3-minute budget so a cold-cache server clone is not misreported as an unreachable server. The run routes and validate already ride the 20-min blocking ceiling, so they need no budget change. - Version cut in the same PR: v0.8.0, changelog heading dated 2026-08-29. Hosted availability of the new selectors on api.pipelex.com follows the platform deploy (Phase 3); tests pin the wire bodies and guards against mocks. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WoLcQhnFiPgVmDzHqPkRmQ --- CHANGELOG.md | 18 ++ README.md | 14 + docs/architecture.md | 47 ++- pipelex_sdk/build_models.py | 58 +++- pipelex_sdk/client.py | 411 +++++++++++++++++++++------ pipelex_sdk/crate_models.py | 156 ++++++++++ pipelex_sdk/execute_result.py | 7 + pipelex_sdk/runs.py | 30 ++ pyproject.toml | 4 +- tests/unit/test_build_inputs.py | 52 ++++ tests/unit/test_client_method_ref.py | 176 ++++++++++++ tests/unit/test_client_validate.py | 81 ++++++ tests/unit/test_crate_routes.py | 168 +++++++++++ uv.lock | 2 +- 14 files changed, 1122 insertions(+), 102 deletions(-) create mode 100644 pipelex_sdk/crate_models.py create mode 100644 tests/unit/test_client_method_ref.py create mode 100644 tests/unit/test_crate_routes.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b22e916..b338a38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [v0.8.0] - 2026-08-29 + +### Added + +- **`method_ref` is a typed run source.** `execute`, `start`, and `start_and_wait` take a published method's address — `github.com//[/][@]` — as a keyword parameter beside the protocol's inline source, mirroring `@pipelex/sdk` v0.16.0. It is a layer-2 Pipelex-API argument the RUNNER resolves (git fetch at the tag, package located by manifest identity), deliberately separate from the hosted-only `method_id`: an address is meaningful against a bare runner, a catalog id is not. Served by pipelex-api >= 0.21.0; on `api.pipelex.com` availability follows the platform deploy that forwards it. An empty string is treated as absent, a non-string raises at the boundary, `extra={"method_ref": …}` is rejected (the key joins the reserved set), and the selector survives `start_and_wait`'s blocking-execute fallback. +- **Provenance comes back typed.** A `method_ref` run's start ack is the new `PipelexRunResultStart` (`pipelex_sdk.runs`), carrying `method_provenance` — the new `MethodProvenance` shape `{address, tag, commit_sha}`, the SHA being what keeps the run explainable when a tag moves — and `PipelexExecuteResult` declares the same field on the blocking path. Both are `None` for inline-source and `method_id` runs. +- **Client-side exclusivity guards mirroring the server's 422s.** A `method_ref` is a complete run source, so it pairs with nothing: combining it with inline `mthds_contents` or with `method_id` raises `PipelineRequestError` whose wording mirrors the server's validator — before anything hits the wire. The documented run-route exception is untouched: inline source + `method_id` stays legal (the inline source runs; the id demotes to run-history linkage), and `pipe_code` beside a `method_ref` stays legal (it overrides the manifest's `main_pipe`). +- **`validate` takes method selectors.** `mthds_contents` is now optional, and the new keyword parameters `method_ref=` (runner-resolved by address, the package's real file names feeding the diagnostics' source labels) and `method_id=` (hosted-only, platform-resolved) select what is validated — under the tooling routes' strict three-way XOR: exactly one selector, no linkage exception, `mthds_sources` legal only beside inline contents. A selector validation sends no `mthds_contents` key at all. A selector-resolution failure (fetch failure, no package at the address, an unknown id) is a non-2xx, never an `is_valid: false` verdict. +- **The crate routes, with the typed `method_id` pass-through.** New `resolve()` and `codegen()` client methods for `POST /v1/resolve` (the normalized library crate) and `POST /v1/codegen` (stamped typed artifacts plus their `codegen.lock`), with the new `pipelex_sdk.crate_models` wire models (`ResolveRequest` / `ResolveResponse`, `CodegenRequest` / `CodegenResponse`, `GeneratedArtifact`, `CodegenKind` / `CodegenTarget`). Their closure is exactly one of inline `files` / an address-form `method_ref` / the hosted `method_id`, enforced at request construction as well as by the server; `method_id` is a pure server pass-through the platform resolves (an unknown or foreign-org id is a `404`, a stored method with no MTHDS source a `422`). This closes the JS-parity gap the architecture doc carried for the two routes. +- **`build_inputs` takes a `method_ref` closure.** `BuildInputsRequest` now extends the shared `CrateRequestBase` envelope (`files` XOR `method_ref`); the address form is server-resolved, the registry form keeps its `501`. +- **A `method_ref` request gets a fetch-sized budget.** Resolving an address can make the server clone a repository before it answers, and the server-side clone timeout runs well past the client's 30s management budget on a cold cache — an abort there would report a healthy, still-cloning server as unreachable. A `method_ref`-carrying `build_inputs`, `resolve`, or `codegen` uses an internal 3-minute budget; it is internal (no new caller-facing parameter) and inert behind the hosted gateway's own cap. The run routes and `validate` need no such override — they already ride the 20-min blocking ceiling, unlike the JS SDK's short start budget. + +### Changed + +- **Breaking: `start` returns `PipelexRunResultStart`.** A widening of the previous `RunResultStart` return type (one typed optional field over the extension-open base) — no caller change needed unless a caller depended on the exact class. +- **Breaking: `BuildInputsRequest.files` is optional** (the closure is `files` XOR `method_ref`, checked at construction), and handing the model a `method_id` raises a teaching error naming the migration — the `/v1/build/*` projections are deliberately excluded from the hosted tooling selector, so a stored method is expanded by the caller (fetch it with `get_method` and pass its source as `files`). This SDK never had client-side by-id expansion legs to delete, so the JS release's deletions have no Python counterpart. +- **`extra` now also rejects `method_ref`**, for the same reason it rejects every named request option: `extra` merges last into the body, so a smuggled copy would overwrite the validated named option and bypass the selector-exclusivity checks. The guard's wording changed from "hosted args" to "reserved request args" now that it spans two layers. + ## [v0.7.0] - 2026-08-28 ### Added diff --git a/README.md b/README.md index 2260e8c..6bdfcc1 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,20 @@ async def main() -> None: print(result.main_stuff) ``` +### Run a published method by address + +A method reaches every method-taking call in exactly one of three forms: inline source, a `method_ref` address (`github.com//[/][@]`, resolved by the server — pipelex-api >= 0.21.0; on `api.pipelex.com` availability follows the platform deploy that forwards it), or a hosted `method_id` (`mt_…`, resolved by the platform). A `method_ref` pairs with nothing — it is a complete run source — and its runs carry typed provenance: + +```python +ack = await client.start(method_ref="github.com/Pipelex/methods/documents@v0.1.0", inputs={...}) +print(ack.method_provenance.commit_sha) # the SHA actually fetched — stable even if the tag moves +result = await client.wait_for_result(ack.pipeline_run_id) + +# The tooling routes take the same selectors under a strict XOR (exactly one, no pairing): +report = await client.validate(method_ref="github.com/Pipelex/methods/documents@v0.1.0") +report = await client.validate(method_id="mt_123") +``` + ### Long runs: start + poll explicitly Behind the hosted gateway, a synchronous `execute()` is cut off at ~30s and surfaces a `PipelineExecuteTimeoutError` pointing here. For long methods, drive the durable lifecycle yourself — the run survives client disconnects and is resumable by `pipeline_run_id`: diff --git a/docs/architecture.md b/docs/architecture.md index d000f34..3f8231c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -79,18 +79,24 @@ Everything else stays the inherited regime: the protocol's optional 202 async-de The override also **enriches the return type**: it re-validates the base result into a `PipelexExecuteResult` (`pipelex_sdk/execute_result.py`), a `DictRunResultExecute` subtype that adds a resolved `.main_stuff` accessor (dug out of `pipe_output`'s working memory via the response's `main_stuff_name`, raising `MissingMainStuffError` if unlocatable). This gives blocking and durable results the **same output accessor** — `result.main_stuff` — so callers never branch on which path ran. `_map_run_result_to_run_results` reads that accessor too, keeping the resolution single-sourced. -## Hosted run extensions (`method_id`) +## Method selectors on the run routes (`method_ref` + `method_id`) -The protocol's run args (`pipe_code`, `mthds_contents`, `inputs`, …) are the base client's named parameters and stay pure. The hosted API's *own* run args are named parameters **here**, on `execute` / `start` / `start_and_wait`. Today that is one, `method_id`, and the reasoning generalizes to every hosted-only argument that follows: +The protocol's run args (`pipe_code`, `mthds_contents`, `inputs`, …) are the base client's named parameters and stay pure. This client's *own* run args are named parameters **here**, on `execute` / `start` / `start_and_wait` — one per layer of the stack it fronts: -- **A named parameter, not an `extra` entry.** Typing its own platform's arguments is the one job a layer-3 client exists to do; `extra` stays the escape hatch for an extension this client does not know about (a third-party server, or a newer version of ours). That split, and the reserved-key guard behind it, follow the layered extension policy: a hosted client types its own platform's arguments and guards them per layer, and that guard must never be pushed down into the protocol package. It was previously passed as `extra={"method_id": …}`, which worked but documented nothing and validated nothing. -- **It reaches the wire through the base client's extension mechanism.** `_merge_hosted_run_extensions` folds it into the `extra` mapping handed to `super().execute` / `super().start`, which merges it into the body as a top-level property without knowing what it means. That is the layering working as designed, not a workaround: the protocol client stays catalog-agnostic while the hosted client owns the concept. -- **The guard is per layer.** `method_id` is rejected inside `extra` *here*, and must never become reserved in `mthds`: a protocol client talking to another vendor's server has no business rejecting that vendor's arguments. -- **Pure pass-through — nothing is expanded client-side.** The platform resolves the id against the org's catalog. The input-preparation helpers are the deliberate contrast: `build_inputs` and `prepare_inputs` take the method closure as inline `files` (with an optional `pipe_ref`) and accept no `method_id` at all — resolving a catalog id to its files client-side is deferred, additive work — so the id is a run option only and reaches no other wire body. -- **Alone it is a run source; alongside an inline source it is linkage.** With no inline source the platform resolves and runs the stored method. With one, the inline source is what RUNS (precedence) and the id is recorded as run-history linkage on the Run row — the index key `GET /v1/runs?method_id=` queries. The base client's "something to run" precondition is satisfied because the merged extension mapping is non-empty, so a `method_id`-only run is sent rather than refused client-side. -- **It rides the blocking fallback too.** `start_and_wait` degrades to `POST /v1/execute` against a runner with no run store, and the selector goes with it — so a bare `pipelex-api` answers the `422` that names the key instead of the client silently dropping it and running something else. -- **An empty string is treated as absent.** `method_id=""` selects no method and links nothing, so it is not sent and does not satisfy the precondition. -- **A non-string raises at the boundary.** A published client validates its request-option types where it names them, raising `PipelineRequestError` rather than dropping or forwarding a wrong-typed value — so one wrong value gets one answer. Without the guard the partition is arbitrary: a bare truthiness check drops the falsy wrong types (`0`, `[]`) and forwards the truthy ones (`123`, `["mt_1"]`) to a server `422`, which is a *different* partition than `@pipelex/sdk` makes for the same argument on the same wire. Both SDKs now make the same one. +- **`method_ref`** (layer 2 — a Pipelex-API extension) is a published method's address, `github.com//[/][@]`, resolved by the RUNNER itself (git fetch at the tag, package located by manifest identity; pipelex-api >= 0.21.0). An address is meaningful against a bare runner — `git clone` needs no catalog — which is exactly why it is a runner argument and not a hosted one. On `api.pipelex.com`, availability follows the platform deploy that forwards it. +- **`method_id`** (layer 3 — a hosted-platform extension) is a stored method's catalog id (`mt_…`), resolved by the PLATFORM against the org's catalog; the runner never sees it. It is meaningless off-platform: an open-source runner answers a `422` naming the key. + +The reasoning below covers both and generalizes to every extension argument that follows: + +- **A named parameter, not an `extra` entry.** Typing its own stack's arguments is the one job this client exists to do; `extra` stays the escape hatch for an extension this client does not know about (a third-party server, or a newer version of ours). That split, and the reserved-key guard behind it, follow the layered extension policy: the client types its own arguments and guards them per layer, and that guard must never be pushed down into the protocol package. +- **They reach the wire through the base client's extension mechanism.** `_merge_run_extensions` folds them into the `extra` mapping handed to `super().execute` / `super().start`, which merges them into the body as top-level properties without knowing what they mean. That is the layering working as designed, not a workaround: the protocol client stays agnostic while this client owns the concepts. +- **The guard is per layer.** `method_ref` and `method_id` are rejected inside `extra` *here* (a smuggled copy would overwrite the validated named option and bypass the exclusivity checks — `extra` merges last), and must never become reserved in `mthds`: a protocol client talking to another vendor's server has no business rejecting that vendor's arguments. +- **`method_ref` pairs with NOTHING.** It is a complete run source — the fetched package carries its `.mthds` and its entry pipe — so combining it with inline `mthds_contents` or with `method_id` raises `PipelineRequestError` before anything hits the wire, mirroring the server's own 422s (an address run has its own provenance and needs no linkage id). `pipe_code` beside it is legal: it overrides the manifest's `main_pipe`. This SDK names no bundle encodings (`files` / `bundle_b64`), so that arm of the server's exclusivity has no client-side twin here. +- **`method_id` is a pure pass-through — nothing is expanded client-side.** The platform resolves the id against the org's catalog. Alone it is a run source; alongside an inline source the inline source is what RUNS (precedence) and the id is recorded as run-history linkage on the Run row — the index key `GET /v1/runs?method_id=` queries. That linkage exception is the run routes' alone; the tooling routes are strict (below). The base client's "something to run" precondition is satisfied because the merged extension mapping is non-empty, so a selector-only run is sent rather than refused client-side. +- **Provenance comes back typed.** A `method_ref` run's `start` ack is `PipelexRunResultStart` (`pipelex_sdk.runs`) carrying `method_provenance: MethodProvenance | None` — `{address, tag, commit_sha}`, the SHA being what keeps the run explainable when a tag moves — and `PipelexExecuteResult` declares the same field on the blocking path. Both are `None` for inline-source and `method_id` runs. The server fetches the package before the 202, so provenance rides the ack; the base `start` already uses `request_timeout_seconds` (the 20-min blocking ceiling) for every request, so a cold-cache clone needs no special client budget here — unlike the JS SDK, whose short start budget had to be widened for `method_ref`. +- **Both ride the blocking fallback too.** `start_and_wait` degrades to `POST /v1/execute` against a runner with no run store, and the selectors go with it — a `method_ref` run must run the same fetched package there, and a bare `pipelex-api` answers the `422` that names `method_id` instead of the client silently dropping it and running something else. +- **An empty string is treated as absent.** An empty selector selects nothing and links nothing, so it is not sent and does not satisfy the precondition. +- **A non-string raises at the boundary.** A published client validates its request-option types where it names them (`_normalized_selector`), raising `PipelineRequestError` rather than dropping or forwarding a wrong-typed value — so one wrong value gets one answer. Without the guard the partition is arbitrary: a bare truthiness check drops the falsy wrong types (`0`, `[]`) and forwards the truthy ones (`123`, `["mt_1"]`) to a server `422`, which is a *different* partition than `@pipelex/sdk` makes for the same argument on the same wire. Both SDKs make the same one. ## Run lifecycle (hosted extension) @@ -128,10 +134,13 @@ These poll GETs go through `_send_or_unreachable`, so a transport failure surfac `RunFailedError`, `RunTimeoutError`, and `RunLifecycleUnavailableError` are owned in `pipelex_sdk/errors.py`. `RunStillRunningError` — the protocol `execute()` 202-degrade error — stays owned by `mthds` and is re-exported from `pipelex_sdk/errors.py` so consumers have a single import home for all run/lifecycle errors. -## `validate` override (Pipelex-API presentation + sources) +## `validate` override (Pipelex-API presentation + sources + selectors) The protocol `validate` is **overridden** (not inherited) to add the Pipelex-API extensions the bare protocol route doesn't carry, while keeping the inherited protocol error regime (a no-verdict non-2xx surfaces as `httpx.HTTPStatusError`, not `ApiResponseError` — the verdict itself is always a 200 discriminated on `is_valid`): +- **WHAT is validated arrives in exactly one of three forms — the tooling routes' strict three-way XOR.** Inline `mthds_contents` (the protocol's own envelope), `method_ref=` (runner-resolved by address through the same fetch path as a `method_ref` run, the package's real file names feeding the diagnostics' source labels), or `method_id=` (hosted-only, platform-resolved before the runner sees the request). The routes are stateless, so there is no linkage exception: zero selectors or any pairing raises `PipelineRequestError` client-side, mirroring the server's request-shape `422`. `mthds_sources` is an inline-contents companion only and is rejected beside a selector — a selector validation gets its labels from the real file names. A selector-resolution failure (a fetch failure, no package at the address, an unknown or foreign-org id) is a non-2xx — never an `is_valid: false` verdict, which is reserved for actual MTHDS content. +- **The two wire paths differ, deliberately.** The inline path reuses the inherited `_post_validate` body-building seam; a selector validation must NOT carry the `mthds_contents` key at all (the server XORs on presence, and an empty list is a request-shape `422`), so its body is built in the override and sent on the same inherited `_send` transport seam. Both paths get the markdown render injection and both parse into `PipelexValidationResultAdapter`. + - **Markdown render is always injected.** `validate(...)` adds `"markdown"` to the `render` list (de-duplicated, caller tokens first) so both a valid `PipelexValidationReport` and a produced `PipelexInvalidReport` carry `rendered_markdown`. Unknown render tokens are server-side lenient-ignored. - **`mthds_sources`** is a named parameter (parallel to `mthds_contents`) threaded onto each diagnostic's `source`; sent only when provided. - **`views`** is the opt-in for the server's structured views, on both `validate` and `validate_files`. `input_form` — named by the `VALIDATION_VIEW_INPUT_FORM` constant — is the only token today. Unlike `render`, the list travels **verbatim**: nothing is injected, nothing is de-duplicated, and an explicit `[]` is sent as `[]`, because the server resolves the tokens as a set and lenient-ignores the ones it does not know (never a `422`), so client-side normalization would only hide what the caller asked for. Left at `None` the key is not sent at all, which is what keeps the default response byte-identical for the consumers that discard views (hook pipelines, CI gates, agent loops). The constant is a constant rather than a closed enum on purpose: the request boundary is deliberately open, so a stale token never fails a call. @@ -162,6 +171,14 @@ The types are imported and used, never re-exported from `pipelex_sdk`. Re-export **The one break.** A valid report whose contracts predate the presence/multiplicity reshape — an input carrying the boolean `optional` instead of `presence`, or missing `multiplicity` / `item_count` — no longer parses, where it used to ride through untyped. The hosted plane emits the reshaped contracts, so this bites only against runners older than that reshape, and there is no compatibility shim by design: an artifact that does not conform to the standard version this package pins is version drift, and saying so at the parse is the point. +## Crate routes (`resolve` / `codegen`) and the build closure selector + +`resolve` and `codegen` (`pipelex_sdk/crate_models.py` + the two client methods) are the second crate-family surface, mirroring the JS SDK's v0.12.0 routes: `POST /v1/resolve` emits the normalized library crate (the MTHDS Library Crate Format — typed as opaque transport, `dict[str, Any]`, because the crate schema is owned by the standard and a restatement here would be free to drift), and `POST /v1/codegen` projects that crate through the `kind` × `target` axes into stamped typed artifacts plus their `codegen.lock` (write both verbatim and the offline `pipelex codegen check` passes on the tree). Both follow the build routes' 200-verdict discipline, sharing `CrateInvalidReport`, and ride `_request_product`, so a no-verdict condition raises the typed `ApiResponseError`. + +Their closure selector is the strict three-way XOR — inline `files`, an address-form `method_ref` (server-resolved; the registry form keeps its `501`), or the hosted `method_id` pass-through — enforced at request construction (`CrateToolingRequest._exactly_one_selector`) as well as by the server. `BuildInputsRequest` shares the `CrateRequestBase` envelope (`files` XOR `method_ref`) but takes NO `method_id`: the `/v1/build/*` projections are deliberately excluded from the hosted tooling selector, and a `method_id` handed to it raises a teaching error naming the migration (fetch the method with `get_method` and pass its source as `files`) rather than letting pydantic silently ignore the key. + +**A `method_ref` closure gets a fetch-sized budget.** Resolving an address can make the server clone a repository before it answers, and the server-side clone timeout runs well past the 30s management budget on a cold cache — an abort there would report a healthy, still-cloning server as unreachable. So a `method_ref`-carrying `build_inputs` / `resolve` / `codegen` uses an internal 3-minute budget (`_METHOD_REF_FETCH_TIMEOUT_SECONDS`, threaded through `_request_product`'s `request_timeout` override); it is internal (no new caller-facing parameter) and inert behind the hosted gateway's own cap. Mirrors the JS SDK's fetch budget; the run routes and `validate` need none because they already ride the 20-min blocking ceiling. + ## Pipelex product surface (hosted management routes) The hosted catalog/account routes the webapp drives (`pipelex_sdk/product_models.py` + the client's product methods). Every route rides the same `{base}/v1/*` surface, `Authorization: Bearer`, org-from-JWT contract as the protocol routes, and goes through `_request_product`, which maps a non-2xx `problem+json` to a typed `ApiResponseError` — **consumers branch on `.code`, never the HTTP status**. @@ -211,13 +228,13 @@ The wire models are snake_case Pydantic v2. Response models are extension-open ( This SDK is a port of the TypeScript `@pipelex/sdk` (`PipelexApiClient`) and tracks it closely, but it is **not surface-complete, and this section is where the gaps are named.** The Checkpoint-5 parity audit walked the JS `src/client.ts`, `src/index.ts` (the public barrel), and `docs/architecture.md`, plus a field-by-field sweep of `runs.ts` / `product-models.ts` / `models.ts`; it concluded surface-completeness, and that conclusion went stale as the JS SDK grew. The honest list of what has no Python counterpart today: -- **Tooling routes** — `lint`, `format`, `resolve`, `codegen`. +- **Tooling routes** — `lint`, `format` (`resolve` and `codegen` shipped in 0.8.0 with the method selectors and are **not** gaps). - **Authoring helpers** — `build_output`, `build_runner`, `concept`, `pipe_spec` (`build_inputs` shipped in 0.5.0 and is **not** a gap). -- **Offline helpers** — `run_codegen_check` (the codegen drift check) and `get_method_closure` (client-side sugar that parses the polymorphic `mthds` source into a run-ready closure). +- **Offline helpers** — `run_codegen_check` (the codegen drift check) and `get_method_closure` (client-side sugar that parses the polymorphic `mthds` source into a run-ready closure — in the JS SDK it is the documented migration target for the deleted by-id expansion legs; this SDK never had such legs, so the utility stays deferred rather than required). -None of them is moved by the releases this SDK last tracked, and each stays deferred rather than silently missing. Everything else — the protocol routes, the durable lifecycle, the whole product surface, and the errors — does have a Python equivalent. +Each stays deferred rather than silently missing. Everything else — the protocol routes, the durable lifecycle, the whole product surface, and the errors — does have a Python equivalent. -**Methods** — everything outside the gap list above has a counterpart: protocol (`execute`, `start`, `validate`, `validate_files`, `models`, `version`), durable lifecycle (`get_run_status`, `get_run_result`, `wait_for_result`, `start_and_wait`, the private `_supports_run_lifecycle` / `_execute_blocking`), the whole product surface (profile, methods CRUD with paged listing and the two iterators, organizations, billing, Pipelex API keys, gateway key, onboarding, storage, run records with `get_run_detail`), `build_inputs`, the input-preparation surface (`upload_file` / `prepare_inputs`), and `health`. +**Methods** — everything outside the gap list above has a counterpart: protocol (`execute`, `start`, `validate`, `validate_files`, `models`, `version`), durable lifecycle (`get_run_status`, `get_run_result`, `wait_for_result`, `start_and_wait`, the private `_supports_run_lifecycle` / `_execute_blocking`), the whole product surface (profile, methods CRUD with paged listing and the two iterators, organizations, billing, Pipelex API keys, gateway key, onboarding, storage, run records with `get_run_detail`), `build_inputs`, the crate routes (`resolve`, `codegen`), the input-preparation surface (`upload_file` / `prepare_inputs`), and `health`. The method selectors (`method_ref` / `method_id`) match the JS v0.16.0 surface across the run and tooling methods, with one signature-shape divergence: JS `validate` takes a `ValidateMethodSelector` object in place of its first argument, while Python takes `method_ref=` / `method_id=` keyword parameters — same wire, same XOR, idiomatic per language. **Models** — field-for-field across the run-lifecycle types and the product wire models. Deliberate idiomatic ports (not gaps): milliseconds → seconds (`interval_seconds` / `timeout_seconds` / `elapsed_seconds`); the JS `AbortSignal` → Python `asyncio` cancellation (no `signal` field); JS inline string-unions promoted to `StrEnum`s (`OrgRole`, `PipeStatus`, the onboarding fields) with identical wire values; response models are `extra="allow"` for forward-compat. The Pipelex validation narrowing is **owned here** (`pipelex_sdk.validation_models`), narrowing `mthds`'s neutral verdict bases (the resolved follow-up #9); the brand-neutral `Dict*` wire concretes (`DictRunResultExecute`) are reused from `mthds` by inheritance — they are a shared wire contract the `pipelex` runtime also builds on — rather than duplicated as `pipelex-sdk-js` does. Two divergences worth naming: the page envelopes keep the wire's snake_case `next_cursor`, where the JS mirror renamed it `nextCursor` for its own consumers; and the method-files catalog converter (`parse_method_files` / `serialize_method_files`) lives in this package, where the JS pair lives in `mthds-js` because `pipelex-mcp` consumes the same format and wanted one owner. There is no second Python consumer, and the catalog serialization is a Pipelex product concern rather than an MTHDS protocol one, so this SDK is a proper home for it. If `mthds-python` ever grows an owner for the format, this SDK adopts it then. diff --git a/pipelex_sdk/build_models.py b/pipelex_sdk/build_models.py index a1c4086..2fceee3 100644 --- a/pipelex_sdk/build_models.py +++ b/pipelex_sdk/build_models.py @@ -4,6 +4,8 @@ The Python SDK had no `/v1/build/*` coverage; `prepare_inputs` needs the explicit inputs template, so this adds the `build_inputs` counterpart of `pipelex-sdk-js`'s `buildInputs` (only this route — the other build projections are not needed here). +The crate routes (`/v1/resolve`, `/v1/codegen`) share this module's `CrateRequestBase` +envelope and `CrateInvalidReport` arm through `crate_models.py`. A produced verdict is a `200` discriminated on `is_valid`; a no-verdict condition (unknown `pipe_ref`, auth, server fault) throws `ApiResponseError`. """ @@ -28,14 +30,64 @@ class MthdsFileItem(BaseModel): source: str | None = None -class BuildInputsRequest(BaseModel): - """Request for `POST /v1/build/inputs`. The closure is supplied as inline `files`.""" +class CrateRequestBase(BaseModel): + """The closure selector every crate-family route shares — `/v1/resolve`, + `/v1/codegen`, and `/v1/build/*` (mirror of the server's `MthdsFilesRequest` and + of `pipelex-sdk-js`'s `CrateRequestBase`). + + Supply the closure EITHER as inline `files` OR as a `method_ref` — never both, and + never neither. An **address-form** `method_ref` + (`github.com//[/][@]`) is resolved by the server + (pipelex-api >= 0.21.0): the repository is fetched at the tag, the package is + located by manifest identity, and its `.mthds` files feed the closure with their + real relative paths as per-file sources. The **registry form** (any non-address + reference) stays reserved and answers `501` until a method registry exists. + + The subclasses own the exclusivity validator, because the crate routes add a third + selector (the hosted `method_id`) that the build projections deliberately refuse. + """ + + files: list[MthdsFileItem] | None = None + method_ref: str | None = None + + +class BuildInputsRequest(CrateRequestBase): + """Request for `POST /v1/build/inputs`. The closure is inline `files` XOR a + `method_ref` address; there is NO by-id form — the `/v1/build/*` projections are + deliberately excluded from the hosted tooling selector (`method_id` covers + `validate` / `resolve` / `codegen` only), so a stored method is expanded first + (fetch it with `get_method` and pass its source as `files`). + """ - files: list[MthdsFileItem] pipe_ref: str | None = None format: InputsTemplateFormat = "json" explicit: bool = False + @model_validator(mode="before") + @classmethod + def _refuse_method_id(cls, data: Any) -> Any: + # A teaching error beats pydantic's default extra="ignore" silently dropping the + # key: a caller migrating from the by-id habit must learn the build routes have + # no by-id form (mirrors the JS `method_id: never` pin + runtime guard). + raw: Any = data + if isinstance(data, dict) and "method_id" in data: + msg = ( + "build_inputs takes no method_id — the /v1/build/* projections are excluded from the " + "hosted tooling selector (it covers validate/resolve/codegen only). Expand the stored " + "method first: fetch it with get_method and pass its MTHDS source as files." + ) + raise ValueError(msg) + return raw + + @model_validator(mode="after") + def _exactly_one_closure_selector(self) -> Self: + # Mirrors the server's own XOR so an illegal shape fails at construction, before + # anything hits the wire. + if (self.files is None) == (self.method_ref is None): + msg = "provide exactly one of `files` or `method_ref`" + raise ValueError(msg) + return self + class BuildInputsValidReport(BaseModel): """The `is_valid: true` arm. The template rides `inputs` (json) or `inputs_toml` (toml).""" diff --git a/pipelex_sdk/client.py b/pipelex_sdk/client.py index e14c1e6..e601af9 100644 --- a/pipelex_sdk/client.py +++ b/pipelex_sdk/client.py @@ -33,6 +33,14 @@ from typing_extensions import override from pipelex_sdk.build_models import BuildInputsRequest, BuildInputsResponse, BuildInputsResponseAdapter, MthdsFileItem +from pipelex_sdk.crate_models import ( + CodegenRequest, + CodegenResponse, + CodegenResponseAdapter, + ResolveRequest, + ResolveResponse, + ResolveResponseAdapter, +) from pipelex_sdk.errors import ( ApiResponseError, ApiUnreachableError, @@ -71,6 +79,7 @@ UserProfile, ) from pipelex_sdk.runs import ( + PipelexRunResultStart, PollInfo, RunRead, RunResultCompleted, @@ -87,7 +96,6 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator - from mthds.protocol.models import RunResultStart from mthds.protocol.pipe_output import VariableMultiplicity from mthds.protocol.pipeline_inputs import PipelineInputs from mthds.protocol.stuff import StuffType @@ -139,6 +147,13 @@ # produced validation-error verdict carry `rendered_markdown`; callers may add more tokens. _VALIDATE_MARKDOWN_RENDER_FORMAT = "markdown" +# The PIPELEX API's own run args — the layer-2 extension the RUNNER resolves itself +# (`method_ref`, a run source in its own right). A named parameter here, travelling to the +# base client through its generic `extra` passthrough exactly like the hosted args below. +# Reserved on `extra` for the same reason: a smuggled copy would arrive by a second path +# with different validation and bypass the selector-exclusivity checks. +_PIPELEX_API_RUN_ARGS: frozenset[str] = frozenset({"method_ref"}) + # The HOSTED API's own run args — the layer-3 extensions this client names itself, on top of # the MTHDS Protocol's basic run args. They are named parameters here and travel to the base # client through its generic `extra` passthrough, which is exactly the layering: the protocol @@ -151,6 +166,20 @@ # extension policy: a hosted client types its own platform's arguments and guards them per layer. _HOSTED_RUN_ARGS: frozenset[str] = frozenset({"method_id"}) +# Every request arg this client names itself and therefore guards on `extra` — the union of +# the layer-2 and layer-3 sets above. +_RESERVED_RUN_ARGS: frozenset[str] = _PIPELEX_API_RUN_ARGS | _HOSTED_RUN_ARGS + +# `method_ref` resolution can make the server CLONE a repository before it answers, and the +# server-side clone timeout runs well past the 30s management budget on a cold cache — an +# abort there would report a healthy, still-cloning server as unreachable. So a +# `method_ref`-carrying crate/build request gets this internal fetch-sized budget instead of +# `_POLL_REQUEST_TIMEOUT_SECONDS` (no new caller-facing parameter, and inert behind the +# hosted gateway's own cap). The run routes and `validate` need no such override: they +# already ride `request_timeout_seconds` (the 20-min blocking-execute ceiling), which clears +# any clone. Mirrors the JS SDK's `METHOD_REF_FETCH_TIMEOUT_MS`. +_METHOD_REF_FETCH_TIMEOUT_SECONDS = 180.0 + class MthdsFile(BaseModel): """One MTHDS file submitted to `validate_files` — content plus an optional provenance URI. @@ -276,16 +305,19 @@ async def _send_or_unreachable(self, method: str, url: str, *, content: bytes | msg = f"Could not reach Pipelex API at {self.base_url} ({code})" raise ApiUnreachableError(msg, api_url=self.base_url, code=code) from exc - async def _request_product(self, method: str, endpoint: str, *, body: object | None = None) -> Any: + async def _request_product(self, method: str, endpoint: str, *, body: object | None = None, request_timeout: float | None = None) -> Any: """Issue a Pipelex-product request (`/v1/me`, `/v1/methods`, `/v1/billing/*`, …) and parse its JSON body, mapping a non-2xx response to the typed `ApiResponseError` so callers branch on the structured `code` discriminant, not the HTTP status. Empty-body tolerant — DELETE / onboarding / update routes answer 2xx with no body, - returned as `None`. Uses the management-call timeout, not the blocking ceiling. + returned as `None`. Uses the management-call timeout, not the blocking ceiling; + `request_timeout` overrides it for the crate/build calls whose `method_ref` closure + the server may have to fetch first (see `_METHOD_REF_FETCH_TIMEOUT_SECONDS`). """ content = to_json(body) if body is not None else None - response = await self._send_or_unreachable(method, self._url(endpoint), content=content, request_timeout=_POLL_REQUEST_TIMEOUT_SECONDS) + effective_timeout = request_timeout if request_timeout is not None else _POLL_REQUEST_TIMEOUT_SECONDS + response = await self._send_or_unreachable(method, self._url(endpoint), content=content, request_timeout=effective_timeout) if not 200 <= response.status_code < 300: self._raise_api_response_error(method=method, endpoint=endpoint, response=response) if not response.content: @@ -352,25 +384,31 @@ async def execute( dynamic_output_concept_ref: str | None = None, extra: dict[str, Any] | None = None, *, + method_ref: str | None = None, method_id: str | None = None, ) -> PipelexExecuteResult: """Execute a method synchronously and wait for its completion — `POST /v1/execute`. Returns a `PipelexExecuteResult` — the protocol's raw execute response enriched with a resolved `.main_stuff` accessor, so a blocking result reads its output the same way as a - durable one (`result.main_stuff`) instead of digging through `pipe_output`. - - Identical to the inherited protocol `execute`, except for two things. First, `method_id` - — the hosted platform's own run arg (see below). Second, a failure consistent with the - hosted gateway's ~30s synchronous ceiling — a gateway `503`/`504`, or a client-side - request timeout, after at least ~28s have elapsed — is translated into a clear - `PipelineExecuteTimeoutError` pointing at the durable start+poll path, matching the JS - SDK. The protocol's optional 202 async-degrade still raises `RunStillRunningError` - (from the inherited `execute`), and every other non-2xx keeps the inherited - `httpx.HTTPStatusError` regime (consistent with the other inherited protocol routes). + durable one (`result.main_stuff`) instead of digging through `pipe_output`. A + `method_ref` run's result additionally carries `method_provenance` — the address, the + tag, and the commit SHA that was actually fetched. + + Identical to the inherited protocol `execute`, except for three things. First, + `method_ref` — the Pipelex API's own run source, resolved by the RUNNER (see below). + Second, `method_id` — the hosted platform's own run arg (see below). Third, a failure + consistent with the hosted gateway's ~30s synchronous ceiling — a gateway `503`/`504`, + or a client-side request timeout, after at least ~28s have elapsed — is translated into + a clear `PipelineExecuteTimeoutError` pointing at the durable start+poll path, matching + the JS SDK. The protocol's optional 202 async-degrade still raises + `RunStillRunningError` (from the inherited `execute`), and every other non-2xx keeps + the inherited `httpx.HTTPStatusError` regime (consistent with the other inherited + protocol routes). Args: - pipe_code: The code identifying the pipe to execute. + pipe_code: The code identifying the pipe to execute. Beside a `method_ref` it + overrides the fetched manifest's `main_pipe`. mthds_contents: List of MTHDS bundle contents to load. inputs: Inputs passed to the method. output_name: Name of the output slot to write to. @@ -378,8 +416,17 @@ async def execute( dynamic_output_concept_ref: Override for the dynamic output concept ref. extra: Server-specific extension args this client does not know about, merged into the request body as top-level properties. Protocol args and this client's own - hosted args (`method_id`) must be passed as named parameters, not through - `extra` (raises `PipelineRequestError`). + named args (`method_ref`, `method_id`) must be passed as named parameters, not + through `extra` (raises `PipelineRequestError`). + method_ref: A published method's address — + `github.com//[/][@]` (e.g. + `github.com/Pipelex/methods/documents@v0.1.0`) — a layer-2 Pipelex-API + extension the RUNNER resolves (git fetch at the tag, package located by + manifest identity; pipelex-api >= 0.21.0). A complete run source of its own, + so it pairs with NOTHING: exclusive with inline `mthds_contents` and with + `method_id` (an address run has its own provenance and needs no linkage id) — + the illegal pairings are rejected client-side, mirroring the server's 422s. + `pipe_code` beside it is fine. An empty string is treated as absent. method_id: A stored method's hosted catalog id (`mt_…`) — a pure PASS-THROUGH the platform resolves against the org's catalog; nothing is expanded client-side, and it is meaningless off-platform (an open-source runner answers a `422` @@ -392,12 +439,15 @@ async def execute( Raises: PipelineExecuteTimeoutError: The blocking request hit the hosted gateway's ~30s synchronous ceiling — use `start_and_wait` (or `start` + `wait_for_result`). - PipelineRequestError: `extra` carries a protocol arg or a hosted arg, or `method_id` - is present and is not a string. + PipelineRequestError: `extra` carries a protocol arg or a reserved named arg, a + selector is present and is not a string, or `method_ref` is combined with + inline `mthds_contents` or with `method_id`. RunStillRunningError: The server answered 202 (the protocol's optional async degrade) — the run continues server-side; resume by `pipeline_run_id`. httpx.HTTPStatusError: Any other non-2xx response (the inherited regime). """ + merged_extra = _merge_run_extensions(extra, method_ref=method_ref, method_id=method_id) + _assert_method_ref_pairs_with_nothing(mthds_contents=mthds_contents, merged_extra=merged_extra) started_at = monotonic() try: result = await super().execute( @@ -407,7 +457,7 @@ async def execute( output_name=output_name, output_multiplicity=output_multiplicity, dynamic_output_concept_ref=dynamic_output_concept_ref, - extra=_merge_hosted_run_extensions(extra, method_id), + extra=merged_extra, ) except (httpx.HTTPStatusError, httpx.TimeoutException) as exc: elapsed_seconds = monotonic() - started_at @@ -431,45 +481,63 @@ async def start( dynamic_output_concept_ref: str | None = None, extra: dict[str, Any] | None = None, *, + method_ref: str | None = None, method_id: str | None = None, - ) -> RunResultStart: + ) -> PipelexRunResultStart: """Start a method asynchronously — `POST /v1/start` (202: `pipeline_run_id` only). - Identical to the inherited protocol `start`, except for `method_id` — the hosted - platform's own run arg, documented on `execute` and carrying the same semantics here — - and that a bare-runner missing-route 404 (no run store) is translated into a clear - `RunLifecycleUnavailableError` instead of a raw `httpx.HTTPStatusError`, matching the JS - SDK and letting `start_and_wait` self-heal to the blocking-execute fallback. The + Identical to the inherited protocol `start`, except for the two method selectors — + `method_ref` (the Pipelex API's own run source, resolved by the runner) and + `method_id` (the hosted platform's own run arg), both documented on `execute` and + carrying the same semantics and exclusivity here — and that a bare-runner + missing-route 404 (no run store) is translated into a clear + `RunLifecycleUnavailableError` instead of a raw `httpx.HTTPStatusError`, matching the + JS SDK and letting `start_and_wait` self-heal to the blocking-execute fallback. The platform's structured 404s (run not found) keep their normal `httpx.HTTPStatusError` behavior. + Returns: + The 202 ack as `PipelexRunResultStart` — the authoritative `pipeline_run_id`, + plus `method_provenance` (`{address, tag, commit_sha}`) for a `method_ref` run + (the server fetches the package before the ack, so provenance rides the 202); + `None` otherwise. + Raises: - PipelineRequestError: `extra` carries a protocol arg or a hosted arg, or `method_id` - is present and is not a string. + PipelineRequestError: `extra` carries a protocol arg or a reserved named arg, a + selector is present and is not a string, or `method_ref` is combined with + inline `mthds_contents` or with `method_id`. RunLifecycleUnavailableError: The configured server has no run store. """ + merged_extra = _merge_run_extensions(extra, method_ref=method_ref, method_id=method_id) + _assert_method_ref_pairs_with_nothing(mthds_contents=mthds_contents, merged_extra=merged_extra) try: - return await super().start( + result = await super().start( pipe_code=pipe_code, mthds_contents=mthds_contents, inputs=inputs, output_name=output_name, output_multiplicity=output_multiplicity, dynamic_output_concept_ref=dynamic_output_concept_ref, - extra=_merge_hosted_run_extensions(extra, method_id), + extra=merged_extra, ) except httpx.HTTPStatusError as exc: self._raise_if_lifecycle_unavailable(exc.response, str(exc.request.url)) raise + # Re-validate the base ack into the Pipelex-branded subtype (types `method_provenance`; + # any other implementation extra keeps riding `model_extra`). + return PipelexRunResultStart.model_validate(result.model_dump()) @override async def validate( # type: ignore[override] self, - mthds_contents: list[str], + mthds_contents: list[str] | None = None, allow_signatures: bool = False, mthds_sources: list[str] | None = None, render: list[str] | None = None, views: list[str] | None = None, + *, + method_ref: str | None = None, + method_id: str | None = None, ) -> PipelexValidationResult: """Parse, validate, and dry-run an MTHDS bundle — `POST /v1/validate`. @@ -477,19 +545,39 @@ async def validate( # type: ignore[override] body discriminated on `is_valid`, returned verbatim as the `PipelexValidationResult` union (an invalid bundle is NOT raised; the caller match/cases `is_valid`). A non-2xx means no verdict could be produced (request shape, auth, server fault) and surfaces as - `httpx.HTTPStatusError` (the inherited protocol error regime). + `httpx.HTTPStatusError` (the inherited protocol error regime). A selector-resolution + failure — a fetch failure, no package at the address, an unknown or foreign-org id — + is a non-2xx too, never an `is_valid: false` verdict, which is reserved for actual + MTHDS content. + + WHAT is validated arrives in exactly one of three forms — the tooling routes' strict + three-way XOR (the routes are stateless, so there is no linkage exception; a second + selector is rejected client-side, mirroring the server's request-shape `422`): + + - **inline `mthds_contents`** — the protocol's own envelope; + - **`method_ref`** — a published method's address, resolved by the server + (pipelex-api >= 0.21.0) through the same fetch path as a `method_ref` run, the + package's real file names feeding the diagnostics' source labels; + - **`method_id`** — a stored method's catalog id, hosted-only: the platform resolves + it and injects the stored source before the runner sees the request (a bare runner + rejects the request as carrying no source it understands). This override differs from the inherited protocol `validate` in these Pipelex-API ways: it always injects `render: ["markdown"]` (so both valid and invalid verdicts carry - `rendered_markdown`), it accepts `mthds_sources` as a named parameter, and it carries - the `views` opt-in for the server's structured views. + `rendered_markdown`), it accepts `mthds_sources` as a named parameter, it carries the + `views` opt-in for the server's structured views, and it takes the two method + selectors. Args: mthds_contents: MTHDS contents to load (always a list, even for one file). + Exactly one of `mthds_contents` / `method_ref` / `method_id`. allow_signatures: Tolerate unimplemented pipe signatures (strict by default). mthds_sources: Optional per-content source names, parallel to `mthds_contents`, threaded onto each diagnostic's `source` (an unnamed content yields - `source: null`). The server 422s a length mismatch. + `source: null`). The server 422s a length mismatch. An inline-contents + companion only: a `method_ref` / `method_id` validation gets its source labels + from the package's (or the stored method's) real file names, so supplying it + beside a selector is rejected client-side. render: Optional Pipelex-API presentation hints; `"markdown"` is always added. Unknown tokens are server-side lenient-ignored (never a 422). views: Optional opt-in for the server's structured views. `input_form` — named by @@ -498,6 +586,12 @@ async def validate( # type: ignore[override] **verbatim**: nothing is injected and nothing is de-duplicated, and an explicit `[]` is sent as `[]`. Left at `None` the key is not sent at all, which is what keeps the default response byte-identical for consumers that discard views. + method_ref: A published method's address — + `github.com//[/][@]` — runner-resolved. An empty + string is treated as absent. + method_id: A stored method's hosted catalog id (`mt_…`), platform-resolved. An + unknown or foreign-org id is a `404` (indistinguishable by design); a stored + method with no MTHDS source is a `422`. An empty string is treated as absent. Returns: The 200-diagnostic union: `PipelexValidationReport` (`is_valid: true`) or @@ -507,16 +601,48 @@ async def validate( # type: ignore[override] `pipe_io_contracts` are typed by the standard's own models (`mthds.protocol`), so a field descriptor narrows on its `kind` and a slot's presence and multiplicity read as enums; import the per-kind types from `mthds.protocol.input_form`. + + Raises: + PipelineRequestError: Zero or several selectors were supplied, a selector is not a + string, or `mthds_sources` was supplied beside a selector. """ - extra: dict[str, Any] = {"render": _with_validate_markdown_render(render)} - if mthds_sources is not None: - extra["mthds_sources"] = mthds_sources + selected_method_ref = _normalized_selector(name="method_ref", value=method_ref) + selected_method_id = _normalized_selector(name="method_id", value=method_id) + selector_count = sum(1 for present in (bool(mthds_contents), selected_method_ref is not None, selected_method_id is not None) if present) + if selector_count != 1: + msg = "validate() takes exactly one method selector: inline mthds_contents, method_ref, or method_id." + raise PipelineRequestError(msg) + if mthds_sources is not None and not mthds_contents: + msg = ( + "mthds_sources labels inline mthds_contents; a method_ref / method_id validation gets " + "its source labels from the package's (or the stored method's) real file names." + ) + raise PipelineRequestError(msg) + + if mthds_contents: + extra: dict[str, Any] = {"render": _with_validate_markdown_render(render)} + if mthds_sources is not None: + extra["mthds_sources"] = mthds_sources + if views is not None: + extra["views"] = views + # Reuse the inherited transport seam (`_post_validate`) for body-building + the wire + # call, then parse the 200-diagnostic body into this SDK's Pipelex-branded narrowing. + # The base's own `validate` parses the same body into the neutral `ValidationResult`. + response = await self._post_validate(mthds_contents, allow_signatures, extra) + return PipelexValidationResultAdapter.validate_python(response.json()) + + # A selector validation must NOT carry the `mthds_contents` key at all (the server XORs + # on presence, and an empty list is a request-shape 422), so the body is built here + # rather than through `_post_validate`, on the same inherited `_send` transport seam. + body: dict[str, Any] = {"allow_signatures": allow_signatures, "render": _with_validate_markdown_render(render)} if views is not None: - extra["views"] = views - # Reuse the inherited transport seam (`_post_validate`) for body-building + the wire call, - # then parse the 200-diagnostic body into this SDK's Pipelex-branded narrowing. The base's - # own `validate` parses the same body into the neutral `mthds` `ValidationResult`. - response = await self._post_validate(mthds_contents, allow_signatures, extra) + body["views"] = views + if selected_method_ref is not None: + body["method_ref"] = selected_method_ref + if selected_method_id is not None: + body["method_id"] = selected_method_id + response = await self._send("POST", self._url("validate"), content=to_json(body), request_timeout=self.request_timeout_seconds) + response.raise_for_status() return PipelexValidationResultAdapter.validate_python(response.json()) async def validate_files( @@ -700,6 +826,7 @@ async def start_and_wait( extra: dict[str, Any] | None = None, wait_options: WaitForResultOptions | None = None, *, + method_ref: str | None = None, method_id: str | None = None, ) -> RunResults: """Start a run and wait for its result — the whole lifecycle in one call, self-healing @@ -715,8 +842,9 @@ async def start_and_wait( from `start`, BEFORE any run is created, so the blocking fallback cannot double-run; the negative is cached so later calls skip the durable attempt. - `method_id` — the hosted platform's own run arg, documented on `execute` — is forwarded - on BOTH paths. Dropping it on the blocking fallback would turn a server-side 422 that + The method selectors — `method_ref` and `method_id`, documented on `execute` — are + forwarded on BOTH paths: a `method_ref` run must run the same fetched package on the + blocking fallback, and dropping `method_id` there would turn a server-side 422 that names the key into a silently different run. Raises: @@ -733,6 +861,7 @@ async def start_and_wait( output_multiplicity=output_multiplicity, dynamic_output_concept_ref=dynamic_output_concept_ref, extra=extra, + method_ref=method_ref, method_id=method_id, ) except RunLifecycleUnavailableError: @@ -745,6 +874,7 @@ async def start_and_wait( output_multiplicity=output_multiplicity, dynamic_output_concept_ref=dynamic_output_concept_ref, extra=extra, + method_ref=method_ref, method_id=method_id, ) return await self.wait_for_result(started.pipeline_run_id, options=wait_options) @@ -757,6 +887,7 @@ async def start_and_wait( output_multiplicity=output_multiplicity, dynamic_output_concept_ref=dynamic_output_concept_ref, extra=extra, + method_ref=method_ref, method_id=method_id, ) @@ -770,15 +901,17 @@ async def _execute_blocking( output_multiplicity: VariableMultiplicity | None, dynamic_output_concept_ref: str | None, extra: dict[str, Any] | None, + method_ref: str | None = None, method_id: str | None = None, ) -> RunResults: """Blocking `POST /v1/execute` adapted onto `RunResults` — the bare-runner path. - Forwards every protocol field PLUS both extension surfaces: the hosted `method_id` and - the generic `extra` passthrough. An extension-only call (`{extra}` with no pipe_code) or - a vendor selector riding `extra` must survive this path, not just the durable one — and - a hosted `method_id` must reach the server here too, so a runner that cannot resolve it - says so instead of the client silently dropping it. + Forwards every protocol field PLUS every extension surface: the runner-resolved + `method_ref`, the hosted `method_id`, and the generic `extra` passthrough. An + extension-only call (`{extra}` with no pipe_code) or a vendor selector riding `extra` + must survive this path, not just the durable one — a `method_ref` run must run the + same fetched package here, and a hosted `method_id` must reach the server too, so a + runner that cannot resolve it says so instead of the client silently dropping it. """ result = await self.execute( pipe_code=pipe_code, @@ -788,6 +921,7 @@ async def _execute_blocking( output_multiplicity=output_multiplicity, dynamic_output_concept_ref=dynamic_output_concept_ref, extra=extra, + method_ref=method_ref, method_id=method_id, ) return _map_run_result_to_run_results(result) @@ -986,15 +1120,71 @@ async def upload(self, upload_input: UploadInput) -> UploadedFile: async def build_inputs(self, request: BuildInputsRequest) -> BuildInputsResponse: """Project a pipe's declared inputs as a fill-in template — `POST /v1/build/inputs`. + The closure is inline `files` XOR a `method_ref` address (server-resolved, + pipelex-api >= 0.21.0; the registry form stays a `501`). There is NO by-id form: the + `/v1/build/*` projections take no `method_id` (the hosted tooling selector covers + `validate`/`resolve`/`codegen` only) — expand a stored method's source into `files` + yourself (fetch it with `get_method`). + Returns a 200 verdict: branch on `is_valid` before reading the arm — an unresolvable closure comes back as `is_valid: false` with `validation_errors`, not a thrown error. - A no-verdict condition (unknown `pipe_ref`, auth, server fault) raises `ApiResponseError`. - This is the signature source `prepare_inputs` reads (with `explicit=True`). + A no-verdict condition (unknown `pipe_ref`, a selector-resolution failure, auth, server + fault) raises `ApiResponseError`. This is the signature source `prepare_inputs` reads + (with `explicit=True`). """ body = request.model_dump(mode="json", exclude_none=True) - raw = await self._request_product("POST", "build/inputs", body=body) + raw = await self._request_product("POST", "build/inputs", body=body, request_timeout=_crate_request_timeout_seconds(request.method_ref)) return BuildInputsResponseAdapter.validate_python(raw) + # ── Crate extensions (Pipelex API — `/v1/resolve`, `/v1/codegen`) ───── + # + # The second crate-family surface, mirroring the JS SDK: `/v1/resolve` emits the + # normalized library crate, `/v1/codegen` projects that crate into stamped typed + # artifacts plus their lock. Same envelope family and same 200-verdict discipline as + # the build routes, PLUS the hosted `method_id` selector under the tooling routes' + # strict three-way XOR (see `crate_models`). + + async def resolve(self, request: ResolveRequest) -> ResolveResponse: + """Resolve a closure into its normalized library crate — `POST /v1/resolve`. + + The closure is loaded and statically validated, then emitted as the normalized + library crate (fully qualified refs, refinement flattened, natives materialized, + fingerprint set) — the MTHDS standard's Library Crate Format. It runs NO dry-run + sweep, so a valid verdict here says the library resolves, never that it runs; that + is `validate`'s vocabulary. + + The closure arrives in exactly one of three forms — inline `files`, an address-form + `method_ref` (server-resolved; registry form `501`), or a hosted `method_id` + (platform-resolved) — enforced at request construction and by the server alike. + + Returns a 200 verdict: branch on `is_valid` before reading the arm. A no-verdict + condition (a malformed selector, a selector-resolution failure — fetch failure, no + package at the address, an unknown or foreign-org id — auth, a server fault) raises + `ApiResponseError`, never an `is_valid: false` verdict. + """ + body = request.model_dump(mode="json", exclude_none=True) + raw = await self._request_product("POST", "resolve", body=body, request_timeout=_crate_request_timeout_seconds(request.method_ref)) + return ResolveResponseAdapter.validate_python(raw) + + async def codegen(self, request: CodegenRequest) -> CodegenResponse: + """Project a closure's crate into stamped typed artifacts — `POST /v1/codegen`. + + Resolves the closure exactly like `resolve`, then projects the crate through the two + explicit axes — `kind` (`types` today) x `target` (`python-pydantic` for Python + consumers, `python-structures`, `ts-zod`) — and returns the artifact set plus its + `codegen.lock`. Write both verbatim and the tree is byte-identical to a local + `pipelex codegen types` run, so the offline `pipelex codegen check` passes on it; + the SDK deliberately does not write files for you. + + Same 200-verdict discipline and same three-form closure selector as `resolve`. A + no-verdict condition (an unknown `kind`/`target`, a `pipe_ref` on the + concept-set-wide `types` kind, a malformed selector, a selector-resolution failure) + raises `ApiResponseError`; a registry-form `method_ref` is a `501`. + """ + body = request.model_dump(mode="json", exclude_none=True) + raw = await self._request_product("POST", "codegen", body=body, request_timeout=_crate_request_timeout_seconds(request.method_ref)) + return CodegenResponseAdapter.validate_python(raw) + async def upload_file( self, source: UploadSource, @@ -1134,52 +1324,109 @@ async def health(self) -> dict[str, Any]: _KNOWN_RUN_STATUS_NAMES: frozenset[str] = frozenset(RunStatus.__members__) -def _merge_hosted_run_extensions(extra: dict[str, Any] | None, method_id: object) -> dict[str, Any] | None: - """Fold the hosted API's own run args into the generic `extra` passthrough handed to the base client. +def _normalized_selector(*, name: str, value: object) -> str | None: + """Normalize one method-selector argument at the client boundary. - This is the seam between layer 3 and layer 2: `method_id` is a named parameter on this - client (it is the hosted platform's argument, so this client must type it), and it reaches - the wire as a top-level body property through the protocol client's extension mechanism — - which merges it without knowing what it means. See `_HOSTED_RUN_ARGS`. + A **non-string** value is refused rather than dropped or forwarded. A published client + validates its request-option types at its own boundary, so that one wrong value gets one + answer: a bare truthiness check would silently drop the falsy wrong types (`0`, `[]`) and + forward the truthy ones (`123`, `["mt_1"]`) to a server `422` — a different partition of + wrong values than the JS client makes for the same argument on the same wire. Typed + `object` rather than `str | None` deliberately — this helper *is* the runtime boundary, + and the callers it guards against are the untyped ones a type checker never sees. - A **non-string** `method_id` is refused here rather than dropped or forwarded. A published - client validates its request-option types at its own boundary, so that one wrong value gets - one answer: a bare truthiness check would silently drop the falsy wrong types (`0`, `[]`) - and forward the truthy ones (`123`, `["mt_1"]`) to a server `422` — a different partition of - wrong values than the JS client makes for the same argument on the same wire. + An absent or **empty** value normalizes to `None`: an empty selector selects nothing, so + it is not sent and does not satisfy the base client's "something to run" precondition. - An absent or empty `method_id` still contributes nothing: `method_id=""` selects no method - and links no run, so it is not sent and does not satisfy the base client's "something to - run" precondition, and neither does `None`. `None` is returned for an empty result, leaving - the base's own handling of an absent `extra` untouched. + Raises: + PipelineRequestError: If the value is present and is not a string. + """ + if value is None: + return None + if not isinstance(value, str): + msg = f"{name} must be a string, received {type(value).__name__}." + raise PipelineRequestError(msg) + return value or None + + +def _merge_run_extensions(extra: dict[str, Any] | None, *, method_ref: object, method_id: object) -> dict[str, Any] | None: + """Fold this client's own named run args into the generic `extra` passthrough handed to + the base client. + + This is the layering seam: `method_ref` (layer 2 — the Pipelex API's run source, resolved + by the runner) and `method_id` (layer 3 — the hosted platform's run arg) are named + parameters on this client because it is the client that types its own stack's arguments, + and each reaches the wire as a top-level body property through the protocol client's + extension mechanism — which merges it without knowing what it means. See + `_PIPELEX_API_RUN_ARGS` / `_HOSTED_RUN_ARGS`. + + Both selectors go through `_normalized_selector`: a non-string is refused, an absent or + empty value contributes nothing. `None` is returned for an empty result, leaving the + base's own handling of an absent `extra` untouched. Args: extra: Server-specific extension args from the caller, or None. - method_id: The hosted catalog id, or None. Typed `object` rather than `str | None` - deliberately — this helper *is* the runtime boundary, and the callers it guards - against are the untyped ones a type checker never sees. + method_ref: The published method's address, or None. + method_id: The hosted catalog id, or None. Returns: The merged extension mapping to hand to the base client, or None if there is nothing. Raises: - PipelineRequestError: If `extra` carries a hosted arg this client names itself, or if - `method_id` is present and is not a string. + PipelineRequestError: If `extra` carries a named arg this client reserves, or if a + selector is present and is not a string. """ extensions: dict[str, Any] = dict(extra or {}) - hosted_overlap = extensions.keys() & _HOSTED_RUN_ARGS - if hosted_overlap: - msg = f"extra carries hosted args {sorted(hosted_overlap)} — pass them as named parameters instead." + reserved_overlap = extensions.keys() & _RESERVED_RUN_ARGS + if reserved_overlap: + msg = f"extra carries reserved request args {sorted(reserved_overlap)} — pass them as named parameters instead." raise PipelineRequestError(msg) - if method_id is not None: - if not isinstance(method_id, str): - msg = f"method_id must be a string, received {type(method_id).__name__}." - raise PipelineRequestError(msg) - if method_id: - extensions["method_id"] = method_id + selected_method_ref = _normalized_selector(name="method_ref", value=method_ref) + if selected_method_ref is not None: + extensions["method_ref"] = selected_method_ref + selected_method_id = _normalized_selector(name="method_id", value=method_id) + if selected_method_id is not None: + extensions["method_id"] = selected_method_id return extensions or None +def _assert_method_ref_pairs_with_nothing(*, mthds_contents: list[str] | None, merged_extra: dict[str, Any] | None) -> None: + """Enforce the run routes' `method_ref` exclusivity, mirroring the server's own 422s so an + illegal pairing fails before anything hits the wire. + + A `method_ref` is a complete run source (the fetched package carries its `.mthds` and its + entry pipe), so it pairs with NOTHING: not with inline `mthds_contents` and not with the + hosted `method_id` — an address run has its own provenance and needs no linkage id. Reads + the MERGED extensions, so the presence semantics are the normalized ones (an empty selector + was already dropped). + + The one documented run-route exception is deliberately NOT here: inline source + + `method_id` stays legal (the inline source runs; the id demotes to run-history linkage). + `pipe_code` beside a `method_ref` is legal too — it overrides the manifest's `main_pipe`. + This SDK names no bundle encodings (`files` / `bundle_b64`), so their arm of the server's + exclusivity has no client-side twin here; the server still enforces it. + """ + if merged_extra is None or "method_ref" not in merged_extra: + return + if mthds_contents: + msg = "method_ref and inline mthds_contents are mutually exclusive; send one or the other." + raise PipelineRequestError(msg) + if "method_id" in merged_extra: + msg = ( + "method_ref and method_id are mutually exclusive: an address run carries its own provenance " + "and takes no run-history linkage id. Send exactly one method selector." + ) + raise PipelineRequestError(msg) + + +def _crate_request_timeout_seconds(method_ref: str | None) -> float: + """The request budget for a call carrying a crate closure (the crate routes and the build + projections alike): the management default, unless the closure is a `method_ref` the + server may have to fetch first — see `_METHOD_REF_FETCH_TIMEOUT_SECONDS`. + """ + return _METHOD_REF_FETCH_TIMEOUT_SECONDS if method_ref else _POLL_REQUEST_TIMEOUT_SECONDS + + def _with_validate_markdown_render(render: list[str] | None) -> list[str]: """Ensure `"markdown"` rides the `/validate` render list, preserving order and de-duplicating. diff --git a/pipelex_sdk/crate_models.py b/pipelex_sdk/crate_models.py new file mode 100644 index 0000000..6ab3a9e --- /dev/null +++ b/pipelex_sdk/crate_models.py @@ -0,0 +1,156 @@ +"""Wire models for the crate routes — `POST /v1/resolve` and `POST /v1/codegen`. + +The second crate-family surface, mirroring `pipelex-sdk-js`: `/v1/resolve` emits the +normalized library crate, `/v1/codegen` projects that crate into stamped typed artifacts +plus their lock. Both are Pipelex API extensions (NOT MTHDS Protocol routes) over the +standard-owned artifact, so their wire fields stay brand-neutral. Same envelope and same +verdict discipline as the build routes: a produced verdict is a `200` discriminated on +`is_valid`, with `CrateInvalidReport` (from `build_models`) as the shared invalid arm; a +no-verdict condition (a malformed selector, a selector-resolution failure, auth, a server +fault) raises `ApiResponseError`. + +The closure arrives in exactly one of three forms — the tooling routes' strict three-way +XOR: inline `files`, an address-form `method_ref` (server-resolved, pipelex-api >= 0.21.0; +the registry form stays a `501`), or a hosted `method_id` (platform-resolved — meaningless +against a bare runner, which has no catalog). The routes are stateless, so there is no +linkage exception: a second selector is a request-shape `422`, mirrored client-side by the +construction-time validator here. +""" + +from __future__ import annotations + +from typing import Annotated, Any, Literal, Self, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator + +from pipelex_sdk.build_models import CrateInvalidReport, CrateRequestBase + + +class CrateToolingRequest(CrateRequestBase): + """The crate envelope plus the hosted tooling selector — the request base + `/v1/resolve` and `/v1/codegen` share. + + `method_id` is a stored method's catalog id (`mt_…`), a **pass-through to the hosted + API**: the platform resolves it against the org's catalog and injects the stored + source before the runner sees the request — nothing is expanded client-side, and it + is meaningless off-platform. An unknown or foreign-org id is a `404` + (indistinguishable by design); a stored method with no MTHDS source is a `422`. + """ + + method_id: str | None = None + + @model_validator(mode="after") + def _exactly_one_selector(self) -> Self: + # The strict tooling XOR, enforced at construction so an illegal shape fails + # before anything hits the wire (the server 422s the same shapes). + selector_count = sum(1 for selector in (self.files, self.method_ref, self.method_id) if selector is not None) + if selector_count != 1: + msg = "provide exactly one of `files`, `method_ref`, or `method_id`" + raise ValueError(msg) + return self + + +class ResolveRequest(CrateToolingRequest): + """Request for `POST /v1/resolve` — the crate envelope (no projection axes) plus the + hosted `method_id` selector. Exactly one of `files` / `method_ref` / `method_id`. + """ + + +class ResolveValidReport(BaseModel): + """The `/v1/resolve` valid arm — the normalized library crate. + + `crate` is the MTHDS **Library Crate Format**: fully qualified refs, refinement + flattened, natives materialized, top-level maps key-sorted. Its `fingerprint` and + `mthds_version` ride INSIDE the payload, not beside it. Typed as opaque transport + (`dict[str, Any]`): the crate schema is owned by the MTHDS standard, not by this SDK, + and restating it here would be a second source of truth free to drift. Do not + recompute the fingerprint by hashing this object — it is a property of the logical + crate, not of any particular serialization; compare `fingerprint` values only. + """ + + model_config = ConfigDict(extra="allow") + + is_valid: Literal[True] + crate: dict[str, Any] + message: str + + +ResolveResponse: TypeAlias = Annotated[ + ResolveValidReport | CrateInvalidReport, + Field(discriminator="is_valid"), +] + +# The single parse path for a 200 `/resolve` body — discriminated on `is_valid`, built once +# at import (TypeAdapter construction is expensive), mirroring `BuildInputsResponseAdapter`. +ResolveResponseAdapter: TypeAdapter[ResolveResponse] = TypeAdapter(ResolveResponse) # pylint: disable=invalid-name + + +CodegenKind = Literal["types"] +"""What `/v1/codegen` projects — the `kind` axis. `types` (the crate's whole concept set +projected into typed models) is the only kind served today.""" + +CodegenTarget = Literal["ts-zod", "python-pydantic", "python-structures"] +"""For whom `/v1/codegen` projects — the `target` axis, mirroring pipelex's `CodegenTarget`. +`python-pydantic` emits self-contained BaseModels (the natural target for Python consumers); +`python-structures` emits runtime StructuredContent classes for a Pipelex host; `ts-zod` +emits zod schemas plus inferred types.""" + + +class CodegenRequest(CrateToolingRequest): + """Request for `POST /v1/codegen` — the crate envelope plus the two explicit + projection axes and the hosted `method_id` selector (exactly one of `files` / + `method_ref` / `method_id`). + + `pipe_ref` exists for the future per-pipe projection kinds; the concept-set-wide + `types` kind REJECTS it with a request-shape `422` rather than silently ignoring it. + """ + + kind: CodegenKind = "types" + target: CodegenTarget + pipe_ref: str | None = None + + +class GeneratedArtifact(BaseModel): + """One stamped generated file. `path` is relative to the output root the caller + chooses; `content` is complete, stamp header included, and is written verbatim. + """ + + path: str + content: str + + +class CodegenValidReport(BaseModel): + """The `/v1/codegen` valid arm — the stamped artifact set plus its lock. + + The trust chain: write every `artifacts` entry at its `path` and the `lock` content + as `lock_filename`, both verbatim, and the tree is byte-identical to what a local + `pipelex codegen types` run produces — same stamps, same lock — so the offline + `pipelex codegen check` passes on it. Editing an artifact (or re-serializing the + lock) breaks that chain. + """ + + model_config = ConfigDict(extra="allow") + + is_valid: Literal[True] + #: Echo of the request's projection axes. + kind: CodegenKind + target: CodegenTarget + #: Fingerprint of the normalized crate the artifacts were generated from. + crate_fingerprint: str + #: The pipelex engine version that generated them. + engine_version: str + artifacts: list[GeneratedArtifact] + #: The lock file's TOML content — write verbatim beside the artifacts. + lock: str + #: The filename `lock` must be written as (`codegen.lock`). + lock_filename: str + message: str + + +CodegenResponse: TypeAlias = Annotated[ + CodegenValidReport | CrateInvalidReport, + Field(discriminator="is_valid"), +] + +# The single parse path for a 200 `/codegen` body — same regime as `ResolveResponseAdapter`. +CodegenResponseAdapter: TypeAdapter[CodegenResponse] = TypeAdapter(CodegenResponse) # pylint: disable=invalid-name diff --git a/pipelex_sdk/execute_result.py b/pipelex_sdk/execute_result.py index a053c4e..12db661 100644 --- a/pipelex_sdk/execute_result.py +++ b/pipelex_sdk/execute_result.py @@ -11,6 +11,7 @@ from mthds.runners.api.models import DictRunResultExecute from pipelex_sdk.errors import MissingMainStuffError +from pipelex_sdk.runs import MethodProvenance class PipelexExecuteResult(DictRunResultExecute): @@ -30,6 +31,12 @@ class PipelexExecuteResult(DictRunResultExecute): #: `.main_stuff` raises `MissingMainStuffError`. main_stuff_name: str | None = None + #: Provenance of a `method_ref` run — the resolved address, the requested tag, and the + #: commit SHA that was actually fetched (a Pipelex-API extension, pipelex-api >= 0.21.0). + #: `None` for inline-source and `method_id` runs, mirroring `PipelexRunResultStart` on + #: the durable path. + method_provenance: MethodProvenance | None = None + @property def main_stuff(self) -> Any: """The resolved main output content, dug out of the working memory via `main_stuff_name`. diff --git a/pipelex_sdk/runs.py b/pipelex_sdk/runs.py index ffba5f3..aca84f7 100644 --- a/pipelex_sdk/runs.py +++ b/pipelex_sdk/runs.py @@ -36,6 +36,7 @@ from enum import StrEnum from typing import TYPE_CHECKING, Annotated, Any, Literal, TypeAlias +from mthds.protocol.models import RunResultStart from mthds.runners.api.models import DictPipeOutputAbstract from pydantic import BaseModel, ConfigDict, Field @@ -92,6 +93,35 @@ def is_success(self) -> bool: # ── Responses ─────────────────────────────────────────────────────── +class MethodProvenance(BaseModel): + """Provenance of a `method_ref` run — a Pipelex-API extension on the run acks. + + The package's resolved full address, the requested tag (`None` for a bare address, + which resolves the default branch at HEAD), and the commit SHA that was actually + fetched — the SHA is what keeps the run explainable when a tag moves. Attached to + the `POST /v1/start` 202 ack (`PipelexRunResultStart.method_provenance`) and the + blocking execute response (`PipelexExecuteResult.method_provenance`) for + `method_ref` runs, absent (or `None`) otherwise. Extension-open (`extra="allow"`) + like every wire model here, so a future server field is preserved. + """ + + model_config = ConfigDict(extra="allow") + + address: str + tag: str | None = None + commit_sha: str + + +class PipelexRunResultStart(RunResultStart): + """The `POST /v1/start` 202 ack as the Pipelex API returns it — the protocol's + `RunResultStart` plus the server's `method_provenance` extension, populated for + `method_ref` runs and absent (`None`) otherwise. The base is extension-open, so + any other implementation field still rides `model_extra`. + """ + + method_provenance: MethodProvenance | None = None + + class RunPublic(BaseModel): """A run record — the BASE shape of the run-lifecycle read surface. diff --git a/pyproject.toml b/pyproject.toml index 87bb981..23f7b6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pipelex-sdk" -version = "0.7.0" +version = "0.8.0" description = "The Python client for the Pipelex hosted API — the MTHDS Protocol surface plus the durable run lifecycle and the Pipelex product surface, built on the `mthds` protocol base." authors = [{ name = "Evotis S.A.S.", email = "oss@pipelex.com" }] maintainers = [{ name = "Pipelex staff", email = "oss@pipelex.com" }] @@ -325,6 +325,8 @@ runtime-evaluated-base-classes = [ "mthds.protocol.models.ValidationReport", "mthds.protocol.models.InvalidValidationReport", "mthds.protocol.models.ValidationDiagnostic", + "mthds.protocol.models.RunResultStart", + "mthds.runners.api.models.DictRunResultExecute", ] [tool.ruff.lint.pydocstyle] diff --git a/tests/unit/test_build_inputs.py b/tests/unit/test_build_inputs.py index 3e07a78..d3d8480 100644 --- a/tests/unit/test_build_inputs.py +++ b/tests/unit/test_build_inputs.py @@ -105,3 +105,55 @@ def test_no_verdict_422_raises_api_response_error(self, mocker: MockerFixture) - with pytest.raises(ApiResponseError) as exc_info: asyncio.run(client.build_inputs(BuildInputsRequest(files=[MthdsFileItem(content="x")], pipe_ref="demo.nope"))) assert exc_info.value.status == 422 + + # ── the closure selector (files XOR method_ref, NO method_id) ──── + + def test_method_ref_closure_rides_the_body_with_the_fetch_budget(self, mocker: MockerFixture) -> None: + """An address closure may make the server clone before answering, so the request gets + the fetch-sized budget instead of the 30s management one. + """ + client = self._client() + valid: dict[str, object] = { + "is_valid": True, + "pipe_ref": "documents.summarize", + "message": "ok", + "format": "json", + "explicit": False, + "inputs": {}, + } + send = self._mock_send(mocker, client, _response(200, json_body=valid)) + + report = asyncio.run(client.build_inputs(BuildInputsRequest(method_ref="github.com/Pipelex/methods/documents@v0.1.0"))) + + call = send.call_args + body = json.loads(call.kwargs["content"]) + assert body == {"method_ref": "github.com/Pipelex/methods/documents@v0.1.0", "format": "json", "explicit": False} + assert call.kwargs["request_timeout"] == 180.0 + assert isinstance(report, BuildInputsValidReport) + + def test_inline_files_keep_the_management_budget(self, mocker: MockerFixture) -> None: + client = self._client() + valid: dict[str, object] = {"is_valid": True, "pipe_ref": "demo.main", "message": "ok", "format": "json", "explicit": False, "inputs": {}} + send = self._mock_send(mocker, client, _response(200, json_body=valid)) + + asyncio.run(client.build_inputs(BuildInputsRequest(files=[MthdsFileItem(content="x")]))) + + assert send.call_args.kwargs["request_timeout"] == 30.0 + + @pytest.mark.parametrize( + "kwargs", + [ + {}, + {"files": [MthdsFileItem(content="x")], "method_ref": "github.com/x/y@v1"}, + ], + ) + def test_request_construction_enforces_files_xor_method_ref(self, kwargs: dict[str, object]) -> None: + with pytest.raises(ValidationError, match="exactly one"): + BuildInputsRequest.model_validate(kwargs) + + def test_method_id_is_refused_with_a_teaching_error(self) -> None: + """The `/v1/build/*` projections take no `method_id` — a teaching error beats pydantic + silently ignoring the unknown key for a caller migrating off the by-id habit. + """ + with pytest.raises(ValidationError, match="build_inputs takes no method_id"): + BuildInputsRequest.model_validate({"method_id": "mt_1"}) diff --git a/tests/unit/test_client_method_ref.py b/tests/unit/test_client_method_ref.py new file mode 100644 index 0000000..f4a9559 --- /dev/null +++ b/tests/unit/test_client_method_ref.py @@ -0,0 +1,176 @@ +"""Tests for the `method_ref` run option — the layer-2 Pipelex-API run source the runner resolves. + +Mirrors `pipelex-sdk-js/tests/client.test.ts` "method_ref run source". The doctrine the +assertions pin: a `method_ref` is a complete run source, so it pairs with nothing (the +client-side guards mirror the server's 422s), provenance comes back typed on both run paths, +and the selector survives the blocking-execute fallback. +""" + +import asyncio + +import httpx +import pytest +from mthds.protocol.exceptions import PipelineRequestError +from pytest_mock import MockerFixture + +from pipelex_sdk.client import PipelexAPIClient +from pipelex_sdk.runs import PipelexRunResultStart + +_BASE_URL = "http://localhost:8081" +_METHOD_REF = "github.com/Pipelex/methods/documents@v0.1.0" + +_BARE_VERSION = {"protocol_version": "0.6.0", "implementation": "pipelex-api", "runner_version": "1.2.3"} +_PROVENANCE = {"address": "github.com/Pipelex/methods/documents", "tag": "v0.1.0", "commit_sha": "23dda75deadbeef"} +_START_BODY = {"pipeline_run_id": "run_1", "state": "RUNNING", "created_at": "2026-08-29T00:00:00Z", "method_provenance": _PROVENANCE} +_EXECUTE_BODY: dict[str, object] = { + "pipeline_run_id": "run-x", + "main_stuff_name": "result", + "method_provenance": _PROVENANCE, + "pipe_output": { + "working_memory": { + "root": {"result": {"concept": "native.Text", "content": {"text": "hi"}}}, + "aliases": {"main_stuff": "result"}, + }, + "pipeline_run_id": "run-x", + }, +} + + +def _response(status_code: int, *, json: object | None = None) -> httpx.Response: + request = httpx.Request("POST", f"{_BASE_URL}/v1/start") + if json is None: + return httpx.Response(status_code, request=request) + return httpx.Response(status_code, json=json, request=request) + + +class TestMethodRefRunOption: + def _client(self) -> PipelexAPIClient: + return PipelexAPIClient(api_key="test-token", base_url=_BASE_URL) + + def test_method_ref_rides_the_body_and_provenance_comes_back_typed(self, mocker: MockerFixture) -> None: + """The typed option reaches the wire as a top-level field, and the 202 ack narrows to + `PipelexRunResultStart` with the `{address, tag, commit_sha}` provenance typed. + """ + client = self._client() + send = mocker.patch.object(client, "_send", mocker.AsyncMock(return_value=_response(202, json=_START_BODY))) + + started = asyncio.run(client.start(method_ref=_METHOD_REF)) + + sent = send.call_args.kwargs["content"].decode("utf-8") + assert f'"method_ref":"{_METHOD_REF}"' in sent + assert '"extra"' not in sent + assert isinstance(started, PipelexRunResultStart) + assert started.method_provenance is not None + assert started.method_provenance.address == "github.com/Pipelex/methods/documents" + assert started.method_provenance.tag == "v0.1.0" + assert started.method_provenance.commit_sha == "23dda75deadbeef" + + def test_start_without_provenance_types_none(self, mocker: MockerFixture) -> None: + """An inline-source ack has no provenance; the typed field is honestly `None`.""" + client = self._client() + mocker.patch.object(client, "_send", mocker.AsyncMock(return_value=_response(202, json={"pipeline_run_id": "run_2"}))) + + started = asyncio.run(client.start(pipe_code="answer")) + + assert started.method_provenance is None + + def test_execute_surfaces_provenance_on_the_blocking_path(self, mocker: MockerFixture) -> None: + """`PipelexExecuteResult` declares `method_provenance` too, so the blocking path reads + it the same way as the durable ack. + """ + client = self._client() + mocker.patch.object(client, "_send", mocker.AsyncMock(return_value=_response(200, json=_EXECUTE_BODY))) + + result = asyncio.run(client.execute(method_ref=_METHOD_REF)) + + assert result.method_provenance is not None + assert result.method_provenance.commit_sha == "23dda75deadbeef" + + def test_pipe_code_beside_method_ref_is_legal(self, mocker: MockerFixture) -> None: + """`pipe_code` overrides the fetched manifest's `main_pipe` — it is a selector WITHIN + the run source, not a second source. + """ + client = self._client() + send = mocker.patch.object(client, "_send", mocker.AsyncMock(return_value=_response(202, json=_START_BODY))) + + asyncio.run(client.start(pipe_code="documents.summarize", method_ref=_METHOD_REF)) + + sent = send.call_args.kwargs["content"].decode("utf-8") + assert '"pipe_code":"documents.summarize"' in sent + assert f'"method_ref":"{_METHOD_REF}"' in sent + + def test_method_ref_and_inline_contents_are_mutually_exclusive(self, mocker: MockerFixture) -> None: + client = self._client() + send = mocker.patch.object(client, "_send", mocker.AsyncMock(return_value=_response(202, json=_START_BODY))) + + with pytest.raises(PipelineRequestError, match="method_ref and inline mthds_contents are mutually exclusive"): + asyncio.run(client.start(mthds_contents=['domain = "x"'], method_ref=_METHOD_REF)) + with pytest.raises(PipelineRequestError, match="method_ref and inline mthds_contents are mutually exclusive"): + asyncio.run(client.execute(mthds_contents=['domain = "x"'], method_ref=_METHOD_REF)) + + send.assert_not_called() + + def test_method_ref_and_method_id_are_mutually_exclusive(self, mocker: MockerFixture) -> None: + """An address run carries its own provenance, so it takes no run-history linkage id — + there is NO linkage exception for `method_ref` (that exception is inline+method_id's). + """ + client = self._client() + send = mocker.patch.object(client, "_send", mocker.AsyncMock(return_value=_response(202, json=_START_BODY))) + + with pytest.raises(PipelineRequestError, match="method_ref and method_id are mutually exclusive"): + asyncio.run(client.start(method_ref=_METHOD_REF, method_id="mt_1")) + with pytest.raises(PipelineRequestError, match="method_ref and method_id are mutually exclusive"): + asyncio.run(client.execute(method_ref=_METHOD_REF, method_id="mt_1")) + + send.assert_not_called() + + def test_extra_rejects_a_smuggled_method_ref(self) -> None: + """`extra` merges last into the body, so a smuggled copy would overwrite the validated + named option and bypass the selector-exclusivity checks — the key is reserved. + """ + client = self._client() + with pytest.raises(PipelineRequestError, match="method_ref"): + asyncio.run(client.start(pipe_code="p", extra={"method_ref": _METHOD_REF})) + with pytest.raises(PipelineRequestError, match="method_ref"): + asyncio.run(client.execute(pipe_code="p", extra={"method_ref": _METHOD_REF})) + + def test_empty_method_ref_is_absent(self, mocker: MockerFixture) -> None: + """`method_ref=""` selects nothing, so it is neither sent nor a run source.""" + client = self._client() + send = mocker.patch.object(client, "_send", mocker.AsyncMock(return_value=_response(202, json=_START_BODY))) + + asyncio.run(client.start(pipe_code="p", method_ref="")) + assert "method_ref" not in send.call_args.kwargs["content"].decode("utf-8") + + with pytest.raises(PipelineRequestError): + asyncio.run(client.start(method_ref="")) + + @pytest.mark.parametrize("wrong_typed_method_ref", [0, 123, [], ["github.com/x/y"], {}, 1.5, True]) + def test_non_string_method_ref_raises_before_any_request(self, mocker: MockerFixture, wrong_typed_method_ref: object) -> None: + """Same boundary rule as `method_id`: one wrong value, one answer, before the wire.""" + client = self._client() + send = mocker.patch.object(client, "_send", mocker.AsyncMock(return_value=_response(202, json=_START_BODY))) + + with pytest.raises(PipelineRequestError, match="method_ref must be a string"): + asyncio.run(client.execute(pipe_code="p", method_ref=wrong_typed_method_ref)) # type: ignore[arg-type] + with pytest.raises(PipelineRequestError, match="method_ref must be a string"): + asyncio.run(client.start(pipe_code="p", method_ref=wrong_typed_method_ref)) # type: ignore[arg-type] + + send.assert_not_called() + + def test_blocking_fallback_forwards_method_ref(self, mocker: MockerFixture) -> None: + """A `method_ref` run must run the same fetched package on the blocking fallback — + dropping the selector there would silently run nothing (or something else). + """ + client = self._client() + send = mocker.patch.object( + client, + "_send", + mocker.AsyncMock(side_effect=[_response(200, json=_BARE_VERSION), _response(200, json=_EXECUTE_BODY)]), + ) + + asyncio.run(client.start_and_wait(method_ref=_METHOD_REF)) + + execute_call = send.call_args_list[1] + assert execute_call.args[1] == f"{_BASE_URL}/v1/execute" + assert f'"method_ref":"{_METHOD_REF}"' in execute_call.kwargs["content"].decode("utf-8") diff --git a/tests/unit/test_client_validate.py b/tests/unit/test_client_validate.py index 0ceb310..0057ab4 100644 --- a/tests/unit/test_client_validate.py +++ b/tests/unit/test_client_validate.py @@ -167,3 +167,84 @@ def test_validate_files_empty_raises(self) -> None: client = self._client() with pytest.raises(PipelineRequestError): asyncio.run(client.validate_files([])) + + # ── method selectors (the strict tooling three-way XOR) ────────── + + def test_method_ref_selector_rides_the_body_without_mthds_contents(self, mocker: MockerFixture) -> None: + """A selector validation must NOT carry the `mthds_contents` key at all — the server + XORs on presence, and an empty list is a request-shape 422. + """ + client = self._client() + send = self._mock_send(mocker, client, json_body=_VALID_BODY) + + asyncio.run(client.validate(method_ref="github.com/Pipelex/methods/documents@v0.1.0")) + + body = self._sent_body(send) + assert send.call_args.args[1] == f"{_BASE_URL}/v1/validate" + assert body["method_ref"] == "github.com/Pipelex/methods/documents@v0.1.0" + assert "mthds_contents" not in body + assert "method_id" not in body + assert body["allow_signatures"] is False + # The markdown render injection holds on the selector path too. + assert body["render"] == ["markdown"] + + def test_method_id_selector_is_a_pure_pass_through(self, mocker: MockerFixture) -> None: + client = self._client() + send = self._mock_send(mocker, client, json_body=_INVALID_BODY) + + result = asyncio.run(client.validate(method_id="mt_1", allow_signatures=True, views=[VALIDATION_VIEW_INPUT_FORM])) + + body = self._sent_body(send) + assert body["method_id"] == "mt_1" + assert body["allow_signatures"] is True + assert body["views"] == ["input_form"] + assert "mthds_contents" not in body + # A produced invalid verdict still parses into the union's invalid arm. + assert isinstance(result, PipelexInvalidReport) + + @pytest.mark.parametrize( + "kwargs", + [ + {}, + {"mthds_contents": []}, + {"mthds_contents": ["bundle"], "method_ref": "github.com/x/y@v1"}, + {"mthds_contents": ["bundle"], "method_id": "mt_1"}, + {"method_ref": "github.com/x/y@v1", "method_id": "mt_1"}, + {"method_ref": ""}, + ], + ) + def test_exactly_one_selector_is_enforced(self, mocker: MockerFixture, kwargs: dict[str, object]) -> None: + """The tooling routes are stateless, so there is NO linkage exception (unlike the run + routes' inline+method_id): zero selectors and every pairing are refused client-side. + """ + client = self._client() + send = self._mock_send(mocker, client, json_body=_VALID_BODY) + + with pytest.raises(PipelineRequestError, match="exactly one method selector"): + asyncio.run(client.validate(**kwargs)) # type: ignore[arg-type] + + send.assert_not_called() + + def test_mthds_sources_is_rejected_beside_a_selector(self, mocker: MockerFixture) -> None: + """Source labels for a selector validation come from the package's (or the stored + method's) real file names — `mthds_sources` labels inline contents only. + """ + client = self._client() + send = self._mock_send(mocker, client, json_body=_VALID_BODY) + + with pytest.raises(PipelineRequestError, match="mthds_sources labels inline mthds_contents"): + asyncio.run(client.validate(method_ref="github.com/x/y@v1", mthds_sources=["a.mthds"])) + + send.assert_not_called() + + @pytest.mark.parametrize("wrong_typed_selector", [0, 123, [], {}, 1.5, True]) + def test_non_string_selector_raises_before_any_request(self, mocker: MockerFixture, wrong_typed_selector: object) -> None: + client = self._client() + send = self._mock_send(mocker, client, json_body=_VALID_BODY) + + with pytest.raises(PipelineRequestError, match="method_ref must be a string"): + asyncio.run(client.validate(method_ref=wrong_typed_selector)) # type: ignore[arg-type] + with pytest.raises(PipelineRequestError, match="method_id must be a string"): + asyncio.run(client.validate(method_id=wrong_typed_selector)) # type: ignore[arg-type] + + send.assert_not_called() diff --git a/tests/unit/test_crate_routes.py b/tests/unit/test_crate_routes.py new file mode 100644 index 0000000..54a474a --- /dev/null +++ b/tests/unit/test_crate_routes.py @@ -0,0 +1,168 @@ +"""The crate routes — `resolve` and `codegen` — and their three-form closure selector. + +Ports the relevant slice of `pipelex-sdk-js/tests/crate-routes.test.ts`: verb + path + body, +the 200-verdict discipline (branch on `is_valid`), the strict three-way XOR at request +construction, the hosted `method_id` pass-through, and the fetch-sized budget a +`method_ref` closure gets (the server may have to clone before it answers). +""" + +import asyncio +import json + +import httpx +import pytest +from pydantic import ValidationError +from pytest_mock import MockerFixture, MockType + +from pipelex_sdk.build_models import CrateInvalidReport, MthdsFileItem +from pipelex_sdk.client import PipelexAPIClient +from pipelex_sdk.crate_models import CodegenRequest, CodegenValidReport, ResolveRequest, ResolveValidReport +from pipelex_sdk.errors import ApiResponseError + +_BASE_URL = "http://localhost:8081" +_METHOD_REF = "github.com/Pipelex/methods/documents@v0.1.0" + +_RESOLVE_VALID: dict[str, object] = {"is_valid": True, "crate": {"concepts": {}, "pipes": {}, "domains": {}, "fingerprint": "abc"}, "message": "ok"} +_CODEGEN_VALID = { + "is_valid": True, + "kind": "types", + "target": "python-pydantic", + "crate_fingerprint": "abc", + "engine_version": "0.55.0", + "artifacts": [{"path": "models.py", "content": "# stamped\n"}], + "lock": 'lock_version = 1\ncrate_fingerprint = "abc"\n', + "lock_filename": "codegen.lock", + "message": "ok", +} +_INVALID = { + "is_valid": False, + "message": "closure did not validate", + "validation_errors": [{"category": "blueprint_validation", "message": "unknown pipe type"}], +} + + +def _response(status_code: int, *, json_body: object | None = None) -> httpx.Response: + request = httpx.Request("POST", f"{_BASE_URL}/x") + if json_body is not None: + return httpx.Response(status_code, json=json_body, request=request) + return httpx.Response(status_code, request=request) + + +class TestCrateRoutes: + def _client(self) -> PipelexAPIClient: + return PipelexAPIClient(api_key="test-token", base_url=_BASE_URL) + + def _mock_send(self, mocker: MockerFixture, client: PipelexAPIClient, response: httpx.Response) -> MockType: + return mocker.patch.object(client, "_send", mocker.AsyncMock(return_value=response)) + + # ── resolve ────────────────────────────────────────────────────── + + def test_resolve_posts_files_and_returns_the_crate(self, mocker: MockerFixture) -> None: + client = self._client() + send = self._mock_send(mocker, client, _response(200, json_body=_RESOLVE_VALID)) + + report = asyncio.run(client.resolve(ResolveRequest(files=[MthdsFileItem(content='domain = "demo"', source="b.mthds")]))) + + call = send.call_args + assert call.args[0] == "POST" + assert call.args[1] == f"{_BASE_URL}/v1/resolve" + assert json.loads(call.kwargs["content"]) == {"files": [{"content": 'domain = "demo"', "source": "b.mthds"}]} + assert isinstance(report, ResolveValidReport) + assert report.crate["fingerprint"] == "abc" + + def test_resolve_method_id_is_a_pure_pass_through(self, mocker: MockerFixture) -> None: + """Nothing is expanded client-side: the id rides the body alone and the platform + resolves it (a bare runner rejects the request as carrying no source it understands). + """ + client = self._client() + send = self._mock_send(mocker, client, _response(200, json_body=_RESOLVE_VALID)) + + asyncio.run(client.resolve(ResolveRequest(method_id="mt_1"))) + + assert json.loads(send.call_args.kwargs["content"]) == {"method_id": "mt_1"} + + def test_resolve_invalid_closure_is_a_200_verdict(self, mocker: MockerFixture) -> None: + client = self._client() + self._mock_send(mocker, client, _response(200, json_body=_INVALID)) + + report = asyncio.run(client.resolve(ResolveRequest(method_ref=_METHOD_REF))) + + assert isinstance(report, CrateInvalidReport) + assert report.validation_errors[0].message == "unknown pipe type" + + def test_resolve_no_verdict_raises_api_response_error(self, mocker: MockerFixture) -> None: + """A selector-resolution failure (no package at the address, an unknown id) is a + non-2xx — never an `is_valid: false` verdict. + """ + client = self._client() + self._mock_send(mocker, client, _response(404, json_body={"detail": "Unknown method", "code": "not_found"})) + + with pytest.raises(ApiResponseError) as exc_info: + asyncio.run(client.resolve(ResolveRequest(method_id="mt_ghost"))) + assert exc_info.value.status == 404 + + # ── codegen ────────────────────────────────────────────────────── + + def test_codegen_posts_axes_and_returns_artifacts_with_lock(self, mocker: MockerFixture) -> None: + client = self._client() + send = self._mock_send(mocker, client, _response(200, json_body=_CODEGEN_VALID)) + + report = asyncio.run(client.codegen(CodegenRequest(files=[MthdsFileItem(content="x")], target="python-pydantic"))) + + call = send.call_args + assert call.args[1] == f"{_BASE_URL}/v1/codegen" + assert json.loads(call.kwargs["content"]) == {"files": [{"content": "x"}], "kind": "types", "target": "python-pydantic"} + assert isinstance(report, CodegenValidReport) + assert report.artifacts[0].path == "models.py" + assert report.lock_filename == "codegen.lock" + + # ── the strict three-way XOR ───────────────────────────────────── + + @pytest.mark.parametrize( + "kwargs", + [ + {}, + {"files": [MthdsFileItem(content="x")], "method_ref": _METHOD_REF}, + {"files": [MthdsFileItem(content="x")], "method_id": "mt_1"}, + {"method_ref": _METHOD_REF, "method_id": "mt_1"}, + {"files": [MthdsFileItem(content="x")], "method_ref": _METHOD_REF, "method_id": "mt_1"}, + ], + ) + def test_request_construction_enforces_exactly_one_selector(self, kwargs: dict[str, object]) -> None: + """The tooling routes are stateless, so there is no linkage exception: zero selectors + and every pairing fail at construction, mirroring the server's request-shape 422. + """ + with pytest.raises(ValidationError, match="exactly one"): + ResolveRequest.model_validate(kwargs) + with pytest.raises(ValidationError, match="exactly one"): + CodegenRequest.model_validate({**kwargs, "target": "python-pydantic"}) + + # ── the fetch-sized budget ─────────────────────────────────────── + + def test_method_ref_closure_gets_the_fetch_budget(self, mocker: MockerFixture) -> None: + """Resolving an address can make the server clone a repository before it answers; the + 30s management budget would abort a legitimate cold-cache clone and blame the network. + """ + client = self._client() + codegen_valid = {**_CODEGEN_VALID, "target": "ts-zod"} + send = mocker.patch.object( + client, + "_send", + mocker.AsyncMock(side_effect=[_response(200, json_body=_RESOLVE_VALID), _response(200, json_body=codegen_valid)]), + ) + + asyncio.run(client.resolve(ResolveRequest(method_ref=_METHOD_REF))) + assert send.call_args.kwargs["request_timeout"] == 180.0 + + asyncio.run(client.codegen(CodegenRequest(method_ref=_METHOD_REF, target="ts-zod"))) + assert send.call_args.kwargs["request_timeout"] == 180.0 + + def test_inline_and_by_id_closures_keep_the_management_budget(self, mocker: MockerFixture) -> None: + client = self._client() + send = self._mock_send(mocker, client, _response(200, json_body=_RESOLVE_VALID)) + + asyncio.run(client.resolve(ResolveRequest(files=[MthdsFileItem(content="x")]))) + assert send.call_args.kwargs["request_timeout"] == 30.0 + + asyncio.run(client.resolve(ResolveRequest(method_id="mt_1"))) + assert send.call_args.kwargs["request_timeout"] == 30.0 diff --git a/uv.lock b/uv.lock index f0ce831..aa5f335 100644 --- a/uv.lock +++ b/uv.lock @@ -303,7 +303,7 @@ wheels = [ [[package]] name = "pipelex-sdk" -version = "0.7.0" +version = "0.8.0" source = { editable = "." } dependencies = [ { name = "httpx" }, From a2f7b0379bc7271a0663341167291b4c6d8d8589 Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Sat, 29 Aug 2026 05:44:10 +0200 Subject: [PATCH 2/2] fix: empty crate selectors normalize to absent before the XOR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile round-1 fix on PR #19: files=[], method_ref="" (or whitespace-only), and method_id="" counted as the sole selector under the non-None XOR in the crate/build request models and reached the wire as unusable values. Field-level validators now apply the same empty-as-absent rule the run routes' _normalized_selector boundary gives: an empty selector alone is zero selectors (the teaching XOR error at construction), and beside a real selector it is simply absent — never sent. Tests cover all three empty forms, alone and beside a real selector. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WoLcQhnFiPgVmDzHqPkRmQ --- CHANGELOG.md | 2 +- pipelex_sdk/build_models.py | 22 ++++++++++++++++++++- pipelex_sdk/crate_models.py | 18 +++++++++++++++-- tests/unit/test_build_inputs.py | 32 +++++++++++++++++++++++++++++++ tests/unit/test_crate_routes.py | 34 +++++++++++++++++++++++++++++++++ 5 files changed, 104 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b338a38..764994e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - **Provenance comes back typed.** A `method_ref` run's start ack is the new `PipelexRunResultStart` (`pipelex_sdk.runs`), carrying `method_provenance` — the new `MethodProvenance` shape `{address, tag, commit_sha}`, the SHA being what keeps the run explainable when a tag moves — and `PipelexExecuteResult` declares the same field on the blocking path. Both are `None` for inline-source and `method_id` runs. - **Client-side exclusivity guards mirroring the server's 422s.** A `method_ref` is a complete run source, so it pairs with nothing: combining it with inline `mthds_contents` or with `method_id` raises `PipelineRequestError` whose wording mirrors the server's validator — before anything hits the wire. The documented run-route exception is untouched: inline source + `method_id` stays legal (the inline source runs; the id demotes to run-history linkage), and `pipe_code` beside a `method_ref` stays legal (it overrides the manifest's `main_pipe`). - **`validate` takes method selectors.** `mthds_contents` is now optional, and the new keyword parameters `method_ref=` (runner-resolved by address, the package's real file names feeding the diagnostics' source labels) and `method_id=` (hosted-only, platform-resolved) select what is validated — under the tooling routes' strict three-way XOR: exactly one selector, no linkage exception, `mthds_sources` legal only beside inline contents. A selector validation sends no `mthds_contents` key at all. A selector-resolution failure (fetch failure, no package at the address, an unknown id) is a non-2xx, never an `is_valid: false` verdict. -- **The crate routes, with the typed `method_id` pass-through.** New `resolve()` and `codegen()` client methods for `POST /v1/resolve` (the normalized library crate) and `POST /v1/codegen` (stamped typed artifacts plus their `codegen.lock`), with the new `pipelex_sdk.crate_models` wire models (`ResolveRequest` / `ResolveResponse`, `CodegenRequest` / `CodegenResponse`, `GeneratedArtifact`, `CodegenKind` / `CodegenTarget`). Their closure is exactly one of inline `files` / an address-form `method_ref` / the hosted `method_id`, enforced at request construction as well as by the server; `method_id` is a pure server pass-through the platform resolves (an unknown or foreign-org id is a `404`, a stored method with no MTHDS source a `422`). This closes the JS-parity gap the architecture doc carried for the two routes. +- **The crate routes, with the typed `method_id` pass-through.** New `resolve()` and `codegen()` client methods for `POST /v1/resolve` (the normalized library crate) and `POST /v1/codegen` (stamped typed artifacts plus their `codegen.lock`), with the new `pipelex_sdk.crate_models` wire models (`ResolveRequest` / `ResolveResponse`, `CodegenRequest` / `CodegenResponse`, `GeneratedArtifact`, `CodegenKind` / `CodegenTarget`). Their closure is exactly one of inline `files` / an address-form `method_ref` / the hosted `method_id`, enforced at request construction as well as by the server — with an empty selector (`files=[]`, a blank or whitespace-only string) normalized to absent before the XOR counts, the same empty-as-absent rule as the run routes, so an unusable value never passes as the sole selector; `method_id` is a pure server pass-through the platform resolves (an unknown or foreign-org id is a `404`, a stored method with no MTHDS source a `422`). This closes the JS-parity gap the architecture doc carried for the two routes. - **`build_inputs` takes a `method_ref` closure.** `BuildInputsRequest` now extends the shared `CrateRequestBase` envelope (`files` XOR `method_ref`); the address form is server-resolved, the registry form keeps its `501`. - **A `method_ref` request gets a fetch-sized budget.** Resolving an address can make the server clone a repository before it answers, and the server-side clone timeout runs well past the client's 30s management budget on a cold cache — an abort there would report a healthy, still-cloning server as unreachable. A `method_ref`-carrying `build_inputs`, `resolve`, or `codegen` uses an internal 3-minute budget; it is internal (no new caller-facing parameter) and inert behind the hosted gateway's own cap. The run routes and `validate` need no such override — they already ride the 20-min blocking ceiling, unlike the JS SDK's short start budget. diff --git a/pipelex_sdk/build_models.py b/pipelex_sdk/build_models.py index 2fceee3..2e32765 100644 --- a/pipelex_sdk/build_models.py +++ b/pipelex_sdk/build_models.py @@ -14,7 +14,7 @@ from typing import Annotated, Any, Literal, Self, TypeAlias -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, field_validator, model_validator from pipelex_sdk.validation_models import ValidationErrorItem @@ -43,6 +43,11 @@ class CrateRequestBase(BaseModel): real relative paths as per-file sources. The **registry form** (any non-address reference) stays reserved and answers `501` until a method registry exists. + An EMPTY selector is normalized to absent before the exclusivity check — `files=[]` + selects no closure and `method_ref=""` (or whitespace-only) no address, the same + empty-as-absent rule the run routes apply — so an unusable value never counts as the + sole selector and never reaches the wire. + The subclasses own the exclusivity validator, because the crate routes add a third selector (the hosted `method_id`) that the build projections deliberately refuse. """ @@ -50,6 +55,21 @@ class CrateRequestBase(BaseModel): files: list[MthdsFileItem] | None = None method_ref: str | None = None + @field_validator("files") + @classmethod + def _empty_files_are_absent(cls, value: list[MthdsFileItem] | None) -> list[MthdsFileItem] | None: + # `files=[]` is not a closure — normalize to absent so the XOR counts real selectors only. + return value or None + + @field_validator("method_ref") + @classmethod + def _blank_method_ref_is_absent(cls, value: str | None) -> str | None: + # A blank address selects nothing — same empty-as-absent rule as the run routes' + # `_normalized_selector` boundary. A real value is passed through untouched. + if value is None or not value.strip(): + return None + return value + class BuildInputsRequest(CrateRequestBase): """Request for `POST /v1/build/inputs`. The closure is inline `files` XOR a diff --git a/pipelex_sdk/crate_models.py b/pipelex_sdk/crate_models.py index 6ab3a9e..5048e7e 100644 --- a/pipelex_sdk/crate_models.py +++ b/pipelex_sdk/crate_models.py @@ -21,7 +21,7 @@ from typing import Annotated, Any, Literal, Self, TypeAlias -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, field_validator, model_validator from pipelex_sdk.build_models import CrateInvalidReport, CrateRequestBase @@ -35,14 +35,28 @@ class CrateToolingRequest(CrateRequestBase): source before the runner sees the request — nothing is expanded client-side, and it is meaningless off-platform. An unknown or foreign-org id is a `404` (indistinguishable by design); a stored method with no MTHDS source is a `422`. + + An EMPTY selector is normalized to absent before the XOR counts (the base normalizes + `files` / `method_ref`; `method_id` follows the same rule here), so an unusable value + never counts as the sole selector and never reaches the wire. """ method_id: str | None = None + @field_validator("method_id") + @classmethod + def _blank_method_id_is_absent(cls, value: str | None) -> str | None: + # A blank id selects nothing — same empty-as-absent rule as the run routes' + # `_normalized_selector` boundary. A real value is passed through untouched. + if value is None or not value.strip(): + return None + return value + @model_validator(mode="after") def _exactly_one_selector(self) -> Self: # The strict tooling XOR, enforced at construction so an illegal shape fails - # before anything hits the wire (the server 422s the same shapes). + # before anything hits the wire (the server 422s the same shapes). Runs after + # the field-level empty-as-absent normalization, so it counts real selectors. selector_count = sum(1 for selector in (self.files, self.method_ref, self.method_id) if selector is not None) if selector_count != 1: msg = "provide exactly one of `files`, `method_ref`, or `method_id`" diff --git a/tests/unit/test_build_inputs.py b/tests/unit/test_build_inputs.py index d3d8480..c262a00 100644 --- a/tests/unit/test_build_inputs.py +++ b/tests/unit/test_build_inputs.py @@ -151,6 +151,38 @@ def test_request_construction_enforces_files_xor_method_ref(self, kwargs: dict[s with pytest.raises(ValidationError, match="exactly one"): BuildInputsRequest.model_validate(kwargs) + @pytest.mark.parametrize( + "kwargs", + [ + {"files": []}, + {"method_ref": ""}, + {"method_ref": " "}, + {"files": [], "method_ref": " "}, + ], + ) + def test_empty_closure_selectors_are_absent_and_fail_the_xor(self, kwargs: dict[str, object]) -> None: + """`files=[]` and a blank `method_ref` select nothing — normalized to absent before the + XOR, so an unusable value is refused at construction instead of reaching the wire. + """ + with pytest.raises(ValidationError, match="exactly one"): + BuildInputsRequest.model_validate(kwargs) + + def test_blank_method_ref_beside_files_is_simply_absent(self, mocker: MockerFixture) -> None: + """Same rule as the run boundary: an empty selector beside a real one is absent, not a + conflict — the closure is `files`, the empty key is not sent, the budget stays 30s. + """ + request = BuildInputsRequest(files=[MthdsFileItem(content="x")], method_ref="") + assert request.method_ref is None + + client = self._client() + valid: dict[str, object] = {"is_valid": True, "pipe_ref": "demo.main", "message": "ok", "format": "json", "explicit": False, "inputs": {}} + send = self._mock_send(mocker, client, _response(200, json_body=valid)) + asyncio.run(client.build_inputs(request)) + + body = json.loads(send.call_args.kwargs["content"]) + assert "method_ref" not in body + assert send.call_args.kwargs["request_timeout"] == 30.0 + def test_method_id_is_refused_with_a_teaching_error(self) -> None: """The `/v1/build/*` projections take no `method_id` — a teaching error beats pydantic silently ignoring the unknown key for a caller migrating off the by-id habit. diff --git a/tests/unit/test_crate_routes.py b/tests/unit/test_crate_routes.py index 54a474a..6d0e88e 100644 --- a/tests/unit/test_crate_routes.py +++ b/tests/unit/test_crate_routes.py @@ -137,6 +137,40 @@ def test_request_construction_enforces_exactly_one_selector(self, kwargs: dict[s with pytest.raises(ValidationError, match="exactly one"): CodegenRequest.model_validate({**kwargs, "target": "python-pydantic"}) + @pytest.mark.parametrize( + "kwargs", + [ + {"files": []}, + {"method_ref": ""}, + {"method_ref": " "}, + {"method_id": ""}, + {"method_id": " \t"}, + {"files": [], "method_ref": "", "method_id": ""}, + ], + ) + def test_empty_selectors_are_absent_and_fail_the_xor(self, kwargs: dict[str, object]) -> None: + """`files=[]` and blank strings select nothing — the same empty-as-absent rule as the + run routes — so an empty selector never counts as the sole one and never reaches the + wire as an unusable value: alone it is zero selectors, refused at construction. + """ + with pytest.raises(ValidationError, match="exactly one"): + ResolveRequest.model_validate(kwargs) + with pytest.raises(ValidationError, match="exactly one"): + CodegenRequest.model_validate({**kwargs, "target": "python-pydantic"}) + + def test_empty_selector_beside_a_real_one_is_simply_absent(self, mocker: MockerFixture) -> None: + """An empty selector beside a real one is absent, not a conflict — exactly-one + semantics stay coherent with the run boundary, and the empty key is not sent. + """ + request = ResolveRequest(files=[MthdsFileItem(content="x")], method_ref="", method_id=" ") + assert request.method_ref is None + assert request.method_id is None + + client = self._client() + send = self._mock_send(mocker, client, _response(200, json_body=_RESOLVE_VALID)) + asyncio.run(client.resolve(request)) + assert json.loads(send.call_args.kwargs["content"]) == {"files": [{"content": "x"}]} + # ── the fetch-sized budget ─────────────────────────────────────── def test_method_ref_closure_gets_the_fetch_budget(self, mocker: MockerFixture) -> None: