diff --git a/CHANGELOG.md b/CHANGELOG.md index 3734092..5540e38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,92 @@ # Changelog +## 1.2.0b1 - 2026-09-28 + +Feature-complete beta for the declarative environment workflow. + +### Architecture reset + +- Replace the discarded lock-then-plan prototype with a resource-aware + workflow: `plan` resolves only from Simple API/Core Metadata, inventories all + resources, and writes `.pload_plan.toml`; `apply` executes that exact file. +- Never download wheel bodies during planning. Indexes without PEP 658/714 + metadata fail with an actionable message instead of silently filling cache. +- Discover exact wheels in pload, pip and uv caches and copyable distributions + in compatible pload environments. Recheck SHA-256 or installed `RECORD` + fingerprints during apply. +- Add an interactive per-package route chooser with previous/next navigation, + undo, reset-to-fastest, save and save-and-apply actions. Show only direct + user requirements; route transitive dependencies automatically. +- Add explicit `pload remote add PACKAGE --from ENV --remote REPO`; apply and + describe never publish package bytes automatically. +- Persist exact artifact URLs and hashes, reject stale or incomplete plans, and + stop on selected-route failure without trying alternatives. + +### Added + +- Introduce one terminal design system for prompts, tables, progress, success, + warning and error output. Guided creation and route planning now use familiar + cursor-key menus with numbered non-TTY fallbacks instead of bespoke command + loops. +- Respect `NO_COLOR` consistently and keep animation limited to real terminals; + set `PLOAD_NO_PROGRESS=1` to replace spinners with plain status text. +- Keep user intent in a clean portable `pload.toml` and store pload-managed + exact dependency, artifact and hash decisions in sibling `.pload_lock.toml`. +- Add `pload describe`, metadata-only `pload plan` and idempotent `pload apply`. +- Inventory the local wheel cache, adjacent artifacts, local/SSH content stores, + configured indexes and Python installations before choosing a route. +- Allow package-specific indexes in `describe`, including CUDA wheel indexes, + while keeping credentials out of the portable configuration. +- Reuse exact wheel archives by SHA-256, deduplicate local/SSH storage, record + detected NVIDIA driver provenance and roll back failed materializations. +- Add complete field-by-field configuration, lock and plan references, enforce agreement between + desired dependency pins and package locks, and make policy repositories and + repositories drive lookup while publication remains explicit. +- Validate locked wheel Python, ABI and platform tags against the requested + runtime during planning; incompatible cache entries can no longer be reported + as ready, and compatible mode explicitly plans network re-resolution instead. +- Accept unpinned dependency intentions such as `numpy` and `pandas>=2`; the + metadata-only planner resolves the complete dependency closure and records + exact versions, hashes, URLs and tags without fetching wheel bodies. +- Treat edits that add or change dependencies beside an existing lock as a stale + lock to reconcile, not an invalid configuration; re-resolution + uses the complete requested set and prefers existing pload cache wheels. +- Migrate lock tables embedded by earlier 1.1 previews into `.pload_lock.toml` + without another dependency resolution or artifact download. +- Show animated status for metadata resolution and artifact-repository inspection + instead of appearing frozen while network or SSH work is captured. +- Refuse `apply` when the lock is absent or stale instead of resolving and + downloading implicitly. +- Add `pload apply --local` / `-l` to reproduce a saved plan into `.venv` + beside its configuration file, matching `pload init` project layout. +- Render apply transfers with Rich's adaptive-width standard progress bar, + explicit boundaries and the acquisition method in brackets. +- Stream direct downloads in 16 KiB chunks and report measured byte progress at + up to 30 updates per second while coalescing non-interactive logs. +- Stream local repository copies and SSH object responses so repository routes + report real byte progress instead of completing at once. +- Remove grey source/status suffixes from progress rows. Cache verification and + local artifact copying now report actual processed bytes before completion. +- Replace environment-copy's opaque zip/extract step with a validated file-list + staging stream that reports total and copied bytes and finishes green before + the next package begins. + +### Removed + +- Drop the unreleased transport-oriented `export`, `restore`, and repository + push/pull interface in favor of the declarative configuration workflow. + +### Fixed + +- Accept Rich's documented ASCII progress-bar fallback on Windows consoles in + the cross-platform release test suite. +- Keep an external environment's interpreter path intact when probing packages, + instead of following its Python symlink out to the base installation. +- Batch SSH repository inventory, publication and retrieval so environments with + several packages do not spend most of their time reconnecting. Show package + and transfer progress, enforce bounded SSH/SFTP operations, and report actual + transfer timeouts instead of appearing frozen. + ## 1.0.0 - 2026-09-24 ### Added diff --git a/README.md b/README.md index 27b1b54..7456cde 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ complete parameter reference and `-h -d` for practical examples and effects. ## Contents - [Install](#install) +- [Reproduce and share environments](#reproduce-and-share-environments) - [Upgrade](#upgrade) - [First run](#first-run) - [Configure later](#configure-later) @@ -20,9 +21,11 @@ complete parameter reference and `-h -d` for practical examples and effects. - [Discover and install Python](#discover-and-install-python) - [Create and activate environments](#create-and-activate-environments) - [Help and aliases](#help-and-aliases) +- [Terminal experience](#terminal-experience) - [Isolation and directory layout](#isolation-and-directory-layout) - [Command reference](#command-reference) - [Development](#development) +- [License](#license) ## Install @@ -50,10 +53,11 @@ pipx install pload pload-install ``` -The installer uses colored numbered choices for the pload data directory, +The installer uses consistent cursor-key choices for the pload data directory, launcher directory, virtual-environment directory, Python directory, runtime -source, pip index, and optional shell integration. Press Enter to accept a -marked default. +source, pip index, and optional shell integration. Use `↑`/`↓` to move and +Enter to accept the highlighted value. Without a terminal UI, the same prompts +fall back to numbered choices. After installation it prints an ASCII welcome screen with copyable next steps. The same screen is shown whenever you run bare `pload` (see the screenshot at @@ -74,6 +78,60 @@ python -m pload.installer --yes \ `--yes` accepts explicit values and saved defaults. It only changes a shell profile when `--shell` is supplied or already configured. +## Reproduce and share environments + +The `1.2` preview uses one clean, user-authored `pload.toml` as portable intent. +pload manages `.pload_lock.toml` (exact versions and artifacts) and +`.pload_plan.toml` (the selected acquisition route for every package): + +```console +pload describe v2 -o pload.toml +pload plan pload.toml +pload apply pload.toml +pload apply --local # reproduce into .venv beside pload.toml +``` + +`plan` fetches only index pages and independent package metadata—never wheel +bodies—and inventories pload/pip/uv caches, compatible existing environments, +local/SSH stores and indexes. Its interactive chooser shows only dependencies +written by the user; transitive dependencies are routed automatically. It +supports previous/next, undo, reset, save and save-and-apply through a +compact shortcut row (`p/n/u/r/s/a/q`); the selection list itself contains only +real package sources. `apply` executes exactly the saved plan: +it never re-resolves, changes route, falls back, or uploads packages. Set +`[plan].mode = "auto"` in `pload.toml` to save recommended routes without the +chooser; strict per-package exceptions live under `[plan.packages.NAME]`. +`apply` prints the saved plan first, then compact Docker-style progress rows. +Each package is prefixed by its route, such as `[cache]`, `[index-exact]`, or +`[repository]`, followed only by its byte progress. Verbose grey source/status +suffixes are intentionally omitted because the saved plan above already names +the selected resource. The bar +uses Rich's standard adaptive-width bar with explicit start and end boundaries, +so it grows naturally on wider terminals without another progress dependency. +Index and repository transfers report real received bytes at up to 30 updates +per second. SSH repository downloads stream the remote object directly; +cache routes report bytes while verifying SHA-256 instead of turning green +before visible work. Environment-copy routes validate the source `RECORD`, then +immediately stage each byte from the compatible environment and finish green +before pload advances to the next package. The staged files are installed after +the target environment exists. +Redirected logs remain coalesced instead of flooding CI. + +Use `pload apply --local` (or `-l`) for a project-local environment. It creates +`.venv` beside the selected `pload.toml`, matching the layout produced by +`pload init` while executing the saved declarative plan exactly. + +Back up a special or slow wheel only when you explicitly request it: + +```console +pload remote add torch --from v10 --remote lab +``` + +See the [declarative environment guide](docs/declarative-environments.md) for the +[complete TOML field reference](docs/declarative-environments.md#complete-ploadtoml-field-reference), +exact versus compatible policies, resource-selection algorithm, CUDA wheel +caching, idempotency and current platform boundaries. + ## Upgrade Check the installed version first: @@ -90,6 +148,12 @@ and virtual environments are kept: pload-install --yes --package-spec "pload==1.0.0" ``` +To opt into the declarative-environment pre-release for testing: + +```console +pload-install --yes --package-spec "pload==1.2.0b1" +``` + To always follow the newest published release instead of pinning a version, omit the version constraint: @@ -147,7 +211,10 @@ pload v1 # activate an environment by ID Run `pload new` without options to start the guided creator. It lists detected Python runtimes, then asks for the interpreter, environment name, description, -and optional packages. Press Enter to accept the suggested value at each step. +and optional packages. When packages are present, choose **Automatic** to let +pload prefer a compatible local wheel cache and otherwise use the configured +index, or **Custom** to choose the source of each top-level package yourself. +Press Enter to accept the suggested value at each step. ## Configure later @@ -265,6 +332,7 @@ Create a named environment with a description: pload new # guided creation pload new -n data -v 3.12 -m "Data analysis" pload new -n web -v 3.12 -r fastapi uvicorn -m "Web API" +pload new -n lab -r numpy torch -s custom # choose each top-level package source pload new -v 3.12 --message "Temporary data tools" # auto-named safely pload list pload v1 @@ -295,6 +363,13 @@ The table includes ID, name, Python version, description, and full path. A spinner is shown in interactive terminals; redirected and CI output uses stable ordinary log lines. +Package requests are installed one at a time. A misspelled, unavailable, or +invalid request is reported and skipped, while later requests still run and +successful installations remain in the environment. The command returns a +failure status at the end when any requested package failed, with one summary +listing those packages. Source selection applies only to packages explicitly +entered by the user; pip continues to resolve their transitive dependencies. + ## Help and aliases Brief help is a complete reference. It starts with an emphasized `USAGE` panel, @@ -345,6 +420,19 @@ pload python install 3.12 pload py p 3.12 ``` +## Terminal experience + +All interactive screens share the same keyboard and visual language. Use +`↑`/`↓` and Enter in guided creation, setup, and package-route selection; +recommended values start highlighted. Status, success, warning, and error +messages use the same symbols and colors everywhere, while compact border-light +tables keep lists readable. + +`NO_COLOR=1` disables color. `PLOAD_NO_PROGRESS=1` replaces animated progress +with plain status lines. Every guided workflow also has an explicit flag-based +form for scripts. See the [terminal experience specification](docs/terminal-experience.md) +for the complete interaction and output contract. + ## Isolation and directory layout Put all pload-managed data under one root: diff --git a/docs/MAINTENANCE.md b/docs/MAINTENANCE.md index 0b3685d..fb30491 100644 --- a/docs/MAINTENANCE.md +++ b/docs/MAINTENANCE.md @@ -3,6 +3,13 @@ Changes are developed on a release branch and merged to `main` only after the release checklist passes. Each stable merge receives a matching annotated tag. +## Pre-release testing + +Every candidate uses a PEP 440 pre-release version. After the cross-platform +matrix passes, install its wheel into the maintainer's private macOS pload runtime +without changing `PLOAD_HOME`, environment state or caches. Promotion to a stable +version requires explicit acceptance after that Mac test. + ## 0.4.x: portable foundation - Centralize filesystem and interpreter resolution. diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md new file mode 100644 index 0000000..30ecd4d --- /dev/null +++ b/docs/declarative-environments.md @@ -0,0 +1,325 @@ +# Declarative environments + +The `1.2` preview treats reproduction as resource scheduling, not as a sequence +of commands the user must design. You carry one small `pload.toml`; pload +resolves exact versions, inventories available resources, saves an auditable +route for every package, and executes exactly those routes. + +```text +pload.toml (intent) + │ + ▼ +metadata-only resolution ──► .pload_lock.toml (exact graph and wheel identities) + │ + ▼ +local/remote inventory ─────► .pload_plan.toml (one chosen route per package) + │ + ▼ +pload apply ────────────────► environment matching the lock +``` + +The normal workflow is: + +```console +pload plan pload.toml # resolve metadata, inventory and choose routes +pload apply pload.toml # execute only the saved plan +pload apply --local # execute into .venv beside pload.toml +``` + +By default, `plan` opens a route chooser for only the direct requirements the +user wrote in `pload.toml`. Set `[plan].mode = "auto"` to save the recommended +routes without opening the chooser. Transitive dependencies remain exact and +auditable, but pload automatically assigns their lowest-cost valid routes from +the measured data available. Arrow keys and Enter select a real route; `n` +moves forward, `p` goes back, `u` undoes, `r` restores the recommended choices, +`s` saves, and `a` saves then applies. Use +`pload plan --no-ui` in CI. Use `--json` for machine-readable inspection. + +## Three files, one user-owned file + +- `pload.toml` is portable user intent. This is the only file users edit. +- `.pload_lock.toml` is pload-managed exact resolution data. +- `.pload_plan.toml` is pload-managed execution data bound to that exact config + and lock. + +The two dotfiles exist because a request such as `torch` does not identify one +version, dependency closure, wheel build, or byte sequence. The lock records +those facts; the plan records where each exact artifact will come from. Editing +`pload.toml` invalidates both through SHA-256 bindings. Deleting either managed +file is safe: the next online `plan` regenerates it, although a new resolution +may differ if an index changed. + +`plan` may request Simple API project pages and independent `.metadata` +files. It never requests a wheel body, invokes `pip download`, copies packages, +or populates a cache. If an index does not expose independent Core Metadata, +planning stops and suggests adding a metadata-capable source. `apply` never +re-resolves, changes route, or publishes artifacts when a selected route fails. + +## Complete `pload.toml` reference + +```toml +schema = 1 +name = "training" + +[environment] +python = "3.12.7" +implementation = "cpython" +system = "Linux" +machine = "x86_64" +dependencies = ["numpy>=2,<3", "torch"] + +[capabilities] +nvidia_driver = ["550.90.07"] + +[policy] +reproducibility = "exact" +network = "allow" +source_build = "forbid" +publish_missing_artifacts = false +repositories = ["lab"] + +[plan] +mode = "auto" + +[plan.packages.torch] +method = "repository" +location = "lab" + +[sources.pypi] +kind = "index" +url = "https://pypi.org/simple" + +[sources.mirror] +kind = "index" +url = "https://pypi.tuna.tsinghua.edu.cn/simple" + +[repositories.lab] +kind = "ssh" +location = "frpxiaoxin:/home/tibless/pload-cloud" +``` + +### Top level + +| Field | Type | Meaning | +| --- | --- | --- | +| `schema` | integer, required | Configuration format; currently `1`. | +| `name` | string, required | Default environment name used by `apply`. Allowed characters are letters, numbers, dots, underscores and hyphens. | + +### `[environment]` + +| Field | Default | Meaning | +| --- | --- | --- | +| `python` | required | Exact target Python, including patch version. | +| `implementation` | `"cpython"` | Required interpreter implementation. | +| `system` | empty | Optional hard `platform.system()` constraint. | +| `machine` | empty | Optional hard `platform.machine()` constraint. Common aliases such as `AMD64`/`x86_64` and `aarch64`/`arm64` are normalized. | +| `dependencies` | `[]` | Direct requirements: names, ranges, or exact pins. Direct URLs and environment markers are rejected in schema 1. Transitive packages stay out of this user file. | + +An unpinned entry is intentional. During `plan`, pload selects the newest +compatible dependency graph using only index metadata, then writes exact +versions and wheel identities to the managed lock. + +### `[capabilities]` + +Every capability is an array of strings. `nvidia_driver` records observed +driver versions for audit and future scheduling. pload does not install CUDA +drivers. CUDA-enabled Python wheels can still be cached locally or remotely and +reused exactly, which avoids repeatedly downloading large special builds. + +### `[policy]` + +| Field | Default | Meaning | +| --- | --- | --- | +| `reproducibility` | `"exact"` | `exact` requires a locked wheel hash or a verified environment-copy route. `compatible` may install the locked version from an index when no exact artifact is known. | +| `network` | `"allow"` | `offline` forbids index access and Python installation; local and configured SSH resources remain usable. CLI `--offline` can only make policy stricter. | +| `source_build` | `"fallback"` | In compatible mode, `forbid` disallows source distributions. Exact wheel routes never build source. | +| `publish_missing_artifacts` | `false` recommended | Retained for schema-1 compatibility but ignored by `apply`. Upload is always explicit through `pload remote add`. | +| `repositories` | `[]` | Repositories to search while planning. Every name must have a matching repository table. This never authorizes upload. | + +### `[plan]` and `[plan.packages.NAME]` + +| Field | Default | Meaning | +| --- | --- | --- | +| `mode` | `"interactive"` | `interactive` opens the route chooser; `auto` immediately saves the recommended measured routes. | +| `plan.packages.NAME.method` | optional | Require one acquisition method for a special package, such as `repository`, `index-exact`, `cache`, or `environment-copy`. | +| `plan.packages.NAME.location` | optional | Further require a specific source, repository, environment, or path. | + +Most configurations only need `mode = "auto"`. Package overrides are strict: +if the requested route is not currently available, planning reports the package +as unavailable instead of silently choosing another route. Overrides are also +checked again by `apply`, so editing the managed plan cannot bypass them. + +### `[sources.NAME]` + +| Field | Meaning | +| --- | --- | +| `kind` | Must be `"index"`. | +| `url` | Public HTTP(S) Simple API base URL. Credentials and query tokens are rejected from portable files. | + +Sources are searched for compatible wheel metadata. A mirror may serve wheel +files but omit PEP 658/714 Core Metadata; in that case add a metadata-capable +source such as PyPI. pload will not hide this limitation by downloading wheels +during analysis. + +### `[repositories.NAME]` + +| Field | Meaning | +| --- | --- | +| `kind` | `"local"` or `"ssh"`. | +| `location` | Local path, or `HOST:/absolute/path` using normal OpenSSH configuration. | + +Objects live at `objects/` and searchable package records at +`packages///.json`. Credentials remain in the +host's SSH agent/config, never in TOML. + +Configure providers separately, then back up selected packages explicitly: + +```console +pload repo add lab frpxiaoxin:/home/tibless/pload-cloud -t ssh +pload remote add torch --from v10 --remote lab +pload remote add numpy==2.1.0 -f ./project/.venv -r lab +``` + +`remote add` reuses a matching original wheel when possible. If it is absent, +the command may fetch that exact installed version from `--index`; this is an +explicit storage operation, unlike `plan` and `apply`. + +## Managed lock reference + +`.pload_lock.toml` contains: + +| Field | Meaning | +| --- | --- | +| `schema` | Managed lock format, currently `1`. | +| `configuration` | Sibling user configuration filename. | +| `configuration_sha256` | Canonical digest that makes edited configs stale. | +| `[environment].dependencies` | Complete exact transitive closure. | +| `[[package]].name/version` | One exact resolved distribution. | +| `[[package]].sources` | Source that supplied its metadata. | +| `[[package.artifact]].filename` | Exact compatible wheel filename. | +| `sha256` | Required wheel body identity. | +| `tags` | Python/ABI/platform compatibility tags. | +| `url` | Exact artifact URL selected from the Simple API. | +| `size` | Advertised artifact bytes when the index supplies it. | +| `metadata_sha256` | Hash of the independently fetched Core Metadata when supplied. | +| `repositories` | Known repositories containing the object; policy repositories are also checked by hash. | + +Resolution honors `Requires-Python`, wheel tags, `Requires-Dist`, dependency +markers, yanked releases, hashes and target Python/platform. A repository +without independent metadata is reported as blocked instead of causing an +analysis-time wheel download. + +## Managed plan reference + +`.pload_plan.toml` stores configuration and lock hashes, the Python action, +readiness, and one route for every locked package: + +```toml +schema = 1 +configuration = "pload.toml" +configuration_sha256 = "..." +lock_sha256 = "..." +name = "training" +ready = true + +[[package]] +name = "numpy" +version = "2.2.1" +method = "external-cache" +location = "/Users/me/Library/Caches/pip/.../numpy-2.2.1.whl" +status = "ready" +measurement = "read 1.1 GiB/s" + +[package.artifact] +filename = "numpy-2.2.1-cp312-cp312-macosx_14_0_arm64.whl" +sha256 = "..." +url = "https://files.pythonhosted.org/..." +``` + +| Method | Meaning | +| --- | --- | +| `cache` | Exact SHA-matching wheel in pload's cache. | +| `configuration-artifact` | Exact wheel under the configuration's `artifacts/`. | +| `external-cache` | Exact wheel in configured or common pip/uv cache roots. | +| `environment-copy` | Exact installed distribution from a compatible pload environment, guarded by its `RECORD` fingerprint. Packages with files outside site-packages are excluded. | +| `repository` | SHA-addressed object from a local/SSH store. | +| `index-exact` | Exact locked URL, downloaded only during apply and verified by SHA-256. | +| `index-resolve` | Compatible-mode fallback for the already locked version. | + +The chooser never prints invented duration estimates. During each plan it +measures local artifact verification throughput, TCP connection RTT for index +hosts, and the actual availability-check RTT for repositories. These readings +appear as values such as `read 1.1 GiB/s`, `RTT 75 ms`, or `RTT unavailable`. +Because RTT is not transfer bandwidth, pload does not turn it into a fictional +download duration. Automatic ranking uses locality first and the current +measurements within each route class: pload cache, adjacent/external artifacts, +compatible local environments, then repository/index routes. The chooser can +override it for direct requirements. Transitive requirements use automatic +ranking so installing `torch` does not ask the user to decide separately for +`filelock`, `sympy`, and every other implementation detail. If any chosen +resource changes or disappears, `apply` stops and asks for a new plan; it never +tries another candidate. The selected method is authoritative: choosing +`index-exact` performs and verifies that network download even when an identical +wheel is already present in pload's cache. Apply then installs all verified +artifacts locally with networking disabled, so `Processing .../cache/wheels/...` +is the installation phase rather than evidence that the selected route changed. + +### What planning reads and writes + +With a current lock, planning only inventories possible routes. It indexes each +external cache once, inspects only matching-Python environments and only the +locked package names, checks configured repositories, and measures endpoint +RTT. The analysis API does not create or overwrite `.pload_plan.toml`. +Interactive Save/Apply, `--no-ui`, and `[plan].mode = "auto"` persist choices; +quitting or `--json` does not. + +`pload apply` first prints the complete persisted plan table. It then uses +compact Docker-style progress rows: every exact package starts with its selected +method in brackets (`[cache]`, `[index-exact]`, `[repository]`, and so on), +followed by the package, Rich's bounded adaptive-width progress bar and real +byte counts. Source descriptions and status prose are omitted from the row +because the saved plan table immediately above already identifies the resource. +The bar consumes the terminal width still available after the useful text +columns, so wide terminals get a substantially longer bar without breaking +narrow ones. Direct artifact downloads read in 16 KiB chunks and +publish real byte counts at up to 30 updates per second; this keeps interactive +motion fluid without inventing intermediate progress. Local repositories stream +their copied bytes, while SSH repositories stream the remote object into the +local destination at the same refresh cadence. Redirected output coalesces those +updates into stable plain-text rows without terminal control characters. Cache +routes report bytes as their SHA-256 content is actually read; configuration +artifacts and external caches report bytes while copying and hashing. +Environment-copy routes first validate the planned version and `RECORD` +fingerprint, then immediately stage the validated file list with a real total +and processed byte count. The row finishes before pload advances to the next +package; staged files move into site-packages after the target environment is +created. This makes the saved decision and +its execution visible in one command without allowing apply to recalculate the +plan. + +By default apply creates a named managed environment. `pload apply --local` +instead creates `.venv` beside the configuration file, matching `pload init`'s +project-local layout while retaining the saved declarative plan. Placement is a +CLI concern so the portable environment description remains independent of a +particular checkout path. + +An unpinned configuration with a missing or stale lock has one additional job: +it must determine exact versions and the transitive graph before routes have a +meaning. That first resolution reads Simple API pages and independently served +Core Metadata and updates `.pload_lock.toml`; it still never downloads wheel +bodies. Later plans reuse the current lock and skip metadata resolution. + +## Current boundaries + +- Wheel reuse requires matching Python, ABI, OS and architecture tags. +- Environment copying additionally requires exact Python implementation, + version, OS, architecture and an unchanged installed `RECORD` fingerprint. +- Conda native libraries, system packages and GPU drivers are context, not + installed by the schema-1 executor. +- Source builds, editable installs and arbitrary direct URLs cannot currently + provide the same byte-for-byte guarantees as locked wheels. +- Index authentication is machine configuration; portable TOML never embeds + secrets. + +These boundaries are reported explicitly. They are never converted into an +unplanned download, fallback route, upload, or silent partial environment. diff --git a/docs/terminal-experience.md b/docs/terminal-experience.md new file mode 100644 index 0000000..e185558 --- /dev/null +++ b/docs/terminal-experience.md @@ -0,0 +1,55 @@ +# Terminal experience + +pload uses one interaction language across installation, environment creation, +planning, configuration, and destructive actions. The goal is predictable +behaviour, not decoration. + +## Interaction model + +- A normal command prints a compact result and exits. It never opens a full + screen interface unexpectedly. +- A command that needs a choice uses `↑`/`↓` and Enter. The active row is + prefixed with `❯`; the recommended value is selected initially. +- Package planning keeps navigation commands out of the source list. Use the + visible first-letter shortcuts: `p` previous, `n` next, `u` undo, `r` reset, + `s` save, `a` apply, and `q` quit. +- `pload new` is guided when no creation options are supplied. Every guided + operation also has flags for scripts and automation. +- When guided creation includes packages, it asks once whether pload should + choose the fastest source or the user wants custom per-package choices. The + custom list contains only real cache/index routes. Transitive dependencies + remain automatic, and one failed request does not prevent later requests. +- Destructive actions use a conventional `y/N` confirmation. `--yes` is the + explicit non-interactive alternative. +- `Ctrl-C` cancels without a traceback. + +## Visual language + +| Treatment | Meaning | +| --- | --- | +| cyan | current focus, headings, and resources | +| green / `✓` | completed successfully | +| yellow / `!` | warning or required attention | +| red / `error:` | command failure | +| dim text | paths, provenance, and secondary explanation | + +Tables are compact and border-light so rows remain readable at normal terminal +widths. Progress animation appears only on a TTY. `NO_COLOR=1` disables colour, +and `PLOAD_NO_PROGRESS=1` replaces spinners with plain text. + +## Output contract + +Primary results remain suitable for copying or piping. JSON modes write only +JSON to stdout. Progress and diagnostics use stderr where a command exposes +machine-readable output. Redirected input uses explicit flags or numbered +fallbacks instead of emitting terminal control sequences. + +## Design references + +The design follows [Atuin's compact keyboard-driven interaction](https://docs.atuin.sh/main/reference/search/), +[uv's concise help and TTY-aware progress](https://docs.astral.sh/uv/getting-started/help/), +[GitHub CLI's explicit prompt and accessibility controls](https://cli.github.com/manual/gh_help_environment), +[Cargo's automatic terminal capability detection](https://doc.rust-lang.org/cargo/reference/config.html#termprogresswhen), +and the human-first principles of the [Command Line Interface Guidelines](https://clig.dev/). +pload intentionally uses these conventions without turning every command into +a full-screen TUI. diff --git a/pyproject.toml b/pyproject.toml index 37322c8..3934127 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pload" -version = "1.0.0" +version = "1.2.0b1" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" @@ -12,7 +12,7 @@ license = { file = "LICENSE" } authors = [{ name = "Yunming Hu", email = "hugonelsonm3@gmail.com" }] keywords = ["python", "venv", "virtual-environment", "pyenv"] classifiers = [ - "Development Status :: 5 - Production/Stable", + "Development Status :: 3 - Alpha", "Environment :: Console", "Operating System :: MacOS", "Operating System :: Microsoft :: Windows", @@ -20,7 +20,13 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", ] -dependencies = ["rich>=13.7,<15"] +dependencies = [ + "packaging>=23.2,<27", + "questionary>=2.0.1,<3", + "rich>=13.7,<15", + "resolvelib>=1.1,<1.2", + "tomli>=2; python_version < '3.11'", +] [project.scripts] pload = "pload.cli:main" diff --git a/src/pload/__init__.py b/src/pload/__init__.py index 5becc17..4834673 100644 --- a/src/pload/__init__.py +++ b/src/pload/__init__.py @@ -1 +1 @@ -__version__ = "1.0.0" +__version__ = "1.2.0b1" diff --git a/src/pload/cli.py b/src/pload/cli.py index 048d8b3..58565f7 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -4,13 +4,14 @@ import sys from pathlib import Path +from questionary import Choice from rich import box -from rich.console import Console from rich.panel import Panel +from rich.progress import BarColumn, Progress, TextColumn from rich.table import Table from rich.text import Text -from pload import __version__ +from pload import __version__, ui from pload.display import print_environment_table, print_python_table from pload.errors import PloadError from pload.help_content import render_detailed_help @@ -28,6 +29,12 @@ "shell-init": ("shell",), "python": ("py",), "config": ("cfg",), + "describe": (), + "lock": (), + "plan": (), + "apply": (), + "repo": (), + "remote": (), } PYTHON_ALIASES = { "install": (), @@ -49,7 +56,7 @@ def build_parser(): - parser = argparse.ArgumentParser( + parser = ui.ArgumentParser( prog="pload", description=( "Create, activate, and remove Python virtual environments without tying " @@ -276,12 +283,95 @@ def build_parser(): ) config.add_argument("key", nargs="?", help="setting name for `config set`") config.add_argument("value", nargs="?", help="new value for `config set`") + describe = subparsers.add_parser( + "describe", description="Describe an existing environment as one portable configuration.", + help="create pload.toml from an existing environment", + ) + describe.add_argument("source", help="environment ID, name, path, or Python executable") + describe.add_argument("--output", "-o", default="pload.toml", + help="configuration file to write (default: pload.toml)") + describe.add_argument("--name", "-n", help="logical environment name") + describe.add_argument("--mode", "-m", choices=["exact", "compatible"], default="exact", + help="lock exact artifacts or permit re-resolution (default: exact)") + describe.add_argument("--repository", "-r", + help="artifact repository; defaults to the first configured local/SSH repo") + describe.add_argument( + "--source", "-s", dest="package_sources", action="append", default=[], + metavar="PACKAGE=URL", + help="package-specific index; repeat for packages such as CUDA-enabled torch", + ) + lock = subparsers.add_parser( + "lock", description="Resolve dependencies and write the managed environment lock.", + help="resolve and write .pload_lock.toml", + ) + lock.add_argument("file", nargs="?", default="pload.toml", + help="environment configuration (default: pload.toml)") + lock.add_argument("--offline", "-o", action="store_true", + help="use only already available resolver resources") + plan = subparsers.add_parser( + "plan", description="Compare a declarative environment with all available resources.", + help="explain how pload would satisfy a configuration", + ) + plan.add_argument("file", nargs="?", default="pload.toml", + help="environment configuration (default: pload.toml)") + plan.add_argument("--offline", "-o", action="store_true", + help="plan using only resources available without the internet") + plan.add_argument("--json", "-j", action="store_true", help="emit machine-readable JSON") + plan.add_argument( + "--no-ui", "-N", action="store_true", + help="save recommended measured routes without opening the interactive chooser", + ) + apply = subparsers.add_parser( + "apply", description="Materialize the desired environment from available resources.", + help="create or verify an environment from pload.toml", + ) + apply.add_argument("file", nargs="?", default="pload.toml", + help="environment configuration (default: pload.toml)") + apply_target = apply.add_mutually_exclusive_group() + apply_target.add_argument( + "--name", "-n", help="override the managed environment name" + ) + apply_target.add_argument( + "--local", "-l", action="store_true", + help="create .venv beside the configuration file", + ) + apply.add_argument("--offline", "-o", action="store_true", + help="forbid internet access; configured local/SSH repositories remain usable") + repo = subparsers.add_parser("repo", help="manage resource providers", + description="Manage local and SSH artifact providers.") + actions = repo.add_subparsers(dest="repo_command", required=True) + add = actions.add_parser("add", description="Save a repository location; no credentials stored.") + add.add_argument("name", help="repository nickname") + add.add_argument("location", help="directory or HOST:/absolute/path") + add.add_argument("--type", "-t", choices=["local", "ssh"], default="local", + help="transport (default: local)") + actions.add_parser("list", aliases=["ls"], description="Show configured repositories.") + remove = actions.add_parser("remove", description="Remove configuration, keeping all remote files.") + remove.add_argument("name", help="repository nickname") + remote = subparsers.add_parser( + "remote", help="explicitly back up installed package artifacts", + description="Search or store package artifacts in configured local/SSH repositories.", + ) + remote_actions = remote.add_subparsers(dest="remote_command", required=True) + remote_add = remote_actions.add_parser( + "add", description="Back up one installed package wheel; never runs during apply.", + ) + remote_add.add_argument("package", help="installed package name or exact NAME==VERSION") + remote_add.add_argument("--from", "-f", dest="environment", default=".", + help="source environment ID, name, or path (default: .)") + remote_add.add_argument("--remote", "-r", dest="repository", + help="configured repository name (default: first configured)") + remote_add.add_argument("--index", "-i", help="index used only if no original wheel is cached") return parser def add_packages(parser): parser.add_argument("--channel", "-c", help="Python package index URL") parser.add_argument("--requirements", "-r", nargs="+", help="packages to install") + parser.add_argument( + "--package-strategy", "-s", choices=["auto", "custom"], + help="choose package sources automatically or interactively (default: auto)", + ) def _set_config_value(config, key, value): @@ -393,6 +483,245 @@ def _global_options_table(root_parser): return table +def print_declarative_plan(plan): + console = ui.console() + if plan["lock"]["required"]: + console.print( + f"[bold yellow]Lock: {plan['lock']['status']}[/] · metadata was unavailable " + "offline; run [bold green]pload plan[/] with network access" + ) + python = plan["python"] + ui.heading( + f"Plan · {plan['name']}", + f"Python {python['method']} · {python['status']} · " + f"{plan.get('planning_mode', 'interactive')} planning", + output=console, + ) + console.print(f"[dim]{python['location']}[/]", overflow="ellipsis", no_wrap=True) + table = ui.table( + ("PACKAGE", "bold", {"width": 15, "overflow": "ellipsis", "no_wrap": True}), + ("VERSION", "yellow", {"width": 10, "no_wrap": True}), + ("SOURCE", "cyan", {"width": 12, "overflow": "ellipsis", "no_wrap": True}), + ("STATUS", "", {"width": 10, "no_wrap": True}), + ("RESOURCE", "dim", {"min_width": 14, "ratio": 1, + "overflow": "ellipsis", "no_wrap": True}), + expand=True, + ) + direct_packages = [ + package for package in plan["packages"] if package.get("direct", True) + ] + transitive_count = len(plan["packages"]) - len(direct_packages) + for package in direct_packages: + selected = package["selected"] + if selected: + table.add_row(package["name"], package["version"], selected["method"], + selected["status"], selected["location"]) + else: + rejected = package.get("rejections", []) + if rejected: + table.add_row(package["name"], package["version"], "—", "incompatible", + rejected[0]["artifact"]) + else: + table.add_row(package["name"], package["version"], "—", "unavailable", + "No resource satisfies the configuration") + console.print(table) + if transitive_count: + console.print( + f"[dim]+ {transitive_count} transitive dependencies planned " + "automatically using their lowest-cost valid routes[/]" + ) + if plan.get("saved"): + console.print( + "[dim]Saved acquisition choices:[/] " + f"[cyan]{plan['plan_path']}[/]" + ) + else: + console.print("[dim]Acquisition choices were not saved.[/]") + + +def _progress_size(value): + size = float(value or 0) + for unit in ("B", "KiB", "MiB", "GiB"): + if size < 1024 or unit == "GiB": + return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + + +class ApplyProgress: + """Persistent Rich progress rows with a plain-text CI fallback.""" + + def __init__(self, output=None): + self.console = output or ui.console() + self.interactive = self.console.is_terminal + self.tasks = {} + self.details = {} + self.progress = Progress( + TextColumn("{task.fields[tag]}", style="bold cyan", markup=False), + TextColumn("{task.fields[package]}", style="bold white", markup=False), + TextColumn("[", style="dim", markup=False), + BarColumn( + bar_width=None, + style="grey23", + complete_style="green", + finished_style="green", + ), + TextColumn("]", style="dim", markup=False), + TextColumn("{task.fields[amount]}", style="cyan", markup=False), + console=self.console, + transient=False, + expand=True, + refresh_per_second=30, + ) + + def __enter__(self): + if self.interactive: + self.progress.start() + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self.interactive: + self.progress.stop() + + def __call__(self, message): + parts = str(message).split("\t") + kind = parts[0].split(" ", 1)[0] + if kind == "PACKAGE": + _, counter, method, package, size = parts + total = int(size) or 1 + detail = { + "tag": f"[{method}]", + "package": package, + "counter": counter, + "method": method, + "total": total, + "last_plain_received": 0, + } + self.details[package] = detail + if self.interactive: + self.tasks[package] = self.progress.add_task( + package, total=total, tag=detail["tag"], package=package, + amount="", status="waiting", + ) + elif kind == "SOURCE": + _, package, status = parts + self.details[package]["status"] = status + if self.interactive: + self.progress.update(self.tasks[package], status=status) + elif kind == "DOWNLOAD": + _, package, received, expected = parts + received, expected = int(received), int(expected) + detail = self.details[package] + total = expected or detail["total"] + amount = _progress_size(received) + if total > 1: + amount += f" / {_progress_size(total)}" + if self.interactive: + self.progress.update( + self.tasks[package], completed=received, total=total, + amount=amount, status="downloading", + ) + else: + final = expected and received >= expected + if (final or received - detail["last_plain_received"] + >= 8 * 1024 * 1024): + self.console.print(Text( + f"{detail['tag']:<24} {package} {amount}" + )) + detail["last_plain_received"] = received + elif kind == "PACKAGE_DONE": + _, package, status = parts + detail = self.details[package] + if self.interactive: + task = self.progress.tasks[self.tasks[package]] + self.progress.update( + self.tasks[package], completed=task.total, + amount=task.fields.get("amount", ""), status=status, + ) + elif "status" not in detail: + self.console.print(Text(f"{detail['tag']:<24} {package} {status}")) + elif kind == "STAGE": + _, _, detail = str(message).partition(" ") + self.console.print(Text(f"▶ {detail}", style="bold cyan")) + elif kind == "DONE": + _, _, detail = str(message).partition(" ") + ui.success(detail, output=self.console) + else: + self.console.print(message) + + +def choose_declarative_routes(plan): + """Choose direct-package routes with familiar cursor-key menus.""" + from pload.resource_plan import PlanSelection + + console = ui.console() + selection = PlanSelection(plan) + decision_indices = selection.decision_indices() + if not decision_indices: + return "save" + cursor = 0 + while True: + packages = plan["packages"] + package_index = decision_indices[cursor] + package = packages[package_index] + routes = selection.routes(package_index) + ui.heading( + f"Choose source · {package['name']}=={package['version']}", + f"Direct requirement {cursor + 1}/{len(decision_indices)} · " + f"{len(packages) - len(decision_indices)} dependencies handled automatically", + output=console, + ) + choices = [] + for index, route in enumerate(routes, 1): + measurement = route.get("measurement", "not measured") + recommended = " recommended" if index == 1 else "" + choices.append(Choice( + f"{route['method']:<18} {measurement:<20} " + f"{route['location']}{recommended}", + value=("route", index - 1), + )) + if not choices: + ui.warning(f"No valid source is available for {package['name']}") + return "quit" + actions = { + "p": ("previous", None), + "n": ("next", None), + "u": ("undo", None), + "r": ("reset", None), + "s": ("save", None), + "a": ("apply", None), + "q": ("quit", None), + } + console.print( + "[dim]p previous · n next · u undo · r reset · " + "s save · a apply · q quit[/]" + ) + command, value = ui.select( + "Select an acquisition source", + choices, + default=("route", 0) if routes else ("quit", None), + instruction=ui.navigation_hint("choose"), + hotkeys=actions, + ) + if command == "route": + selection.select(package_index, value) + if cursor < len(decision_indices) - 1: + cursor += 1 + elif command == "next": + cursor = min(cursor + 1, len(decision_indices) - 1) + elif command == "previous": + cursor = max(cursor - 1, 0) + elif command == "undo": + selection.undo() + elif command == "reset": + selection.reset() + elif command == "save": + return "save" + elif command == "apply": + return "apply" + elif command == "quit": + return "quit" + + def _print_spaced_section(console, renderable): """Give a standalone section consistent visual breathing room.""" console.print() @@ -412,10 +741,11 @@ def _print_arrow_table(console, title, table): def _brief_help(parser, command_path, root_parser=None): - console = Console(highlight=False) + console = ui.console() title = f"pload {__version__}" if not command_path else "pload " + " ".join(command_path) description = parser.description or "Command-line help" usage = " ".join(parser.format_usage().split()) + usage = re.sub(r"\{[^}]+\} \.\.\.", "", usage) if command_path and usage.startswith(f"usage: {parser.prog}"): usage = usage.replace( f"usage: {parser.prog}", @@ -463,10 +793,16 @@ def _brief_help(parser, command_path, root_parser=None): "path": "Print an environment path", "config": "Show effective configuration", "shell-init": "Print shell activation integration", + "describe": "Write one portable configuration for an existing environment", + "lock": "Resolve dependencies into .pload_lock.toml", + "plan": "Compare a configuration with available resources", + "apply": "Materialize the environment declared by pload.toml", + "repo": "Manage local and SSH artifact providers", + "remote": "Explicitly back up installed package artifacts", } - for command in ("new", "init", "list", "rm", "python", "path", "config", "shell-init"): + for command, summary in summaries.items(): aliases = COMMAND_ALIASES[command] - table.add_row(command, ", ".join(aliases) if aliases else "—", summaries[command]) + table.add_row(command, ", ".join(aliases) if aliases else "—", summary) console.print(table) console.print("[bold cyan]Quick start[/]") console.print(" [green]pload new[/] [dim]# guided creation[/]") @@ -525,22 +861,18 @@ def render_help(argv): def render_landing(): """Show the no-argument welcome screen and the smallest useful command tour.""" - console = Console(highlight=False) - art = Text( - """ ____ _ _ - | _ \\| | ___ __ _ __| | - | |_) | |/ _ \\ / _` |/ _` | - | __/| | (_) | (_| | (_| | - |_| |_|\\___/ \\__,_|\\__,_|""", - style="bold cyan", - ) - console.print(art) + console = ui.console() + ui.logo(output=console) console.print(f"[bold white]pload {__version__}[/] [dim]Python environments, kept simple.[/]") table = Table(box=box.SIMPLE, header_style="bold cyan", show_edge=False) table.add_column("COMMAND", style="bold green", no_wrap=True) table.add_column("WHAT IT DOES") table.add_row("pload cfg", "Show where pload stores its data") table.add_row("pload python list", "Find every usable Python interpreter") + table.add_row("pload describe v1", "Describe an environment in pload.toml") + table.add_row("pload plan", "Resolve metadata and choose package routes") + table.add_row("pload apply", "Execute exactly the saved package routes") + table.add_row("pload remote add torch -f v1 -r lab", "Explicitly back up one wheel") table.add_row("pload new", "Open guided environment creation") table.add_row("pload new -n data -v 3.12", "Create a named virtual environment") table.add_row("pload list", "List environments, IDs, and descriptions") @@ -554,7 +886,7 @@ def shell_script(shell): if shell in {"bash", "zsh"}: return r'''pload() { case "${1:-}" in - ""|new|init|i|rm|remove|del|delete|list|ls|path|p|shell-init|shell|python|py|config|cfg|-*) + ""|new|init|i|rm|remove|del|delete|list|ls|path|p|shell-init|shell|python|py|config|cfg|describe|lock|plan|apply|repo|-*) command pload "$@" ;; *) @@ -567,7 +899,7 @@ def shell_script(shell): if shell == "fish": return r'''function pload switch "$argv[1]" - case '' new init i rm remove del delete list ls path p shell-init shell python py config cfg '-*' + case '' new init i rm remove del delete list ls path p shell-init shell python py config cfg describe lock plan apply repo '-*' command pload $argv case '*' set -l name . @@ -581,7 +913,7 @@ def shell_script(shell): end''' return r'''function pload { param([Parameter(ValueFromRemainingArguments = $true)][string[]]$PloadArgs) - $commands = @('new', 'init', 'i', 'rm', 'remove', 'del', 'delete', 'list', 'ls', 'path', 'p', 'shell-init', 'shell', 'python', 'py', 'config', 'cfg', '-h', '--help', '--version') + $commands = @('new', 'init', 'i', 'rm', 'remove', 'del', 'delete', 'list', 'ls', 'path', 'p', 'shell-init', 'shell', 'python', 'py', 'config', 'cfg', 'describe', 'lock', 'plan', 'apply', 'repo', '-h', '--help', '--version') $backend = Get-Command -Name @('pload.exe', 'pload.cmd') -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1 if (-not $backend) { throw 'pload executable not found on PATH' } if ($PloadArgs.Count -eq 0) { @@ -600,36 +932,60 @@ def shell_script(shell): def _guided_new(config): """Collect useful creation choices when ``pload new`` is run by itself.""" - console = Console(highlight=False) - console.print(Panel.fit( - "Choose an interpreter and an optional name. Press Enter to accept defaults.", - title="[bold cyan]Guided environment creation[/]", - border_style="blue", - )) + console = ui.console() + ui.heading( + "Create an environment", + "Review each value, then create. Nothing is changed until the final step.", + output=console, + ) runtimes = PythonManager(config).discover() if runtimes: - console.print("[dim]Use a Python ID, alias, version, or interpreter path:[/]") - print_python_table(runtimes) + runtime_choices = [ + Choice( + f"{runtime.id or '—':<5} {runtime.version:<10} " + f"{runtime.source:<10} {runtime.path}", + value=runtime.id or runtime.alias or str(runtime.path), + ) + for runtime in runtimes + ] + runtime_choices.append(Choice("Enter a version or path…", value="__custom__")) + version = ui.select( + "Python interpreter", + runtime_choices, + default=runtime_choices[0].value, + ) + if version == "__custom__": + version = ui.text("Python version or executable path", default="current") else: console.print("[yellow]No Python was discovered; the current Python is the default.[/]") - - def ask(label, default=""): - suffix = f" [{default}]" if default else "" - value = input(f"{label}{suffix}: ").strip() - return value or default - - version = ask("Python ID, alias, version, or path", "current") + version = "current" _, suggested_name = config.resolve_venv_path(version=version) - name = ask("Environment name (blank generates one)") - if not name: - console.print(f"[dim]Generated name: [bold]{suggested_name}[/][/dim]") - description = ask("Description (optional)") - packages = ask("Packages, separated by spaces (optional)") + name = ui.text("Environment name", default=suggested_name) + description = ui.text("Description (optional)") + packages = ui.text("Packages, separated by spaces (optional)") + requirements = packages.split() or None + package_strategy = None + if requirements: + package_strategy = ui.select( + "Package source strategy", + [ + Choice( + "Automatic prefer compatible cache, then the configured index", + value="auto", + ), + Choice( + "Custom choose a source for each package", + value="custom", + ), + ], + default="auto", + ) return { "python_version": version, - "name": name or None, + "name": name, "description": description or None, - "requirements": packages.split() or None, + "requirements": requirements, + "package_strategy": package_strategy, } @@ -647,20 +1003,113 @@ def run(argv=None): venvs = VenvManager(config) dependencies = DependencyManager(config) + if command in {"describe", "lock", "plan", "apply", "repo", "remote"}: + if command == "repo": + from pload.snapshots import RepositoryManager + + repositories = RepositoryManager(config) + if args.repo_command == "add": + repositories.add(args.name, args.location, args.type) + elif args.repo_command == "remove": + repositories.remove(args.name) + else: + print(json.dumps(repositories.repositories(), indent=2)) + return 0 + from pload.declarative import DeclarativeEnvironmentManager + + manager = DeclarativeEnvironmentManager(config) + if command == "remote": + result = manager.publish_package( + args.package, args.environment, args.repository, args.index, + ) + ui.success( + f"Backed up {result['package']}=={result['version']} " + f"to {result['repository']}", + detail=f"{result['filename']} · sha256:{result['sha256']}", + ) + elif command == "describe": + progress_console = ui.console() + print(manager.describe( + args.source, args.output, args.name, args.mode, args.repository, + args.package_sources, + progress=lambda message: progress_console.print( + Text("• " + message, style="cyan") + ), + )) + elif command == "lock": + with ui.status("Reading configuration…") as update_status: + result = manager.lock( + args.file, args.offline, + progress=lambda message: update_status(f"{message}…"), + ) + count = len(result["resolved"]) + action = "Wrote" if result["updated"] else "Reused" + ui.success( + f"{action} {count} locked packages", + detail=str(result["path"]), + ) + elif command == "plan": + if args.json: + plan = manager.plan(args.file, args.offline) + print(json.dumps(plan, ensure_ascii=False, indent=2)) + else: + with ui.status("Reading configuration and available resources…") as update_status: + plan = manager.plan( + args.file, args.offline, + progress=lambda message: update_status(f"{message}…"), + ) + action = "save" + chooser_console = ui.console() + if (chooser_console.is_terminal and not args.no_ui + and plan.get("planning_mode") != "auto"): + action = choose_declarative_routes(plan) + if action != "quit": + plan["plan_path"] = str(manager.save_plan(args.file, plan)) + plan["saved"] = True + print_declarative_plan(plan) + if action == "apply": + with ApplyProgress(chooser_console) as apply_progress: + restored = manager.apply( + args.file, offline=args.offline, + progress=apply_progress, + ) + chooser_console.print(restored) + else: + apply_console = ui.console() + saved_plan = manager.saved_plan(args.file) + print_declarative_plan(saved_plan) + target = None + if args.local: + target = Path(args.file).expanduser().resolve().parent / ".venv" + with ApplyProgress(apply_console) as apply_progress: + restored = manager.apply( + args.file, args.name, args.offline, + target=target, + progress=apply_progress, + ) + print(restored) + return 0 + if command == "new": - if not any((args.python_version, args.name, args.path, args.description, args.requirements, args.channel)): + if not any(( + args.python_version, args.name, args.path, args.description, + args.requirements, args.channel, args.package_strategy, + )): guided = _guided_new(config) args.python_version = guided["python_version"] args.name = guided["name"] args.description = guided["description"] args.requirements = guided["requirements"] + args.package_strategy = guided["package_strategy"] path = venvs.create_venv( version=args.python_version, target=args.path, name=args.name, description=args.description, ) - dependencies.install_dependencies(path, args.requirements, args.channel) + dependencies.install_dependencies( + path, args.requirements, args.channel, args.package_strategy or "auto", + ) return 0 if command == "init": @@ -673,7 +1122,9 @@ def run(argv=None): target=args.venv_dir, description=args.description, ) - dependencies.install_dependencies(path, args.requirements, args.channel) + dependencies.install_dependencies( + path, args.requirements, args.channel, args.package_strategy or "auto", + ) return 0 if command == "rm": @@ -684,11 +1135,15 @@ def run(argv=None): names = list(dict.fromkeys(names)) if not names: raise PloadError("no environments selected") + if not args.yes: + ui.heading("Remove environments", "\n".join(f" {name}" for name in names)) + if not ui.confirm( + f"Remove {len(names)} environment{'s' if len(names) != 1 else ''}?", + default=False, + ): + ui.warning("Removal cancelled") + return 0 for name in names: - if not args.yes: - confirmation = input(f"Remove {name!r}? Type its name to confirm: ") - if confirmation != name: - raise PloadError("removal cancelled") venvs.remove_venv(name, project_dir=args.project_dir) return 0 @@ -741,7 +1196,7 @@ def run(argv=None): if not args.key or args.value is None: raise PloadError("usage: pload cfg set SETTING VALUE") path = _set_config_value(config, args.key, args.value) - print(f"[*] Updated configuration: {path}") + ui.success("Updated configuration", detail=str(path)) return 0 effective = dict(config.settings) effective.update({ @@ -759,8 +1214,11 @@ def run(argv=None): def main(argv=None): try: return run(argv) - except (PloadError, PythonNotFoundError, re.error) as exc: - print(f"pload: error: {exc}", file=sys.stderr) + except (KeyboardInterrupt, EOFError): + ui.warning("Cancelled") + return 130 + except (PloadError, PythonNotFoundError, re.error, OSError) as exc: + ui.error(str(exc)) return 1 diff --git a/src/pload/declarative.py b/src/pload/declarative.py new file mode 100644 index 0000000..0556afe --- /dev/null +++ b/src/pload/declarative.py @@ -0,0 +1,2227 @@ +"""Declarative environment manifests and resource-driven materialization.""" + +import hashlib +import json +import os +import platform +import re +import shlex +import shutil +import socket +import subprocess +import sys +import tempfile +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + +from packaging.requirements import InvalidRequirement, Requirement +from packaging.tags import compatible_tags, cpython_tags, platform_tags +from packaging.utils import InvalidWheelFilename, parse_wheel_filename + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - exercised by the Python 3.8 CI job + import tomli as tomllib + +from pload.errors import PloadError +from pload.managers.platform import PythonNotFoundError +from pload.managers.pyversion import PythonManager +from pload.managers.venv import VenvManager +from pload.metadata_resolver import resolve_metadata +from pload.resource_plan import file_digest, load_plan, plan_path, write_plan +from pload.snapshots import ( + RepositoryManager, + digest, + execute, + probe, + probe_resources, + public_index, + safe_name, +) + +PIN = re.compile(r"([A-Za-z0-9][A-Za-z0-9_.-]*)==([A-Za-z0-9][A-Za-z0-9_.+!-]*)") +WHEEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+!-]*\.whl") +SHA256 = re.compile(r"[0-9a-f]{64}") +DOWNLOAD_CHUNK_SIZE = 16 * 1024 +DOWNLOAD_PROGRESS_INTERVAL = 1 / 30 + + +def _human_size(value): + size = float(value or 0) + units = ("B", "KiB", "MiB", "GiB") + for unit in units: + if size < 1024 or unit == units[-1]: + return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + + +def _duration(value): + return f"{value * 1000:.0f} ms" if value < 1 else f"{value:.2f} s" + + +def _report_transfer(progress, label, received, total): + if progress: + progress(f"DOWNLOAD\t{label}\t{received}\t{total}") + + +def _digest_with_progress(path, label, progress=None): + """Hash a local artifact while reporting bytes actually verified.""" + path = Path(path) + total = path.stat().st_size + checksum = hashlib.sha256() + received = 0 + last_reported_at = time.monotonic() + with path.open("rb") as stream: + while True: + chunk = stream.read(1024 * 1024) + if not chunk: + break + checksum.update(chunk) + received += len(chunk) + now = time.monotonic() + if now - last_reported_at >= DOWNLOAD_PROGRESS_INTERVAL: + _report_transfer(progress, label, received, total) + last_reported_at = now + _report_transfer(progress, label, received, total) + return checksum.hexdigest() + + +def _copy_with_progress(source, destination, label, progress=None): + """Copy and hash a local artifact while reporting real processed bytes.""" + source = Path(source) + total = source.stat().st_size + checksum = hashlib.sha256() + received = 0 + last_reported_at = time.monotonic() + with source.open("rb") as input_stream, Path(destination).open("wb") as output: + while True: + chunk = input_stream.read(1024 * 1024) + if not chunk: + break + output.write(chunk) + checksum.update(chunk) + received += len(chunk) + now = time.monotonic() + if now - last_reported_at >= DOWNLOAD_PROGRESS_INTERVAL: + _report_transfer(progress, label, received, total) + last_reported_at = now + _report_transfer(progress, label, received, total) + return checksum.hexdigest() + + +def normalized_name(value): + return re.sub(r"[-_.]+", "-", value).lower() + + +def _parse_dependency(value): + if not isinstance(value, str) or not value.strip(): + raise PloadError("environment.dependencies must contain package requirements") + try: + requirement = Requirement(value) + except InvalidRequirement as exc: + raise PloadError(f"invalid dependency requirement: {value}") from exc + if requirement.url: + raise PloadError("direct-URL dependencies are not portable; declare an index source") + if requirement.marker: + raise PloadError("dependency environment markers are not supported in schema 1") + return requirement + + +def _normalized_machine(value): + aliases = { + "amd64": "x86_64", "x64": "x86_64", + "aarch64": "arm64", "arm64": "arm64", + } + normalized = str(value).lower() + return aliases.get(normalized, normalized) + + +def _target_wheel_tags(environment): + """Return wheel tags installable on the requested runtime on this host.""" + if (environment.get("system") + and environment["system"].lower() != platform.system().lower()): + return set() + if (environment.get("machine") + and _normalized_machine(environment["machine"]) + != _normalized_machine(platform.machine())): + return set() + try: + version = tuple(int(part) for part in environment["python"].split(".")[:2]) + except (KeyError, TypeError, ValueError): + return set() + if len(version) != 2: + return set() + platforms = list(platform_tags()) + implementation = environment.get("implementation", "cpython").lower() + interpreter = f"cp{version[0]}{version[1]}" if implementation == "cpython" else None + result = set(compatible_tags( + python_version=version, interpreter=interpreter, platforms=platforms, + )) + if implementation == "cpython": + result.update(cpython_tags(python_version=version, platforms=platforms)) + return result + + +def _artifact_is_compatible(artifact, supported_tags): + try: + wheel_tags = parse_wheel_filename(artifact["filename"])[3] + except (InvalidWheelFilename, KeyError, TypeError): + return False + return bool(wheel_tags.intersection(supported_tags)) + + +def manifest_digest(data): + encoded = json.dumps(data, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _quoted(value): + return json.dumps(str(value), ensure_ascii=False) + + +def _array(values): + return "[" + ", ".join(_quoted(item) for item in values) + "]" + + +def dump_manifest(data): + """Serialize only the user-authored environment declaration.""" + environment = data["environment"] + policy = data["policy"] + lines = [ + f"schema = {data['schema']}", + f"name = {_quoted(data['name'])}", + "", + "[environment]", + f"python = {_quoted(environment['python'])}", + f"implementation = {_quoted(environment.get('implementation', 'cpython'))}", + f"system = {_quoted(environment.get('system', ''))}", + f"machine = {_quoted(environment.get('machine', ''))}", + f"dependencies = {_array(environment.get('dependencies', []))}", + "", + "[policy]", + f"reproducibility = {_quoted(policy.get('reproducibility', 'exact'))}", + f"network = {_quoted(policy.get('network', 'allow'))}", + f"source_build = {_quoted(policy.get('source_build', 'fallback'))}", + "publish_missing_artifacts = " + + ("true" if policy.get("publish_missing_artifacts", False) else "false"), + ] + repositories = policy.get("repositories", []) + if repositories: + lines.append(f"repositories = {_array(repositories)}") + planning = data.get("plan") + if planning: + lines += ["", "[plan]", f"mode = {_quoted(planning.get('mode', 'interactive'))}"] + for package_name, route in sorted(planning.get("packages", {}).items()): + lines += ["", f"[plan.packages.{_quoted(package_name)}]"] + lines.append(f"method = {_quoted(route['method'])}") + if route.get("location"): + lines.append(f"location = {_quoted(route['location'])}") + capabilities = data.get("capabilities", {}) + if capabilities: + lines += ["", "[capabilities]"] + for key, value in sorted(capabilities.items()): + lines.append(f"{key} = {_array(value)}") + for name, source in sorted(data.get("sources", {}).items()): + lines += [ + "", + f"[sources.{_quoted(name)}]", + f"kind = {_quoted(source['kind'])}", + f"url = {_quoted(source['url'])}", + ] + for name, repository in sorted(data.get("repositories", {}).items()): + lines += [ + "", + f"[repositories.{_quoted(name)}]", + f"kind = {_quoted(repository['kind'])}", + f"location = {_quoted(repository['location'])}", + ] + return "\n".join(lines) + "\n" + + +def lock_path(manifest_path): + """Return the pload-managed lock beside a user configuration.""" + return Path(manifest_path).parent / ".pload_lock.toml" + + +def configuration_digest(data): + return hashlib.sha256(dump_manifest(data).encode("utf-8")).hexdigest() + + +def dump_lock(manifest_path, data): + """Serialize resolver output separately from the user configuration.""" + lines = [ + "schema = 1", + f"configuration = {_quoted(Path(manifest_path).name)}", + f"configuration_sha256 = {_quoted(configuration_digest(data))}", + "", + "[environment]", + f"dependencies = {_array(data.get('resolved_dependencies', []))}", + ] + for package in data.get("package", []): + lines += [ + "", + "[[package]]", + f"name = {_quoted(package['name'])}", + f"version = {_quoted(package['version'])}", + ] + if package.get("sources"): + lines.append(f"sources = {_array(package['sources'])}") + for artifact in package.get("artifact", []): + lines += [ + "", + "[[package.artifact]]", + f"filename = {_quoted(artifact['filename'])}", + f"sha256 = {_quoted(artifact['sha256'])}", + f"tags = {_array(artifact.get('tags', []))}", + f"repositories = {_array(artifact.get('repositories', []))}", + ] + if artifact.get("url"): + lines.append(f"url = {_quoted(artifact['url'])}") + if artifact.get("size") is not None: + lines.append(f"size = {int(artifact['size'])}") + if artifact.get("metadata_sha256"): + lines.append( + f"metadata_sha256 = {_quoted(artifact['metadata_sha256'])}" + ) + return "\n".join(lines) + "\n" + + +def load_lock(manifest_path, configuration): + """Load a current sidecar lock; stale locks are ignored and regenerated.""" + path = lock_path(manifest_path) + if not path.is_file(): + return path, None, False + try: + with path.open("rb") as stream: + lock = tomllib.load(stream) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise PloadError(f"cannot read environment lock {path}: {exc}") from exc + if lock.get("schema") != 1: + raise PloadError(f"unsupported environment lock schema in {path}") + packages = lock.setdefault("package", []) + environment = lock.setdefault("environment", {}) + environment.setdefault("dependencies", []) + for package in packages: + package.setdefault("sources", []) + package.setdefault("artifact", []) + current = ( + lock.get("configuration") == Path(manifest_path).name + and lock.get("configuration_sha256") == configuration_digest(configuration) + ) + if current: + combined = dict(configuration) + combined["package"] = packages + validate_manifest(combined) + return path, lock, current + + +def load_manifest(path): + path = Path(path).expanduser().resolve() + try: + with path.open("rb") as stream: + data = tomllib.load(stream) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise PloadError(f"cannot read environment configuration {path}: {exc}") from exc + data.setdefault("sources", {}) + data.setdefault("repositories", {}) + data.setdefault("capabilities", {}) + data.setdefault("package", []) + environment = data.setdefault("environment", {}) + environment.setdefault("implementation", "cpython") + environment.setdefault("system", "") + environment.setdefault("machine", "") + environment.setdefault("dependencies", []) + policy = data.get("policy") + if isinstance(policy, dict): + policy.setdefault("reproducibility", "exact") + policy.setdefault("network", "allow") + policy.setdefault("source_build", "fallback") + policy.setdefault("publish_missing_artifacts", False) + policy.setdefault("repositories", []) + for package in data["package"]: + package.setdefault("sources", []) + package.setdefault("artifact", []) + validate_manifest(data) + return path, data + + +def validate_manifest(data): + if not isinstance(data, dict) or data.get("schema") != 1: + raise PloadError("unsupported environment configuration schema") + safe_name(data.get("name")) + environment = data.get("environment") + policy = data.get("policy") + if not isinstance(environment, dict) or not environment.get("python"): + raise PloadError("environment.python is required") + if not isinstance(policy, dict): + raise PloadError("a [policy] section is required") + if policy.get("reproducibility", "exact") not in {"exact", "compatible"}: + raise PloadError("policy.reproducibility must be exact or compatible") + if policy.get("network", "allow") not in {"allow", "offline"}: + raise PloadError("policy.network must be allow or offline") + if policy.get("source_build", "fallback") not in {"fallback", "forbid"}: + raise PloadError("policy.source_build must be fallback or forbid") + if not isinstance(policy.get("publish_missing_artifacts", False), bool): + raise PloadError("policy.publish_missing_artifacts must be true or false") + planning = data.get("plan", {}) + if not isinstance(planning, dict): + raise PloadError("plan must be a table") + if planning.get("mode", "interactive") not in {"interactive", "auto"}: + raise PloadError("plan.mode must be interactive or auto") + package_routes = planning.get("packages", {}) + if not isinstance(package_routes, dict): + raise PloadError("plan.packages must be a table") + route_methods = { + "cache", "configuration-artifact", "external-cache", "environment-copy", + "repository", "index-exact", "index-resolve", + } + normalized_routes = set() + for package_name, route in package_routes.items(): + if not isinstance(package_name, str) or not package_name: + raise PloadError("plan.packages keys must be package names") + normalized = normalized_name(package_name) + if normalized in normalized_routes: + raise PloadError(f"duplicate plan override for package {package_name}") + normalized_routes.add(normalized) + if not isinstance(route, dict) or route.get("method") not in route_methods: + raise PloadError(f"plan.packages.{package_name}.method is invalid") + if route.get("location") is not None and not isinstance(route["location"], str): + raise PloadError(f"plan.packages.{package_name}.location must be a string") + policy_repositories = policy.get("repositories", []) + if (not isinstance(policy_repositories, list) + or any(not isinstance(name, str) for name in policy_repositories)): + raise PloadError("policy.repositories must be an array of repository names") + dependencies = environment.get("dependencies", []) + if not isinstance(dependencies, list): + raise PloadError("environment.dependencies must be an array of package requirements") + requirements = [_parse_dependency(item) for item in dependencies] + dependency_names = [normalized_name(item.name) for item in requirements] + if len(dependency_names) != len(set(dependency_names)): + raise PloadError("environment.dependencies contains duplicate package requirements") + seen = set() + for name, source in data.get("sources", {}).items(): + safe_name(name) + if source.get("kind") != "index" or not source.get("url"): + raise PloadError(f"source {name} must be a Python package index") + parsed = urlsplit(source["url"]) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise PloadError(f"source {name} must use an HTTP(S) URL") + if public_index(source["url"]) != source["url"]: + raise PloadError(f"source {name} must not contain credentials or query parameters") + capabilities = data.get("capabilities", {}) + if not isinstance(capabilities, dict) or any( + not isinstance(values, list) or any(not isinstance(value, str) for value in values) + for values in capabilities.values() + ): + raise PloadError("capabilities must contain arrays of strings") + for package in data.get("package", []): + if not isinstance(package, dict): + raise PloadError("invalid package lock entry") + pin = f"{package.get('name', '')}=={package.get('version', '')}" + if not PIN.fullmatch(pin): + raise PloadError(f"invalid locked package: {pin}") + key = normalized_name(package["name"]) + if key in seen: + raise PloadError(f"duplicate locked package: {package['name']}") + seen.add(key) + for source_name in package.get("sources", []): + if source_name not in data.get("sources", {}): + raise PloadError( + f"locked package {package['name']} references unknown source {source_name}" + ) + for artifact in package.get("artifact", []): + if (not WHEEL.fullmatch(artifact.get("filename", "")) + or not SHA256.fullmatch(artifact.get("sha256", ""))): + raise PloadError(f"invalid artifact for {package['name']}") + for repository_name in artifact.get("repositories", []): + if repository_name not in data.get("repositories", {}): + raise PloadError( + f"artifact {artifact['filename']} references unknown repository " + f"{repository_name}" + ) + if artifact.get("url"): + parsed_artifact_url = urlsplit(artifact["url"]) + if (parsed_artifact_url.scheme not in {"http", "https"} + or not parsed_artifact_url.netloc + or parsed_artifact_url.username + or parsed_artifact_url.query): + raise PloadError(f"invalid public artifact URL for {package['name']}") + # A dependency edit can intentionally make the existing package lock stale. + # Planning reconciles and rewrites it; validation only checks each section's + # shape and internal safety. + for name, repository in data.get("repositories", {}).items(): + safe_name(name) + kind = repository.get("kind") + location = repository.get("location") + if kind == "ssh": + RepositoryManager.ssh_location(location) + elif kind == "local": + if not location: + raise PloadError(f"repository {name} has no location") + else: + raise PloadError(f"repository {name} must be local or ssh") + unknown_policy_repositories = set(policy_repositories).difference( + data.get("repositories", {}) + ) + if unknown_policy_repositories: + raise PloadError( + "policy.repositories references unknown repositories: " + + ", ".join(sorted(unknown_policy_repositories)) + ) + return data + + +def _wheel_tags(filename): + parts = filename[:-4].rsplit("-", 3) + return ["-".join(parts[-3:])] if len(parts) == 4 else [] + + +def _wheel_identity(filename): + parts = Path(filename).name[:-4].split("-") if str(filename).endswith(".whl") else [] + if len(parts) < 5: + return None, None + return normalized_name(parts[0]), parts[1] + + +class ArtifactRepository: + """Content-addressed local and SSH artifact access.""" + + SSH_OPTIONS = ( + "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", + "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=4", + ) + + @staticmethod + def _local_root(spec, base): + root = Path(spec["location"]).expanduser() + if not root.is_absolute(): + root = base / root + return root.resolve() + + @classmethod + def contains(cls, spec, checksum, base): + return checksum in cls.contains_many(spec, [checksum], base) + + @classmethod + def contains_many(cls, spec, checksums, base): + checksums = sorted(set(checksums)) + if spec["kind"] == "local": + root = cls._local_root(spec, base) / "objects" + return { + checksum for checksum in checksums + if (root / checksum).is_file() and digest(root / checksum) == checksum + } + host, root = RepositoryManager.ssh_location(spec["location"]) + object_root = root + "/objects" + checks = "; ".join( + f"test -f {shlex.quote(object_root + '/' + checksum)} && printf '%s\\n' " + f"{shlex.quote(checksum)} || true" + for checksum in checksums + ) + return set(execute( + ["ssh", *cls.SSH_OPTIONS, host, checks], timeout=75, + ).splitlines()) + + @classmethod + def publish(cls, spec, source, checksum, base): + cls.publish_many(spec, [(source, checksum)], base) + + @classmethod + def publish_many(cls, spec, artifacts, base): + """Publish multiple objects with one SSH query and one transfer.""" + artifacts = [(Path(source), checksum) for source, checksum in artifacts] + if not artifacts: + return + for source, checksum in artifacts: + if digest(source) != checksum: + raise PloadError("artifact changed before publication") + if spec["kind"] == "local": + for source, checksum in artifacts: + target = cls._local_root(spec, base) / "objects" / checksum + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and digest(target) != checksum: + raise PloadError("repository object checksum mismatch") + if not target.exists(): + temporary = target.with_name(target.name + "." + uuid.uuid4().hex) + shutil.copyfile(source, temporary) + temporary.replace(target) + return + host, root = RepositoryManager.ssh_location(spec["location"]) + checksums = sorted({checksum for _, checksum in artifacts}) + object_root = root + "/objects" + checks = "; ".join( + f"test -f {shlex.quote(object_root + '/' + checksum)} && printf '%s\\n' " + f"{shlex.quote(checksum)} || true" + for checksum in checksums + ) + existing = set(execute( + ["ssh", *cls.SSH_OPTIONS, host, + f"mkdir -p {shlex.quote(object_root)}; {checks}"], + timeout=75, + ).splitlines()) + missing = [(source, checksum) for source, checksum in artifacts + if checksum not in existing] + if missing: + pending_name = ".upload-" + uuid.uuid4().hex + remote_pending = root + "/" + pending_name + with tempfile.TemporaryDirectory(prefix="pload-publish-") as temporary: + stage = Path(temporary) + for source, checksum in missing: + staged = stage / checksum + try: + os.link(source, staged) + except OSError: + shutil.copyfile(source, staged) + execute( + ["ssh", *cls.SSH_OPTIONS, host, + f"mkdir -p {shlex.quote(remote_pending)}"], + timeout=75, + ) + execute( + ["scp", "-q", *cls.SSH_OPTIONS, + *[str(stage / checksum) for _, checksum in missing], + host + ":" + remote_pending + "/"], + timeout=300, + ) + commands = [] + for _, checksum in missing: + pending = remote_pending + "/" + checksum + target = object_root + "/" + checksum + commands.append( + f"test \"$(sha256sum {shlex.quote(pending)} | cut -d' ' -f1)\" = " + f"{shlex.quote(checksum)}" + ) + commands.append(f"mv -n {shlex.quote(pending)} {shlex.quote(target)}") + commands.append(f"rm -f {shlex.quote(pending)}") + commands.append(f"rmdir {shlex.quote(remote_pending)}") + execute( + ["ssh", *cls.SSH_OPTIONS, host, " && ".join(commands)], + timeout=75, + ) + verify = " && ".join( + f"test \"$(sha256sum {shlex.quote(object_root + '/' + checksum)} " + f"| cut -d' ' -f1)\" = {shlex.quote(checksum)}" + for checksum in checksums + ) + execute(["ssh", *cls.SSH_OPTIONS, host, verify], timeout=75) + + @classmethod + def publish_package_record(cls, spec, package, version, artifact, base): + """Publish a small searchable record separately from the immutable object.""" + record = { + "schema": 1, "name": normalized_name(package), "version": version, + "filename": artifact["filename"], "sha256": artifact["sha256"], + } + relative = ( + f"packages/{record['name']}/{version}/{artifact['filename']}.json" + ) + payload = json.dumps(record, sort_keys=True, indent=2) + "\n" + if spec["kind"] == "local": + target = cls._local_root(spec, base) / relative + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + "." + uuid.uuid4().hex) + temporary.write_text(payload, encoding="utf-8") + temporary.replace(target) + return + host, root = RepositoryManager.ssh_location(spec["location"]) + remote = root + "/" + relative + with tempfile.TemporaryDirectory(prefix="pload-record-") as temporary: + local = Path(temporary) / "record.json" + local.write_text(payload, encoding="utf-8") + pending = remote + "." + uuid.uuid4().hex + execute([ + "ssh", *cls.SSH_OPTIONS, host, + f"mkdir -p {shlex.quote(remote.rsplit('/', 1)[0])}", + ], timeout=75) + execute([ + "scp", "-q", *cls.SSH_OPTIONS, str(local), host + ":" + pending, + ], timeout=75) + execute([ + "ssh", *cls.SSH_OPTIONS, host, + f"mv {shlex.quote(pending)} {shlex.quote(remote)}", + ], timeout=75) + + @classmethod + def fetch( + cls, spec, checksum, destination, base, progress=None, expected_size=0, + ): + if spec["kind"] == "ssh": + cls._fetch_ssh_stream( + spec, checksum, Path(destination), progress, expected_size, + ) + return + callback = None + if progress: + callback = lambda _checksum, received, total: progress(received, total) + cls.fetch_many( + spec, [(checksum, destination)], base, + progress=callback, sizes={checksum: expected_size}, + ) + + @classmethod + def _fetch_ssh_stream( + cls, spec, checksum, destination, progress=None, expected_size=0, + ): + """Stream one SSH object so received bytes are visible immediately.""" + host, root = RepositoryManager.ssh_location(spec["location"]) + remote = root + "/objects/" + checksum + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(destination.name + "." + uuid.uuid4().hex) + command = [ + "ssh", *cls.SSH_OPTIONS, host, + ( + f"size=$(wc -c < {shlex.quote(remote)}) && " + f"printf '%s\\n' \"$size\" && cat -- {shlex.quote(remote)}" + ), + ] + try: + process = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + except OSError as exc: + raise PloadError(f"cannot run ssh: {exc}") from exc + received = 0 + started = time.monotonic() + last_reported_at = started + try: + size_line = process.stdout.readline(64) + try: + announced_size = int(size_line.strip()) + except (TypeError, ValueError) as exc: + raise PloadError("ssh repository returned an invalid object size") from exc + total = announced_size or expected_size + with temporary.open("wb") as output: + while True: + reader = ( + process.stdout.read1 + if hasattr(process.stdout, "read1") + else process.stdout.read + ) + chunk = reader(DOWNLOAD_CHUNK_SIZE) + if not chunk: + break + output.write(chunk) + received += len(chunk) + now = time.monotonic() + if now - started > 300: + process.kill() + raise PloadError("ssh artifact download timed out after 300 seconds") + if progress and (now - last_reported_at + >= DOWNLOAD_PROGRESS_INTERVAL): + progress(received, total) + last_reported_at = now + remaining = max(0.1, 300 - (time.monotonic() - started)) + try: + returncode = process.wait(timeout=remaining) + except subprocess.TimeoutExpired as exc: + process.kill() + process.wait() + raise PloadError( + "ssh artifact download timed out after 300 seconds" + ) from exc + stderr = process.stderr.read().decode(errors="replace") if process.stderr else "" + if returncode: + raise PloadError(f"ssh failed: {stderr.strip() or returncode}") + if progress: + progress(received, total or received) + if digest(temporary) != checksum: + raise PloadError("retrieved artifact checksum mismatch") + temporary.replace(destination) + except Exception: + temporary.unlink(missing_ok=True) + if process.poll() is None: + process.kill() + process.wait() + raise + + @classmethod + def fetch_many(cls, spec, artifacts, base, progress=None, sizes=None): + """Fetch multiple content-addressed objects through one SFTP session.""" + artifacts = [(checksum, Path(destination)) for checksum, destination in artifacts] + sizes = sizes or {} + if not artifacts: + return + for _, destination in artifacts: + destination.parent.mkdir(parents=True, exist_ok=True) + if spec["kind"] == "local": + root = cls._local_root(spec, base) / "objects" + for checksum, destination in artifacts: + source = root / checksum + temporary = destination.with_name( + destination.name + "." + uuid.uuid4().hex + ) + expected = sizes.get(checksum) or source.stat().st_size + received = 0 + last_reported_at = time.monotonic() + with source.open("rb") as input_stream, temporary.open("wb") as output: + while True: + chunk = input_stream.read(DOWNLOAD_CHUNK_SIZE) + if not chunk: + break + output.write(chunk) + received += len(chunk) + now = time.monotonic() + if progress and (now - last_reported_at + >= DOWNLOAD_PROGRESS_INTERVAL): + progress(checksum, received, expected) + last_reported_at = now + if progress: + progress(checksum, received, expected) + if digest(temporary) != checksum: + temporary.unlink(missing_ok=True) + raise PloadError("retrieved artifact checksum mismatch") + temporary.replace(destination) + return + host, root = RepositoryManager.ssh_location(spec["location"]) + with tempfile.TemporaryDirectory(prefix="pload-fetch-") as temporary: + stage = Path(temporary) + batch = stage / "commands.sftp" + commands = [] + for checksum, _ in artifacts: + commands.append( + f"get {root}/objects/{checksum} {stage / checksum}" + ) + batch.write_text("\n".join(commands) + "\n", encoding="utf-8") + command = ["sftp", "-q", "-b", str(batch), *cls.SSH_OPTIONS, host] + try: + process = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + ) + except OSError as exc: + raise PloadError(f"cannot run sftp: {exc}") from exc + started = time.monotonic() + last_sizes = {checksum: -1 for checksum, _ in artifacts} + while process.poll() is None: + now = time.monotonic() + if now - started > 300: + process.kill() + process.wait() + raise PloadError("sftp timed out after 300 seconds") + if progress: + for checksum, _ in artifacts: + downloaded = stage / checksum + received = downloaded.stat().st_size if downloaded.is_file() else 0 + if received != last_sizes[checksum]: + progress(checksum, received, sizes.get(checksum, 0)) + last_sizes[checksum] = received + time.sleep(DOWNLOAD_PROGRESS_INTERVAL) + stdout = process.stdout.read() if process.stdout else "" + stderr = process.stderr.read() if process.stderr else "" + if process.returncode: + detail = (stderr or stdout).strip() + raise PloadError(f"sftp failed: {detail or process.returncode}") + for checksum, destination in artifacts: + downloaded = stage / checksum + if not downloaded.is_file() or digest(downloaded) != checksum: + raise PloadError("retrieved artifact checksum mismatch") + if progress: + progress( + checksum, downloaded.stat().st_size, + sizes.get(checksum) or downloaded.stat().st_size, + ) + temporary_destination = destination.with_name( + destination.name + "." + uuid.uuid4().hex + ) + shutil.copyfile(downloaded, temporary_destination) + temporary_destination.replace(destination) + +class DeclarativeEnvironmentManager: + def __init__(self, config): + self.config = config + self.cache = config.home / "cache" / "wheels" + + def _external_cache_roots(self): + """Return package cache roots without creating or changing them.""" + configured = self.config.settings.get("resource_cache_dirs") + if configured is not None: + values = configured + elif os.environ.get("PLOAD_RESOURCE_CACHE_DIRS"): + values = os.environ["PLOAD_RESOURCE_CACHE_DIRS"].split(os.pathsep) + elif sys.platform == "win32": + local = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData/Local")) + values = [local / "pip/Cache", local / "uv/cache"] + elif sys.platform == "darwin": + values = [Path.home() / "Library/Caches/pip", Path.home() / "Library/Caches/uv"] + else: + cache_home = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + values = [cache_home / "pip", cache_home / "uv"] + roots = [] + for value in values: + root = Path(value).expanduser() + if root.is_dir() and root.resolve() != self.cache.resolve(): + roots.append(root.resolve()) + return list(dict.fromkeys(roots)) + + @staticmethod + def _index_external_artifacts(roots, filenames): + """Walk every external cache once instead of once per locked package.""" + wanted = set(filenames) + found = {filename: [] for filename in wanted} + for root in roots: + try: + for candidate in root.rglob("*.whl"): + if candidate.name in wanted and candidate.is_file(): + found[candidate.name].append(candidate) + except OSError: + continue + return found + + @staticmethod + def _verify_local_artifact(path, checksum): + started = time.perf_counter() + try: + valid = Path(path).is_file() and digest(path) == checksum + except OSError: + valid = False + elapsed = max(time.perf_counter() - started, 1e-6) + if not valid: + return False, elapsed, None + size = Path(path).stat().st_size + rate = size / elapsed + return True, elapsed, f"read {_human_size(rate)}/s" + + @staticmethod + def _network_rtt(url): + parsed = urlsplit(url) + if not parsed.hostname: + return None + port = parsed.port or (443 if parsed.scheme == "https" else 80) + started = time.perf_counter() + try: + with socket.create_connection((parsed.hostname, port), timeout=2): + pass + except OSError: + return None + return time.perf_counter() - started + + def _environment_resources(self, expected, package_names, progress=None): + """Index exact, safely copyable distributions in compatible pload environments.""" + resources = {} + for environment in VenvManager(self.config).environments(register_discovered=False): + if environment.get("python") != expected.get("python"): + continue + root = Path(environment["path"]) + interpreter = root / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + try: + runtime = probe_resources(interpreter, package_names) + except PloadError: + continue + if any(runtime.get(key) != expected.get(key) for key in + ("python", "implementation", "system", "machine")): + continue + if progress: + progress(f"Inspecting packages in environment {environment['id']}") + for package in runtime.get("packages", []): + if not package.get("copyable") or not package.get("record_sha256"): + continue + key = (normalized_name(package["name"]), package["version"]) + resources.setdefault(key, []).append({ + "path": str(root), + "fingerprint": package["record_sha256"], + "id": environment["id"], + }) + return resources + + def describe( + self, source, output, name=None, mode="exact", repository=None, sources=None, + progress=None, + ): + venvs = VenvManager(self.config) + candidate = Path(source).expanduser() + if candidate.is_file(): + interpreter = candidate.resolve() + inferred_name = interpreter.parent.parent.name or "environment" + else: + environment = venvs.resolve_existing(source) + interpreter = environment / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + inferred_name = environment.name + runtime = probe(interpreter) + if runtime.get("conda"): + raise PloadError("native Conda environments require a future Conda provider") + packages = sorted(runtime.pop("packages"), key=lambda item: item["name"].lower()) + accelerators = runtime.pop("accelerators", []) + pins = [f"{item['name']}=={item['version']}" for item in packages] + if any(not PIN.fullmatch(pin) for pin in pins): + raise PloadError("the environment contains package metadata that cannot be locked") + output = Path(output).expanduser().resolve() + repositories = { + key: {"kind": value["kind"], "location": value["location"]} + for key, value in self.config.settings.get("repositories", {}).items() + if value.get("kind") in {"local", "ssh"} + } + selected_repository = repository + if selected_repository and selected_repository not in repositories: + raise PloadError(f"unknown artifact repository: {selected_repository}") + if not selected_repository and repositories: + selected_repository = min(repositories) + index = public_index(self.config.settings.get("pip_index") or "https://pypi.org/simple") + parsed_index = urlsplit(index) + if parsed_index.scheme not in {"http", "https"} or not parsed_index.netloc: + raise PloadError("configured package index must use an HTTP(S) URL") + source_overrides = {} + declared_sources = {"default": {"kind": "index", "url": index}} + installed_names = {normalized_name(item["name"]) for item in packages} + for assignment in sources or []: + package_name, separator, url = assignment.partition("=") + package_name = normalized_name(package_name.strip()) + url = public_index(url.strip()) + if not separator or not package_name or not url: + raise PloadError("package sources must use PACKAGE=INDEX_URL") + parsed_source = urlsplit(url) + if parsed_source.scheme not in {"http", "https"} or not parsed_source.netloc: + raise PloadError("package source must use an HTTP(S) URL") + if package_name not in installed_names: + raise PloadError(f"package source does not match an installed package: {package_name}") + source_name = "package-" + package_name + source_overrides[package_name] = (source_name, url) + declared_sources[source_name] = {"kind": "index", "url": url} + data = { + "schema": 1, + "name": name or inferred_name, + "environment": { + "python": runtime.get("python"), + "implementation": runtime.get("implementation", "cpython"), + "system": runtime.get("system", ""), + "machine": runtime.get("machine", ""), + "dependencies": pins, + }, + "policy": { + "reproducibility": mode, + "network": "allow", + "source_build": "fallback", + "publish_missing_artifacts": False, + "repositories": [selected_repository] if selected_repository else [], + }, + "sources": declared_sources, + "repositories": repositories, + "capabilities": { + "nvidia_driver": sorted({ + version for item in accelerators if item.get("kind") == "nvidia" + for version in item.get("driver_versions", []) + }), + }, + "package": [], + } + self.cache.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="pload-describe-") as temporary: + wheel_dir = Path(temporary) + for index_number, installed in enumerate(packages, 1): + if progress: + progress( + f"[{index_number}/{len(packages)}] Locking " + f"{installed['name']}=={installed['version']}" + ) + source_name, package_index = source_overrides.get( + normalized_name(installed["name"]), ("default", index) + ) + record = { + "name": installed["name"], "version": installed["version"], + "sources": [source_name], "artifact": [], + } + if mode == "exact": + wheel = self._obtain_wheel( + interpreter, installed, wheel_dir, package_index + ) + checksum = digest(wheel) + cached = self.cache / wheel.name + if cached.exists() and digest(cached) != checksum: + raise PloadError(f"conflicting cached artifact: {wheel.name}") + if not cached.exists(): + shutil.copyfile(wheel, cached) + artifact = { + "filename": wheel.name, "sha256": checksum, + "tags": _wheel_tags(wheel.name), "repositories": [], + } + record["artifact"].append(artifact) + data["package"].append(record) + validate_manifest(data) + output.parent.mkdir(parents=True, exist_ok=True) + self._write_manifest(output, data) + data["resolved_dependencies"] = pins + self._write_lock(output, data) + return output + + def publish_package(self, package_name, environment=".", repository=None, index=None): + """Explicitly preserve one installed package wheel in a configured repository.""" + requirement = _parse_dependency(package_name) + if requirement.specifier and not str(requirement.specifier).startswith("=="): + raise PloadError("remote add accepts a package name or one exact NAME==VERSION") + env = VenvManager(self.config).resolve_existing(environment) + interpreter = env / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + installed = next( + (item for item in probe(interpreter).get("packages", []) + if normalized_name(item["name"]) == normalized_name(requirement.name)), + None, + ) + if not installed: + raise PloadError(f"{requirement.name} is not installed in {environment}") + if requirement.specifier and not requirement.specifier.contains(installed["version"]): + raise PloadError( + f"{requirement.name} {installed['version']} does not satisfy " + f"{requirement.specifier}" + ) + repositories = self.config.settings.get("repositories", {}) + selected_repository = repository or (min(repositories) if repositories else None) + if not selected_repository or selected_repository not in repositories: + raise PloadError("choose a configured repository with --remote") + source_index = public_index( + index or self.config.settings.get("pip_index") or "https://pypi.org/simple" + ) + with tempfile.TemporaryDirectory(prefix="pload-remote-add-") as temporary: + wheel = self._obtain_wheel( + interpreter, installed, Path(temporary), source_index, + ) + checksum = digest(wheel) + artifact = {"filename": wheel.name, "sha256": checksum} + ArtifactRepository.publish( + repositories[selected_repository], wheel, checksum, Path.cwd(), + ) + ArtifactRepository.publish_package_record( + repositories[selected_repository], installed["name"], + installed["version"], artifact, Path.cwd(), + ) + self.cache.mkdir(parents=True, exist_ok=True) + cached = self.cache / wheel.name + if not cached.exists(): + temporary_cache = cached.with_name(cached.name + "." + uuid.uuid4().hex) + shutil.copyfile(wheel, temporary_cache) + temporary_cache.replace(cached) + return { + "package": installed["name"], "version": installed["version"], + "repository": selected_repository, "filename": artifact["filename"], + "sha256": checksum, + } + + def _obtain_wheel(self, interpreter, installed, wheel_dir, index): + expected_name = normalized_name(installed["name"]) + expected_version = installed["version"] + direct_checksum = None + if installed.get("direct_url"): + try: + provenance = json.loads(installed["direct_url"]) + archive = provenance.get("archive_info", {}) + direct_checksum = archive.get("hashes", {}).get("sha256") + if not direct_checksum and archive.get("hash", "").startswith("sha256="): + direct_checksum = archive["hash"].split("=", 1)[1] + except (TypeError, ValueError): + pass + for wheel in self.cache.glob("*.whl"): + if (_wheel_identity(wheel.name) == (expected_name, expected_version) + and (not direct_checksum or digest(wheel) == direct_checksum)): + return wheel + if installed.get("direct_url"): + raise PloadError( + f"cannot lock direct installation {installed['name']}=={installed['version']}; " + "place its original SHA-matching wheel in the pload cache" + ) + command = [ + str(interpreter), "-m", "pip", "download", "--only-binary=:all:", + "--no-deps", "--dest", str(wheel_dir), "--index-url", index, + f"{installed['name']}=={installed['version']}", + ] + try: + execute(command) + except PloadError as exc: + raise PloadError( + f"cannot lock an exact wheel for {installed['name']}=={installed['version']}; " + "use --mode compatible only if re-resolution is acceptable" + ) from exc + matches = [ + wheel for wheel in wheel_dir.glob("*.whl") + if _wheel_identity(wheel.name) == (expected_name, expected_version) + ] + if len(matches) != 1: + raise PloadError(f"download did not produce one exact wheel for {installed['name']}") + return matches[0] + + def _load_state(self, manifest_path): + path, data = load_manifest(manifest_path) + legacy_packages = list(data.get("package", [])) + _, lock, current = load_lock(path, data) + if current: + data["package"] = lock.get("package", []) + data["resolved_dependencies"] = lock["environment"].get("dependencies", []) + elif legacy_packages: + # Versions before 1.1.0a8 embedded resolver output in pload.toml. + # Keep it long enough to migrate it into the sidecar without a download. + data["package"] = legacy_packages + data["resolved_dependencies"] = [ + f"{item['name']}=={item['version']}" for item in legacy_packages + ] + else: + data["package"] = [] + data["resolved_dependencies"] = [] + return path, data, current, bool(legacy_packages) + + def _lock_requested_dependencies( + self, path, data, lock_current=False, legacy=False, offline=False, + progress=None, + ): + requested = list(data["environment"].get("dependencies", [])) + if not requested: + return None + requirements = [_parse_dependency(value) for value in requested] + locked = { + normalized_name(package["name"]): package["version"] + for package in data.get("package", []) + } + satisfies_requests = all( + normalized_name(requirement.name) in locked + and (not requirement.specifier or requirement.specifier.contains( + locked[normalized_name(requirement.name)] + )) + for requirement in requirements + ) + if data.get("package") and satisfies_requests and (lock_current or legacy): + resolved = [ + f"{package['name']}=={package['version']}" + for package in sorted( + data["package"], key=lambda item: normalized_name(item["name"]) + ) + ] + data["resolved_dependencies"] = resolved + if legacy: + self._write_manifest(path, data) + self._write_lock(path, data) + return { + "updated": True, "migrated": True, + "path": str(lock_path(path)), + "requested": requested, "resolved": resolved, + } + return None + + network_allowed = ( + data["policy"].get("network", "allow") == "allow" and not offline + ) + if not network_allowed: + raise PloadError( + "unlocked dependencies require one online resolution before offline planning" + ) + try: + interpreter = self.config.get_python_path(data["environment"]["python"]) + except PythonNotFoundError: + PythonManager(self.config).install_python(data["environment"]["python"]) + interpreter = self.config.get_python_path(data["environment"]["python"]) + self._check_runtime(interpreter, data["environment"]) + + sources = data.get("sources", {}) + if not sources: + index = public_index( + self.config.settings.get("pip_index") or "https://pypi.org/simple" + ) + data["sources"] = {"default": {"kind": "index", "url": index}} + sources = data["sources"] + source_names = sorted(sources, key=lambda name: (name != "default", name)) + primary = sources[source_names[0]]["url"] + + self.cache.mkdir(parents=True, exist_ok=True) + known_repositories = { + (artifact["filename"], artifact["sha256"]): artifact.get("repositories", []) + for package in data.get("package", []) + for artifact in package.get("artifact", []) + } + with tempfile.TemporaryDirectory(prefix="pload-lock-") as temporary: + destination = Path(temporary) + command = [ + str(interpreter), "-m", "pip", "download", + "--disable-pip-version-check", "--only-binary=:all:", + "--dest", str(destination), "--find-links", str(self.cache), + "--index-url", primary, + ] + for source_name in source_names[1:]: + command += ["--extra-index-url", sources[source_name]["url"]] + try: + if progress: + progress( + "Configuration changed; resolving dependencies for Python " + f"{data['environment']['python']}" + ) + execute(command + requested) + except PloadError as exc: + raise PloadError( + "cannot resolve unlocked dependencies to compatible wheels for " + f"Python {data['environment']['python']}" + ) from exc + + wheels = sorted(destination.glob("*.whl"), key=lambda item: item.name.lower()) + if not wheels: + raise PloadError("dependency resolution did not produce any wheels") + records = [] + for wheel in wheels: + try: + package_name, package_version, _, wheel_tags = parse_wheel_filename( + wheel.name + ) + except InvalidWheelFilename as exc: + raise PloadError(f"resolver produced an invalid wheel: {wheel.name}") from exc + checksum = digest(wheel) + cached = self.cache / wheel.name + if cached.is_file() and digest(cached) != checksum: + raise PloadError(f"conflicting cached artifact: {wheel.name}") + if not cached.exists(): + shutil.copyfile(wheel, cached) + records.append({ + "name": str(package_name), + "version": str(package_version), + "sources": source_names, + "artifact": [{ + "filename": wheel.name, + "sha256": checksum, + "tags": sorted(str(tag) for tag in wheel_tags), + "repositories": known_repositories.get( + (wheel.name, checksum), [] + ), + }], + }) + + locked = {normalized_name(item["name"]): item["version"] for item in records} + for value in requested: + requirement = _parse_dependency(value) + version = locked.get(normalized_name(requirement.name)) + if version is None or (requirement.specifier + and not requirement.specifier.contains(version)): + raise PloadError(f"resolver did not satisfy dependency requirement: {value}") + records.sort(key=lambda item: normalized_name(item["name"])) + resolved = [f"{item['name']}=={item['version']}" for item in records] + data["package"] = records + data["resolved_dependencies"] = resolved + validate_manifest(data) + self._write_lock(path, data) + return { + "updated": True, "path": str(lock_path(path)), + "requested": requested, "resolved": resolved, + } + + @staticmethod + def _write_manifest(path, data): + temporary = path.with_name(path.name + ".tmp") + temporary.write_text(dump_manifest(data), encoding="utf-8") + temporary.replace(path) + + @staticmethod + def _write_lock(path, data): + destination = lock_path(path) + temporary = destination.with_name(destination.name + ".tmp") + temporary.write_text(dump_lock(path, data), encoding="utf-8") + temporary.replace(destination) + + def lock(self, manifest_path, offline=False, progress=None): + """Resolve and persist an exact sidecar lock as an explicit mutation.""" + path, data, current, legacy = self._load_state(manifest_path) + result = self._lock_requested_dependencies( + path, data, lock_current=current, legacy=legacy, offline=offline, + progress=progress, + ) + if result: + return result + return { + "updated": False, + "path": str(lock_path(path)), + "requested": list(data["environment"].get("dependencies", [])), + "resolved": list(data.get("resolved_dependencies", [])), + } + + def plan(self, manifest_path, offline=False, progress=None): + path, data, current, legacy = self._load_state(manifest_path) + planning = data.get("plan", {}) + planning_mode = planning.get("mode", "interactive") + route_overrides = { + normalized_name(name): route + for name, route in planning.get("packages", {}).items() + } + dependencies = data["environment"].get("dependencies", []) + direct_names = { + normalized_name(_parse_dependency(value).name) for value in dependencies + } + lock_required = bool(dependencies) and not current + sidecar = lock_path(path) + lock = { + "status": ( + "current" if current else "legacy" if legacy + else "stale" if sidecar.exists() else "missing" + ), + "path": str(sidecar), + "required": lock_required, + } + policy = data["policy"] + network_allowed = policy.get("network", "allow") == "allow" and not offline + repositories = data.get("repositories", {}) + packages = self._locked_packages(data) + preferred_repositories = policy.get("repositories", []) + supported_tags = _target_wheel_tags(data["environment"]) + try: + python = str(self.config.get_python_path(data["environment"]["python"])) + python_action = {"method": "reuse-python", "location": python, "status": "ready"} + except PythonNotFoundError: + python_action = { + "method": "install-python", "location": data["environment"]["python"], + "status": "network" if network_allowed else "unavailable", + } + if lock_required: + if network_allowed: + sources = [ + (name, source["url"]) for name, source in data.get("sources", {}).items() + if source.get("kind") == "index" + ] + if not sources: + raise PloadError("metadata resolution requires at least one index source") + if progress: + progress( + f"Resolving dependency metadata for Python " + f"{data['environment']['python']}" + ) + packages = resolve_metadata( + dependencies, sources, data["environment"], supported_tags, + ) + data["package"] = packages + data["resolved_dependencies"] = [ + f"{package['name']}=={package['version']}" for package in packages + ] + self._write_lock(path, data) + current = True + lock_required = False + lock.update({"status": "generated", "required": False}) + else: + pending = [] + for value in dependencies: + requirement = _parse_dependency(value) + exact = PIN.fullmatch(value) + pending.append({ + "name": requirement.name, + "version": exact.group(2) if exact else str(requirement.specifier) or "unresolved", + "direct": True, + "selected": self._candidate( + "lock-required", sidecar, "pending", (0, 0, 0, 0) + ), + "alternatives": [], + "rejections": [], + }) + result = { + "path": str(path), "name": data["name"], "python": python_action, + "packages": pending, "ready": False, "lock": lock, + "planning_mode": planning_mode, + } + result["plan_path"] = str(plan_path(path)) + result["saved"] = False + return result + repository_checksums = {} + for package in packages: + for artifact in package.get("artifact", []): + if not _artifact_is_compatible(artifact, supported_tags): + continue + repo_names = dict.fromkeys( + artifact.get("repositories", []) + preferred_repositories + ) + for repo_name in repo_names: + if repo_name in repositories: + repository_checksums.setdefault(repo_name, set()).add( + artifact["sha256"] + ) + repository_objects = {} + repository_rtt = {} + repository_pool = ThreadPoolExecutor(max_workers=max(1, len(repository_checksums))) + repository_futures = {} + + def check_repository(repo_name, checksums): + started = time.perf_counter() + try: + objects = ArtifactRepository.contains_many( + repositories[repo_name], checksums, path.parent, + ) + except PloadError: + objects = set() + return objects, time.perf_counter() - started + + for repo_name, checksums in repository_checksums.items(): + if progress: + progress(f"Checking artifact repository {repo_name}") + repository_futures[repo_name] = repository_pool.submit( + check_repository, repo_name, checksums, + ) + external_cache_roots = self._external_cache_roots() + compatible_filenames = { + artifact["filename"] + for package in packages + for artifact in package.get("artifact", []) + if _artifact_is_compatible(artifact, supported_tags) + } + external_artifacts = self._index_external_artifacts( + external_cache_roots, compatible_filenames, + ) + environment_resources = self._environment_resources( + data["environment"], + [package["name"] for package in packages], + progress=progress, + ) + endpoint_urls = {} + if network_allowed: + urls = [ + artifact.get("url") + for package in packages for artifact in package.get("artifact", []) + if artifact.get("url") + ] + [ + source.get("url") for source in data.get("sources", {}).values() + if source.get("kind") == "index" and source.get("url") + ] + for url in urls: + parsed = urlsplit(url) + key = (parsed.scheme, parsed.hostname, parsed.port) + endpoint_urls.setdefault(key, url) + endpoint_rtt = {} + if endpoint_urls: + with ThreadPoolExecutor(max_workers=min(4, len(endpoint_urls))) as pool: + measured = pool.map(self._network_rtt, endpoint_urls.values()) + endpoint_rtt = dict(zip(endpoint_urls, measured)) + for repo_name, future in repository_futures.items(): + repository_objects[repo_name], repository_rtt[repo_name] = future.result() + repository_pool.shutdown() + + def network_measurement(url): + parsed = urlsplit(url or "") + latency = endpoint_rtt.get((parsed.scheme, parsed.hostname, parsed.port)) + return (f"RTT {_duration(latency)}" if latency is not None + else "RTT unavailable") + + def network_cost(url): + parsed = urlsplit(url or "") + latency = endpoint_rtt.get((parsed.scheme, parsed.hostname, parsed.port)) + return latency if latency is not None else float("inf") + + plans = [] + for package in packages: + candidates = [] + rejections = [] + for artifact in package.get("artifact", []): + if not _artifact_is_compatible(artifact, supported_tags): + rejections.append({ + "artifact": artifact["filename"], + "reason": ( + f"incompatible with {data['environment']['implementation']} " + f"{data['environment']['python']} on " + f"{data['environment'].get('system') or platform.system()} " + f"{data['environment'].get('machine') or platform.machine()}" + ), + }) + continue + cached = self.cache / artifact["filename"] + adjacent = path.parent / "artifacts" / artifact["filename"] + valid, elapsed, measurement = self._verify_local_artifact( + cached, artifact["sha256"], + ) + if valid: + candidates.append(self._candidate( + "cache", cached, "ready", (0, elapsed, 0), artifact, + measurement, + )) + valid, elapsed, measurement = self._verify_local_artifact( + adjacent, artifact["sha256"], + ) + if valid: + candidates.append(self._candidate( + "configuration-artifact", adjacent, "ready", (1, elapsed, 1), + artifact, measurement, + )) + for external in external_artifacts.get(artifact["filename"], []): + valid, elapsed, measurement = self._verify_local_artifact( + external, artifact["sha256"], + ) + if valid: + candidates.append(self._candidate( + "external-cache", external, "ready", (1, elapsed, 2), + artifact, measurement, + )) + break + repo_names = dict.fromkeys( + artifact.get("repositories", []) + preferred_repositories + ) + for repo_name in repo_names: + spec = repositories.get(repo_name) + if (spec and artifact["sha256"] + in repository_objects.get(repo_name, set())): + candidates.append(self._candidate( + "repository", repo_name, "remote", + (3, repository_rtt[repo_name], repo_name), artifact, + f"RTT {_duration(repository_rtt[repo_name])}", + )) + if network_allowed: + for source_name in package.get("sources", []): + source = data.get("sources", {}).get(source_name) + if source and source.get("kind") == "index": + candidates.append(self._candidate( + "index-exact", source_name, "network", + (3, network_cost( + artifact.get("url") or source["url"] + ), source_name), + artifact, + network_measurement(artifact.get("url") or source["url"]), + )) + environment_key = (normalized_name(package["name"]), package["version"]) + for resource in environment_resources.get(environment_key, []): + candidate = self._candidate( + "environment-copy", resource["path"], "ready", + (2, 0, resource["id"]), measurement="local package", + ) + candidate["fingerprint"] = resource["fingerprint"] + candidates.append(candidate) + if policy.get("reproducibility") == "compatible" and network_allowed: + source_name = (package.get("sources") or ["default"])[0] + candidates.append(self._candidate( + "index-resolve", source_name, "network", + (3, network_cost(data["sources"][source_name]["url"]), source_name), + measurement=network_measurement(data["sources"][source_name]["url"]), + )) + candidates.sort(key=lambda item: item["cost"]) + override = route_overrides.get(normalized_name(package["name"])) + if override: + matching = [ + candidate for candidate in candidates + if candidate["method"] == override["method"] + and (not override.get("location") + or candidate["location"] == override["location"]) + ] + if matching: + candidates = matching + else: + requested = override["method"] + if override.get("location"): + requested += ":" + override["location"] + rejections.insert(0, { + "artifact": package["name"], + "reason": f"configured plan route is unavailable: {requested}", + }) + candidates = [] + plans.append({ + "name": package["name"], "version": package["version"], + "direct": normalized_name(package["name"]) in direct_names, + "selected": candidates[0] if candidates else None, + "alternatives": candidates[1:], + "rejections": rejections, + }) + ready = (not lock_required and python_action["status"] != "unavailable" + and all(item["selected"] for item in plans)) + result = {"path": str(path), "name": data["name"], "python": python_action, + "packages": plans, "ready": ready, "lock": lock, + "planning_mode": planning_mode} + result["plan_path"] = str(plan_path(path)) + result["saved"] = False + return result + + @staticmethod + def _candidate( + method, location, status, cost, artifact=None, measurement=None, + ): + result = {"method": method, "location": str(location), "status": status, + "cost": list(cost)} + if measurement: + result["measurement"] = measurement + if artifact: + result["artifact"] = artifact + return result + + @staticmethod + def _locked_packages(data): + if data.get("package"): + return data["package"] + result = [] + for pin in data["environment"].get("dependencies", []): + match = PIN.fullmatch(pin) + if not match: + continue + result.append({"name": match.group(1), "version": match.group(2), + "sources": ["default"], "artifact": []}) + return result + + def saved_plan(self, manifest_path): + """Return the exact persisted plan in the same shape used by the plan UI.""" + _, _, plan = self._load_saved_execution_plan(manifest_path) + return plan + + def _load_saved_execution_plan(self, manifest_path): + path, data, current, _ = self._load_state(manifest_path) + if data["environment"].get("dependencies") and not current: + raise PloadError( + "environment lock is missing or stale; run pload plan after resolving metadata" + ) + lock_file = lock_path(path) + _, saved = load_plan( + path, configuration_digest(data), + file_digest(lock_file) if lock_file.is_file() else "", + ) + self._validate_saved_plan(saved, data) + plan = { + "path": str(path), "name": saved["name"], "ready": saved["ready"], + "python": saved["python"], "packages": [], + "lock": {"status": "current", "required": False, "path": str(lock_file)}, + "plan_path": str(plan_path(path)), "saved": True, + "planning_mode": data.get("plan", {}).get("mode", "interactive"), + } + for package in saved.get("package", []): + selected = None + if package["method"] != "unavailable": + selected = { + "method": package["method"], "location": package["location"], + "status": package["status"], + } + if package.get("measurement"): + selected["measurement"] = package["measurement"] + if package.get("fingerprint"): + selected["fingerprint"] = package["fingerprint"] + if package.get("artifact"): + selected["artifact"] = package["artifact"] + plan["packages"].append({ + "name": package["name"], "version": package["version"], + "direct": package.get("direct", True), + "selected": selected, "alternatives": [], + "rejections": ([{"reason": package["reason"]}] + if package.get("reason") else []), + }) + unavailable = [item for item in plan["packages"] if not item["selected"]] + if unavailable: + details = [] + for item in unavailable: + label = f"{item['name']}=={item['version']}" + if item.get("rejections"): + label += f" ({item['rejections'][0]['reason']})" + details.append(label) + raise PloadError("no valid reproduction route for: " + ", ".join(details)) + return path, data, plan + + def apply( + self, manifest_path, name=None, offline=False, progress=None, target=None, + ): + path, data, plan = self._load_saved_execution_plan(manifest_path) + python_method = plan["python"].get("method") + if offline and python_method == "install-python": + raise PloadError("offline apply forbids the selected Python network route") + if python_method == "install-python": + if plan["python"]["status"] == "unavailable": + raise PloadError("required Python is unavailable while offline") + interpreter = Path(PythonManager(self.config).install_python( + data["environment"]["python"] + )).resolve() + elif python_method == "reuse-python": + interpreter = Path(plan["python"]["location"]).expanduser().resolve() + if not interpreter.is_file(): + raise PloadError( + "planned Python interpreter changed or disappeared; run pload plan again" + ) + else: + raise PloadError(f"unsupported Python acquisition method: {python_method}") + self._check_runtime(interpreter, data["environment"]) + target_name = name or data["name"] + target_path = Path(target).expanduser().resolve() if target else None + target_reference = str(target_path) if target_path else target_name + expected_state = manifest_digest(data) + try: + existing = VenvManager(self.config).resolve_existing(target_reference) + except PloadError: + existing = None + if existing: + marker = existing / ".pload-manifest.sha256" + if marker.is_file() and marker.read_text(encoding="utf-8").strip() == expected_state: + return existing + raise PloadError(f"environment {target_name!r} exists with different state") + artifacts = [] + compatible = [] + environment_copies = [] + self.cache.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="pload-apply-") as temporary: + stage = Path(temporary) + for index_number, package_plan in enumerate(plan["packages"], 1): + selected = package_plan["selected"] + label = f"{package_plan['name']}=={package_plan['version']}" + artifact = selected.get("artifact") + size = artifact.get("size", 0) if artifact else 0 + if progress: + progress( + f"PACKAGE\t{index_number}/{len(plan['packages'])}\t" + f"{selected['method']}\t{label}\t{size}" + ) + if offline and selected["method"] in {"index-exact", "index-resolve"}: + raise PloadError( + f"offline apply forbids the selected network route for " + f"{package_plan['name']}=={package_plan['version']}" + ) + if artifact: + cached = self.cache / artifact["filename"] + method = selected["method"] + if method == "cache": + source = Path(selected["location"]) + if progress: + progress(f"SOURCE\t{label}\tusing cache {source}") + if not source.is_file() or _digest_with_progress( + source, label, progress, + ) != artifact["sha256"]: + raise PloadError( + f"planned route failed for {label} (cache): " + "selected cache artifact changed or disappeared; " + "run pload plan to choose another route" + ) + install_artifact = source + else: + size_note = f" ({_human_size(size)})" if size else "" + if method == "index-exact": + source_description = artifact.get("url") or data[ + "sources" + ][selected["location"]]["url"] + action = "downloading from" + elif method == "repository": + source_description = data["repositories"][ + selected["location"] + ]["location"] + action = "fetching from repository" + else: + source_description = selected["location"] + action = "copying from" + if progress: + progress( + f"SOURCE\t{label}\t{action} " + f"{source_description}{size_note}" + ) + acquired_dir = stage / f"artifact-{index_number}" + acquired_dir.mkdir() + acquired = acquired_dir / artifact["filename"] + try: + self._acquire( + path, data, package_plan, selected, acquired, stage, + interpreter=interpreter, progress=progress, + ) + except (OSError, PloadError) as exc: + raise PloadError( + f"planned route failed for {package_plan['name']}==" + f"{package_plan['version']} ({method}): {exc}; " + "run pload plan to choose another route" + ) from exc + if not acquired.is_file() or digest(acquired) != artifact["sha256"]: + raise PloadError("planned artifact checksum mismatch") + pending = cached.with_name(cached.name + "." + uuid.uuid4().hex) + shutil.copyfile(acquired, pending) + pending.replace(cached) + install_artifact = cached + artifacts.append( + install_artifact.resolve().as_uri() + + "#sha256=" + artifact["sha256"] + ) + if progress: + progress(f"PACKAGE_DONE\t{label}\tverified") + else: + if selected["method"] == "environment-copy": + if progress: + progress( + f"SOURCE\t{label}\tcopying installed package from " + f"{selected['location']}" + ) + staged_distribution = stage / f"environment-{index_number}" + try: + self._stage_distribution( + package_plan, staged_distribution, stage, + progress=progress, + ) + except (OSError, PloadError) as exc: + raise PloadError( + f"planned route failed for {label} " + f"(environment-copy): {exc}; " + "run pload plan to choose another route" + ) from exc + environment_copies.append( + (package_plan, staged_distribution) + ) + if progress: + progress(f"PACKAGE_DONE\t{label}\tcopied") + else: + source = data.get("sources", {}).get(selected["location"], {}) + if progress: + progress( + f"SOURCE\t{label}\tdownloading from " + f"{source.get('url', selected['location'])}" + ) + compatible.append(package_plan) + venvs = VenvManager(self.config) + if progress: + progress(f"STAGE creating environment {target_name}") + env = venvs.create_venv( + version=str(interpreter), name=target_name, target=target_path, + description=f"Applied from {path.name}", + ) + try: + pip = self.config.get_pip_command(env) + if artifacts: + if progress: + progress("STAGE installing verified wheel artifacts") + execute(pip + ["install", "--no-index", "--no-deps"] + artifacts, + capture=False) + for package_plan, staged_distribution in environment_copies: + self._install_staged_distribution( + staged_distribution, env, + ) + for package_plan in compatible: + source_name = package_plan["selected"]["location"] + source = data.get("sources", {}).get(source_name, {}) + command = pip + ["install", "--no-deps", "--no-cache-dir"] + if data["policy"].get("source_build") == "forbid": + command += ["--only-binary=:all:"] + if source.get("url"): + command += ["--index-url", source["url"]] + execute(command + [ + f"{package_plan['name']}=={package_plan['version']}" + ], capture=False) + if progress: + progress( + f"PACKAGE_DONE\t{package_plan['name']}==" + f"{package_plan['version']}\tinstalled" + ) + requirements = stage / "requirements.txt" + requirements.write_text( + "".join( + f"{item['name']}=={item['version']}\n" for item in plan["packages"] + ), + encoding="utf-8", + ) + execute(pip + ["install", "--no-index", "--no-deps", "-r", + str(requirements)]) + execute(pip + ["check"], capture=False) + if progress: + progress("DONE verified exact requirements and dependency consistency") + (env / ".pload-manifest.sha256").write_text( + expected_state + "\n", encoding="utf-8" + ) + return env + except (OSError, PloadError): + venvs.remove_venv(str(env)) + raise + + def save_plan(self, manifest_path, plan): + """Persist user-selected routes against the exact current configuration and lock.""" + path, data, current, _ = self._load_state(manifest_path) + if data["environment"].get("dependencies") and not current: + raise PloadError("cannot save routes against a missing or stale lock") + sidecar = lock_path(path) + return write_plan( + path, configuration_digest(data), + file_digest(sidecar) if sidecar.is_file() else "", plan, + ) + + @staticmethod + def _validate_saved_plan(saved, data): + """Bind every executable route to one exact package in the current lock.""" + python = saved.get("python") + if not isinstance(python, dict): + raise PloadError("acquisition plan has no Python route") + if python.get("method") not in {"reuse-python", "install-python"}: + raise PloadError("acquisition plan has an unsupported Python route") + if python.get("status") not in {"ready", "network", "unavailable"}: + raise PloadError("acquisition plan has an invalid Python route status") + if python.get("method") == "install-python" and str(python.get("location")) != str( + data["environment"]["python"] + ): + raise PloadError("planned Python installation does not match the configuration") + expected_python_status = { + "reuse-python": {"ready"}, + "install-python": {"network", "unavailable"}, + }[python["method"]] + if python["status"] not in expected_python_status: + raise PloadError("Python route status does not match its acquisition method") + locked = { + (normalized_name(package["name"]), package["version"]): package + for package in DeclarativeEnvironmentManager._locked_packages(data) + } + planned = saved.get("package", []) + planned_keys = [ + (normalized_name(package.get("name", "")), package.get("version", "")) + for package in planned + ] + if len(planned_keys) != len(set(planned_keys)) or set(planned_keys) != set(locked): + raise PloadError("acquisition plan packages do not match the current lock") + artifact_methods = { + "cache", "configuration-artifact", "external-cache", + "repository", "index-exact", + } + allowed_methods = artifact_methods | { + "environment-copy", "index-resolve", "unavailable", + } + for package, key in zip(planned, planned_keys): + method = package.get("method") + if method not in allowed_methods: + raise PloadError(f"unsupported method in acquisition plan: {method}") + expected_status = { + "cache": "ready", + "configuration-artifact": "ready", + "external-cache": "ready", + "environment-copy": "ready", + "repository": "remote", + "index-exact": "network", + "index-resolve": "network", + "unavailable": "unavailable", + }[method] + if package.get("status") != expected_status: + raise PloadError( + f"route status does not match method {method} for " + f"{package.get('name')}=={package.get('version')}" + ) + configured_route = next(( + route for name, route in data.get("plan", {}).get("packages", {}).items() + if normalized_name(name) == key[0] + ), None) + if configured_route and ( + method != configured_route["method"] + or (configured_route.get("location") + and package.get("location") != configured_route["location"]) + ): + raise PloadError( + f"saved route does not match the configured plan override for " + f"{package.get('name')}=={package.get('version')}" + ) + artifact = package.get("artifact") + if method in artifact_methods: + allowed_artifacts = { + (item["filename"], item["sha256"]): item + for item in locked[key].get("artifact", []) + } + identity = ( + artifact.get("filename"), artifact.get("sha256") + ) if isinstance(artifact, dict) else None + if identity not in allowed_artifacts: + raise PloadError( + f"planned artifact is not present in the lock for " + f"{package['name']}=={package['version']}" + ) + locked_artifact = allowed_artifacts[identity] + if artifact.get("url", "") != locked_artifact.get("url", ""): + raise PloadError( + f"planned artifact URL does not match the lock for " + f"{package['name']}=={package['version']}" + ) + elif artifact: + raise PloadError(f"method {method} must not contain a wheel artifact") + if method == "environment-copy" and not package.get("fingerprint"): + raise PloadError("environment-copy route has no package fingerprint") + if method in {"index-exact", "index-resolve"}: + source_name = package.get("location") + if source_name not in locked[key].get("sources", []): + raise PloadError( + f"planned index is not locked for {package['name']}==" + f"{package['version']}" + ) + source = data.get("sources", {}).get(source_name) + if not source or source.get("kind") != "index": + raise PloadError(f"planned package index is unavailable: {source_name}") + if method == "repository": + repository = package.get("location") + if repository not in data.get("repositories", {}): + raise PloadError(f"planned repository is unavailable: {repository}") + allowed_repositories = set(locked_artifact.get("repositories", [])) + allowed_repositories.update(data["policy"].get("repositories", [])) + if repository not in allowed_repositories: + raise PloadError( + f"planned repository is not permitted for " + f"{package['name']}=={package['version']}" + ) + if (method == "index-resolve" + and data["policy"].get("reproducibility") != "compatible"): + raise PloadError("index-resolve is forbidden by exact reproducibility policy") + + def _stage_distribution( + self, package, staged_distribution, stage, progress=None, + ): + selected = package["selected"] + source_environment = Path(selected["location"]) + source_python = source_environment / ( + "Scripts/python.exe" if os.name == "nt" else "bin/python" + ) + file_manifest = stage / (normalized_name(package["name"]) + ".files.json") + script = r''' +import hashlib, json, os, sys, sysconfig +from importlib import metadata +name, version, fingerprint, output = sys.argv[1:] +dist = metadata.distribution(name) +record = dist.read_text('RECORD') or '' +if dist.version != version or hashlib.sha256(record.encode()).hexdigest() != fingerprint: + raise SystemExit('source distribution changed after planning') +roots = [os.path.realpath(sysconfig.get_path(key)) for key in ('purelib', 'platlib') + if sysconfig.get_path(key)] +files = [] +for item in dist.files or []: + source = os.path.realpath(str(dist.locate_file(item))) + match = next((root for root in roots + if source == root or source.startswith(root + os.sep)), None) + if not match: + raise SystemExit('distribution contains files outside site-packages') + relative = os.path.relpath(source, match) + if relative.startswith('..' + os.sep) or os.path.isabs(relative): + raise SystemExit('unsafe distribution path') + if os.path.isfile(source): + files.append((source, relative)) +with open(output, 'w', encoding='utf-8') as stream: + json.dump(files, stream) +''' + execute([ + str(source_python), "-c", script, package["name"], package["version"], + selected["fingerprint"], str(file_manifest), + ]) + try: + files = json.loads(file_manifest.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise PloadError("cannot read environment package file list") from exc + checked = [] + total = 0 + staged_root = Path(staged_distribution).resolve() + staged_root.mkdir(parents=True) + for source_value, relative in files: + source = Path(source_value) + destination = (staged_root / relative).resolve() + if staged_root not in destination.parents or not source.is_file(): + raise PloadError("environment package contains an unsafe path") + size = source.stat().st_size + total += size + checked.append((source, destination)) + label = f"{package['name']}=={package['version']}" + received = 0 + last_reported_at = time.monotonic() + _report_transfer(progress, label, 0, total) + for source, destination in checked: + destination.parent.mkdir(parents=True, exist_ok=True) + with source.open("rb") as input_stream, destination.open("wb") as output: + while True: + chunk = input_stream.read(1024 * 1024) + if not chunk: + break + output.write(chunk) + received += len(chunk) + now = time.monotonic() + if now - last_reported_at >= DOWNLOAD_PROGRESS_INTERVAL: + _report_transfer(progress, label, received, total) + last_reported_at = now + _report_transfer(progress, label, received, total) + + @staticmethod + def _install_staged_distribution(staged_distribution, target_environment): + target_python = Path(target_environment) / ( + "Scripts/python.exe" if os.name == "nt" else "bin/python" + ) + site_packages = Path(execute([ + str(target_python), "-c", + "import sysconfig; print(sysconfig.get_path('purelib'))", + ])).resolve() + for source in Path(staged_distribution).rglob("*"): + if not source.is_file(): + continue + relative = source.relative_to(staged_distribution) + destination = (site_packages / relative).resolve() + if site_packages not in destination.parents: + raise PloadError("staged environment package contains an unsafe path") + destination.parent.mkdir(parents=True, exist_ok=True) + try: + source.replace(destination) + except OSError: + shutil.copy2(source, destination) + source.unlink() + + def _acquire( + self, path, data, package, selected, destination, stage, interpreter, + progress=None, + ): + artifact = selected["artifact"] + label = f"{package['name']}=={package['version']}" + if selected["method"] == "configuration-artifact": + checksum = _copy_with_progress( + selected["location"], destination, label, progress, + ) + if checksum != artifact["sha256"]: + raise PloadError("selected configuration artifact changed or disappeared") + elif selected["method"] == "external-cache": + source = Path(selected["location"]) + if not source.is_file(): + raise PloadError("selected external cache artifact changed or disappeared") + checksum = _copy_with_progress(source, destination, label, progress) + if checksum != artifact["sha256"]: + raise PloadError("selected external cache artifact changed or disappeared") + elif selected["method"] == "repository": + transfer_progress = None + if progress: + def transfer_progress(received, total): + _report_transfer(progress, label, received, total) + + ArtifactRepository.fetch( + data["repositories"][selected["location"]], artifact["sha256"], + destination, path.parent, progress=transfer_progress, + expected_size=artifact.get("size", 0), + ) + elif selected["method"] == "index-exact": + if artifact.get("url"): + temporary = stage / artifact["filename"] + try: + request = Request( + artifact["url"], headers={"User-Agent": "pload-apply/1"}, + ) + with urlopen(request, timeout=60) as response, temporary.open("wb") as output: + received = 0 + response_headers = getattr(response, "headers", {}) + expected = artifact.get("size") or response_headers.get( + "Content-Length" + ) + expected = int(expected) if expected else None + last_reported = 0 + last_reported_at = time.monotonic() + while True: + chunk = response.read(DOWNLOAD_CHUNK_SIZE) + if not chunk: + break + output.write(chunk) + received += len(chunk) + now = time.monotonic() + if progress and (now - last_reported_at + >= DOWNLOAD_PROGRESS_INTERVAL): + progress( + f"DOWNLOAD\t{package['name']}=={package['version']}\t" + f"{received}\t{expected or 0}" + ) + last_reported = received + last_reported_at = now + if progress and received != last_reported: + progress( + f"DOWNLOAD\t{package['name']}=={package['version']}\t" + f"{received}\t{expected or 0}" + ) + except OSError as exc: + raise PloadError(f"cannot download planned artifact: {exc}") from exc + if digest(temporary) != artifact["sha256"]: + raise PloadError("downloaded artifact does not match the lock checksum") + shutil.copyfile(temporary, destination) + else: + source = data["sources"][selected["location"]] + download = stage / normalized_name(package["name"]) + download.mkdir(exist_ok=True) + execute([ + str(interpreter), + "-m", "pip", "download", "--only-binary=:all:", "--no-deps", + "--no-cache-dir", "--dest", str(download), + "--index-url", source["url"], + f"{package['name']}=={package['version']}", + ]) + candidate = download / artifact["filename"] + if not candidate.is_file(): + raise PloadError( + f"index did not provide locked artifact {artifact['filename']}" + ) + shutil.copyfile(candidate, destination) + else: + raise PloadError(f"unsupported acquisition method: {selected['method']}") + + @staticmethod + def _check_runtime(interpreter, expected): + actual = probe(interpreter) + if actual.get("implementation") != expected.get("implementation"): + raise PloadError("Python implementation does not satisfy the configuration") + if actual.get("python") != expected.get("python"): + raise PloadError( + f"Python version mismatch: need {expected.get('python')}, got {actual.get('python')}" + ) + for key in ("system", "machine"): + if expected.get(key) and actual.get(key) != expected.get(key): + raise PloadError( + f"target {key} mismatch: need {expected.get(key)}, got {actual.get(key)}" + ) diff --git a/src/pload/display.py b/src/pload/display.py index f1b3bd1..84e4084 100644 --- a/src/pload/display.py +++ b/src/pload/display.py @@ -1,19 +1,16 @@ -from rich import box -from rich.console import Console -from rich.table import Table - - -def _console(): - return Console(highlight=False) +from pload import ui def print_environment_table(environments): - table = Table(box=box.ROUNDED, header_style="bold cyan", border_style="blue") - table.add_column("ID", style="bold green", no_wrap=True) - table.add_column("NAME", style="bold", no_wrap=True) - table.add_column("PYTHON", style="yellow", no_wrap=True) - table.add_column("DESCRIPTION") - table.add_column("PATH", style="dim", overflow="fold") + table = ui.table( + ("ID", "bold cyan", {"width": 4, "no_wrap": True}), + ("NAME", "bold", {"width": 16, "overflow": "ellipsis", "no_wrap": True}), + ("PYTHON", "yellow", {"width": 8, "no_wrap": True}), + ("DESCRIPTION", "", {"width": 18, "overflow": "ellipsis", "no_wrap": True}), + ("PATH", "dim", {"min_width": 12, "ratio": 1, + "overflow": "ellipsis", "no_wrap": True}), + expand=True, + ) for item in environments: table.add_row( item.get("id", "-"), @@ -22,16 +19,19 @@ def print_environment_table(environments): item.get("description") or "—", item.get("path", "-"), ) - _console().print(table) + ui.console().print(table) def print_python_table(runtimes): - table = Table(box=box.ROUNDED, header_style="bold cyan", border_style="blue") - table.add_column("ID", style="bold green", no_wrap=True) - table.add_column("ALIAS", style="bold cyan", no_wrap=True) - table.add_column("VERSION", style="bold green", no_wrap=True) - table.add_column("TYPE", style="yellow", no_wrap=True) - table.add_column("PATH", style="dim", overflow="fold") + table = ui.table( + ("ID", "bold cyan", {"width": 5, "no_wrap": True}), + ("ALIAS", "cyan", {"width": 20, "overflow": "ellipsis", "no_wrap": True}), + ("VERSION", "green", {"width": 10, "no_wrap": True}), + ("TYPE", "yellow", {"width": 10, "no_wrap": True}), + ("PATH", "dim", {"min_width": 14, "ratio": 1, + "overflow": "ellipsis", "no_wrap": True}), + expand=True, + ) for runtime in runtimes: table.add_row( runtime.id or "-", @@ -40,4 +40,4 @@ def print_python_table(runtimes): runtime.source, str(runtime.path), ) - _console().print(table) + ui.console().print(table) diff --git a/src/pload/help_content.py b/src/pload/help_content.py index 7f677b7..8e56d82 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -1,7 +1,6 @@ -from rich import box -from rich.console import Console from rich.markdown import Markdown -from rich.table import Table + +from pload import ui GUIDES = { (): r""" @@ -50,6 +49,20 @@ - project-local `pload init` environments may live outside the managed root. Use `pload cfg` to see the exact effective paths before creating anything. + +## Reproduce an environment from one file + +```console +$ pload describe v1 -o pload.toml +$ pload plan pload.toml +$ pload apply pload.toml +``` + +`pload.toml` describes user intent. `plan` resolves from independent metadata +when its managed lock is stale, then inventories resources without downloading +wheel bodies. Analysis does not overwrite saved choices; Save, Apply, +`--no-ui`, or `[plan].mode = "auto"` writes the plan. `apply` executes only that saved plan and never +changes route or uploads. """, ("new",): r""" # Create a managed environment @@ -72,6 +85,12 @@ guided creator. Use `--message` (or `-m`) for the human description; it never affects the generated directory name. +When packages are requested, `--package-strategy auto` (the default) prefers +a compatible wheel already in pload's cache and otherwise uses the configured +package index. `--package-strategy custom` (or `-s custom`) opens one source +choice for each package written on the command line. Dependencies introduced +by those packages remain automatic, so the chooser stays short. + ## Choose an exact location or interpreter ```console @@ -89,8 +108,11 @@ - Creates one virtual-environment directory and one registry entry. - Does not modify the system Python or activate the environment automatically. - If Python cannot create the environment, the incomplete directory is removed. -- If later package installation fails, the valid environment remains so it can - be inspected or removed with `pload rm`. +- Requested packages are attempted one at a time. A bad request is reported, + later requests still run, and successful installations remain available. +- If any package failed, pload returns a failure status after all requests and + lists every failure. The valid environment remains so it can be inspected or + removed with `pload rm`. """, ("init",): r""" # Create an environment for the current project @@ -331,8 +353,141 @@ } +GUIDES[("describe",)] = r""" +# Turn an existing environment into one portable configuration + +```console +pload describe v2 +pload describe /project/.venv -o project.pload.toml +pload describe /project/.venv/bin/python -n project -r lab +pload describe v2 -s torch=https://download.pytorch.org/whl/cu121 +``` + +The default exact mode locks every installed application package to an actual +wheel and SHA-256. pload reuses its local cache and may obtain a missing wheel, +but does not upload it. Use `pload remote add PACKAGE` for explicit backup. + +Use `--mode compatible` only when exact wheels do not exist and future +re-resolution is acceptable. Native Conda packages are not silently converted. +Credentials and URL query tokens are never written into the portable file. +Use repeatable `--source PACKAGE=URL` options when an installed package came from +a dedicated index that cannot be inferred reliably from installed metadata. +""" +GUIDES[("plan",)] = r""" +# Explain how the desired state can be satisfied + +```console +pload plan +pload plan project.pload.toml +pload plan --offline +pload plan --json +pload plan --no-ui +``` + +Planning resolves missing/stale locks using Simple API pages and independent +Core Metadata, then checks pload/pip/uv caches, compatible environments, +adjacent artifacts and content-addressed repositories. It never downloads a +wheel body. A current lock skips metadata resolution. Cache roots are indexed +once and only matching-Python environments are inspected. The chooser reports +measured local read throughput and network/repository RTT instead of fixed time +guesses. Analysis does not overwrite `.pload_plan.toml`; Save, Apply, +`--no-ui`, and `[plan].mode = "auto"` persist choices, while quit and `--json` +do not. + +The chooser shows only requirements written in `pload.toml` and +automatically assigns routes to their transitive dependencies. It supports +previous/next, undo, reset, save, and save-and-apply; `--no-ui` selects the +lowest-cost measured routes for automation. + +For configuration-driven automation, add: + +```toml +[plan] +mode = "auto" +``` + +This saves recommended routes without opening the chooser. Strict exceptions +can be declared under `[plan.packages.NAME]` with `method` and optional +`location`; an unavailable override fails planning instead of falling back. +""" +GUIDES[("lock",)] = r""" +# Resolve the user declaration into an exact managed lock + +```console +pload lock +pload lock project.pload.toml +pload lock --offline +``` + +This legacy explicit command is retained for preview compatibility. The normal +workflow is now `pload plan`, whose metadata-only resolver writes the managed +lock without obtaining wheel bodies. Prefer `plan` for new configurations. +""" +GUIDES[("apply",)] = r""" +# Materialize the configuration + +```console +pload apply +pload apply project.pload.toml -n project-copy +pload apply --local +pload apply --offline +``` + +Apply requires a current `.pload_lock.toml` and `.pload_plan.toml`, then installs +Python when needed. Before changing anything it prints the complete saved plan. +Execution then uses compact Docker-style progress rows. Each package starts with +its selected method in brackets, followed by `NAME==VERSION`, source, progress +bar and real network byte counts. It executes each selected route, verifies hashes, creates +the environment, installs packages and runs `pip check`. It never resolves, tries an +alternative route, or uploads package bytes. A selected `index-exact` route +downloads from the locked URL even if the same wheel is already cached; the +verified wheel is then installed locally with package networking disabled. +Reapplying the same configuration is idempotent; an +existing environment with different state is never silently overwritten. + +`--local` (or `-l`) creates `.venv` beside the selected configuration file, +giving declarative projects the same local-environment layout as `pload init`. +The location follows the configuration file rather than the caller's current +working directory, so `pload apply path/to/pload.toml --local` reliably targets +`path/to/.venv`. +""" +GUIDES[("repo",)] = r""" +# Configure reusable artifact providers + +```console +pload repo add lab frpxiaoxin:/home/tibless/pload-cloud -t ssh +pload repo add disk /mnt/shared/pload -t local +pload repo list +pload repo remove lab +``` + +`describe` embeds configured local/SSH providers in `pload.toml`; `apply` uses them +automatically. SSH relies on OpenSSH aliases, keys and agents and stores no password. +Objects are addressed by SHA-256 so identical large wheels are stored only once. +Removing a provider forgets its configuration but does not delete remote data. +""" +for _command in ("add", "list", "remove"): + GUIDES[("repo", _command)] = GUIDES[("repo",)] + +GUIDES[("remote",)] = r""" +# Back up package artifacts explicitly + +```console +pload remote add torch --from v10 --remote lab +pload remote add numpy==2.1.0 -f ./project/.venv -r disk +``` + +`remote add` finds the exact installed version, reuses its original cached wheel +when available, verifies its SHA-256 identity, and stores it in the selected +content-addressed repository. If the original wheel is unavailable it may fetch +that exact version from `--index`. Planning and applying never upload packages; +publication happens only through this explicit command. +""" +GUIDES[("remote", "add")] = GUIDES[("remote",)] + + def render_detailed_help(parser, command_path): - console = Console(highlight=False) + console = ui.console() content = GUIDES.get(tuple(command_path), GUIDES[()]) console.print(Markdown(content.strip())) @@ -343,14 +498,11 @@ def render_detailed_help(parser, command_path): label = ", ".join(action.option_strings) if action.option_strings else action.dest rows.append((label, action.help or "")) if rows: - table = Table( - title="Related options", - box=box.ROUNDED, - header_style="bold cyan", - border_style="blue", + console.print("\n[bold cyan]Options[/]") + table = ui.table( + ("OPTION", "cyan", {"no_wrap": True}), + ("MEANING", "", {}), ) - table.add_column("OPTION", style="bold green", no_wrap=True) - table.add_column("MEANING") for label, help_text in rows: table.add_row(label, help_text) console.print(table) diff --git a/src/pload/installer.py b/src/pload/installer.py index 1d8d276..57998f2 100644 --- a/src/pload/installer.py +++ b/src/pload/installer.py @@ -6,16 +6,15 @@ from argparse import Namespace from pathlib import Path +from questionary import Choice from rich import box -from rich.console import Console from rich.markdown import Markdown from rich.panel import Panel from rich.table import Table from rich.text import Text -from pload import __version__ +from pload import __version__, ui from pload.errors import PloadError -from pload.managers.color import Colors from pload.settings import ( USTC_PYTHON_MIRROR, default_bin_dir, @@ -44,7 +43,7 @@ def build_parser(): - parser = argparse.ArgumentParser( + parser = ui.ArgumentParser( prog="pload-install", description=( "Install pload into its own private runtime, create a stable executable in a " @@ -104,7 +103,7 @@ def build_parser(): def render_help(detailed=False): parser = build_parser() if detailed: - console = Console(highlight=False) + console = ui.console() console.print(Markdown(r""" # Install pload without tying it to a project environment @@ -120,19 +119,17 @@ def render_help(detailed=False): managed virtual environment directory [/mnt/tools/pload/venvs]: /mnt/venvs managed Python directory [/mnt/tools/pload/pythons]: /mnt/python -Choose the Python runtime download source - 1) official (default) - 2) ustc - 3) custom -Enter a number [1]: 2 - -Choose the Python package index used to install pload and uv - 1) official (default) - 2) tsinghua - 3) ustc - 4) aliyun - 5) custom -Enter a number [1]: 3 +? Choose the Python runtime download source (↑/↓ move • enter select) + official Astral's official releases +❯ ustc USTC mirror in China + custom Your own mirror + +? Choose the Python package index (↑/↓ move • enter select) + official Official PyPI + tsinghua Tsinghua University mirror +❯ ustc USTC mirror + aliyun Alibaba Cloud mirror + custom Private compatible index ``` ## Resulting layout @@ -178,21 +175,18 @@ def render_help(detailed=False): `--yes` reuses explicit values and saved defaults. It does not edit a shell profile unless `--shell` is supplied or already saved in the configuration. """.strip())) - table = Table( - title="Related options", - box=box.ROUNDED, - header_style="bold cyan", - border_style="blue", + console.print("\n[bold cyan]Options[/]") + table = ui.table( + ("OPTION", "cyan", {"no_wrap": True}), + ("MEANING", "", {}), ) - table.add_column("OPTION", style="bold green", no_wrap=True) - table.add_column("MEANING") for action in parser._actions: if action.dest == "help" or not action.option_strings: continue table.add_row(", ".join(action.option_strings), action.help or "") console.print(table) return - console = Console(highlight=False) + console = ui.console() usage = " ".join(parser.format_usage().split()) usage_text = Text() for index, token in enumerate(usage.split()): @@ -217,7 +211,7 @@ def render_help(detailed=False): table = Table(box=box.SIMPLE, header_style="bold cyan", show_edge=False) table.add_column("COMMAND", style="bold green", no_wrap=True) table.add_column("PURPOSE") - table.add_row("pload-install", "Start the colored, numbered guided setup") + table.add_row("pload-install", "Start the cursor-key guided setup") table.add_row("pload-install --yes", "Reuse defaults without interactive questions") table.add_row( "pload-install --yes --home PATH", @@ -241,49 +235,132 @@ def render_help(detailed=False): def ask(prompt, default, non_interactive=False): if non_interactive: return str(default) - answer = input(f"{Colors.cyan(prompt)} [{Colors.green(default)}]: ").strip() - return answer or str(default) + return ui.text(prompt, default=str(default)) def ask_choice(prompt, choices, default, non_interactive=False): if non_interactive: return default items = [item if isinstance(item, tuple) else (item, "") for item in choices] - keys = [item[0] for item in items] - default_index = keys.index(default) if default in keys else 0 - while True: - print(f"\n{Colors.bold(prompt)}") - for index, (key, description) in enumerate(items, 1): - marker = Colors.green(" (default)") if index - 1 == default_index else "" - print(f" {Colors.cyan(index)}) {Colors.bold(key)}{marker}") - if description: - print(f" {description}") - answer = input( - f"{Colors.cyan('Enter a number')} [{Colors.green(default_index + 1)}]: " - ).strip().lower() - if not answer: - return keys[default_index] - if answer.isdigit() and 1 <= int(answer) <= len(items): - return keys[int(answer) - 1] - if answer in keys: - return answer - print(Colors.yellow(f"Please enter 1-{len(items)}.")) + choices = [ + Choice(f"{key:<10} {description}", value=key) + for key, description in items + ] + return ui.select(prompt, choices, default=default) def print_welcome(): - print(Colors.bold("\npload guided setup")) - print( - "This installer keeps pload itself, downloaded Python runtimes, and virtual " - "environments independent. Press Enter to accept any recommended default." + output = ui.console() + ui.heading( + "Set up pload", + "Choose where pload, Python runtimes, and environments live. " + "Press Enter to accept the recommended value.", + output=output, ) - print(Colors.cyan("\nDirectory layout")) - print(" pload home configuration, private runtime, state, and caches") - print(" executable bin stable `pload` command; add this directory to PATH") - print(" environments every managed project environment") - print(" Python runtimes Python versions downloaded through uv") + output.print("[dim]You can revisit these choices later with `pload cfg -t`.[/]") + + +def review_settings(args): + """Edit setup values from one revisitable settings screen.""" + home = Path(args.home or Path.home() / ".pload").expanduser().resolve() + existing = load_settings(home) + python = existing.get("python", {}) + values = { + "home": str(home), + "bin_dir": str(Path(args.bin_dir or existing.get("bin_dir") or default_bin_dir())), + "venvs_dir": str(Path(args.venvs_dir or existing.get("venvs_dir") or home / "venvs")), + "python_dir": str(Path(args.python_dir or python.get("install_dir") or home / "pythons")), + "source": args.source or python.get("source", "official"), + "mirror_url": args.mirror_url or python.get("mirror"), + "pip_source": args.pip_source or existing.get("pip_source", "official"), + "pip_index": args.pip_index or existing.get("pip_index"), + "shell": args.shell or existing.get("shell") or detect_shell(), + } + labels = { + "home": "pload data", + "bin_dir": "executable bin", + "venvs_dir": "environments", + "python_dir": "Python runtimes", + "source": "Python source", + "pip_source": "package index", + "shell": "shell integration", + } + path_fields = {"home", "bin_dir", "venvs_dir", "python_dir"} + while True: + choices = [ + Choice(f"{labels[key]:<19} {values[key]}", value=key) + for key in labels + ] + choices.extend([ + Choice(f"{ui.glyph('✓', '[ok]')} Save configuration", value="__save__"), + Choice(f"{ui.glyph('×', 'x')} Cancel", value="__cancel__"), + ]) + field = ui.select( + "Review setup", + choices, + default="__save__", + instruction=ui.navigation_hint("edit"), + ) + if field == "__save__": + break + if field == "__cancel__": + raise KeyboardInterrupt + if field in path_fields: + values[field] = ui.text(labels[field], default=values[field]) + elif field == "source": + values[field] = ui.select( + "Python runtime source", + [Choice(f"{key:<10} {description}", value=key) + for key, description in PYTHON_SOURCE_CHOICES], + default=values[field], + ) + if values[field] == "custom": + values["mirror_url"] = ui.text( + "Custom mirror base URL", + default=values["mirror_url"] or "file:///path/to/mirror", + ) + elif field == "pip_source": + values[field] = ui.select( + "Python package index", + [Choice(f"{key:<10} {description}", value=key) + for key, description in PIP_SOURCE_CHOICES], + default=values[field], + ) + if values[field] == "custom": + values["pip_index"] = ui.text( + "Custom package index URL", + default=values["pip_index"] or "https://example.com/simple", + ) + elif field == "shell": + detected = detect_shell() + shells = list(dict.fromkeys([detected, "bash", "zsh", "fish", "powershell"])) + values[field] = ui.select( + "Shell integration", + [Choice( + f"{shell:<12} " + f"{'detected shell' if shell == detected else 'update profile'}", + value=shell, + ) for shell in shells] + + [Choice("none print manual instructions", value="none")], + default=values[field], + ) + + args.home = values["home"] + args.bin_dir = values["bin_dir"] + args.venvs_dir = values["venvs_dir"] + args.python_dir = values["python_dir"] + args.source = values["source"] + args.mirror_url = values["mirror_url"] + args.pip_source = values["pip_source"] + args.pip_index = values["pip_index"] + args.shell = values["shell"] + args.yes = True + return args def collect_settings(args): + if not args.yes and ui.is_interactive(): + args = review_settings(args) home = Path(ask("pload home", args.home or Path.home() / ".pload", args.yes)).expanduser().resolve() existing = load_settings(home) bin_dir = Path( @@ -367,10 +444,11 @@ def collect_settings(args): pip_index = ask("custom Python package index URL", "https://example.com/simple", False) return { + **existing, "home": str(home), "bin_dir": str(bin_dir), "venvs_dir": str(venvs_dir), - "state_dir": str(home / "state"), + "state_dir": existing.get("state_dir", str(home / "state")), "pip_source": pip_source, "pip_index": pip_index, "package_spec": args.package_spec or existing.get("package_spec") or default_package_spec(), @@ -544,30 +622,23 @@ def configure_shell(settings): def print_welcome_screen(version, launcher, settings, profile=None): """Show a friendly post-install landing screen with copyable next steps.""" - print() - print(Colors.cyan(" ____ _ _ ")) - print(Colors.cyan(" | _ \\| | ___ __ _ __| |")) - print(Colors.cyan(" | |_) | |/ _ \\ / _` |/ _` |")) - print(Colors.cyan(" | __/| | (_) | (_| | (_| |")) - print(Colors.cyan(" |_| |_|\\___/ \\__,_|\\__,_|")) - print(Colors.bold(f"\n pload {version} is ready")) - print(" Your private runtime and launcher are installed.") - print() - print(Colors.bold(" Next steps")) - print(f" {Colors.green('1')} Check configuration: pload cfg") - print(f" {Colors.green('2')} Find Python versions: pload python list") - print(f" {Colors.green('3')} Create an environment: pload new -n data -v 3.12") - print(f" {Colors.green('4')} Read the walkthrough: pload -h -d") - print() - print(Colors.bold(" Installed paths")) - print(f" launcher {launcher}") - print(f" data {settings['home']}") - print(f" environments {settings['venvs_dir']}") + output = ui.console() + ui.logo(output=output) + output.print() + ui.success(f"pload {version} is ready", detail="Private runtime and launcher installed") + ui.heading("Next steps", output=output) + output.print(" [cyan]1[/] Check configuration [bold]pload cfg[/]") + output.print(" [cyan]2[/] Find Python versions [bold]pload python list[/]") + output.print(" [cyan]3[/] Create an environment [bold]pload new -n data -v 3.12[/]") + output.print(" [cyan]4[/] Read the walkthrough [bold]pload -h -d[/]") + ui.heading("Installed paths", output=output) + output.print(f" launcher [dim]{launcher}[/]") + output.print(f" data [dim]{settings['home']}[/]") + output.print(f" environments [dim]{settings['venvs_dir']}[/]") if profile: - print(f" shell updated {profile}") + output.print(f" shell [dim]updated {profile}[/]") else: - print(f" shell add {settings['bin_dir']} to PATH, then start a new shell") - print() + output.print(f" shell [dim]add {settings['bin_dir']} to PATH[/]") def run(argv=None): @@ -581,18 +652,18 @@ def run(argv=None): settings = collect_settings(args) path = save_settings(settings["home"], settings) if args.no_runtime_install: - print(f"[*] Wrote configuration: {path}") + ui.success("Wrote configuration", detail=str(path)) return 0 python = install_private_runtime(settings) launcher = write_launcher(settings, python) profile = configure_shell(settings) installed_version = installed_pload_version(python) - print(f"[*] Installed pload {installed_version}: {launcher}") - print(f"[*] Configuration: {path}") + ui.success(f"Installed pload {installed_version}", detail=str(launcher)) + ui.success("Saved configuration", detail=str(path)) if profile: - print(f"[*] Updated shell profile: {profile}") + ui.success("Updated shell profile", detail=str(profile)) else: - print(f"[!] Add {settings['bin_dir']} to PATH, then run: pload shell-init ") + ui.warning(f"Add {settings['bin_dir']} to PATH, then run: pload shell-init ") print_welcome_screen(installed_version, launcher, settings, profile) return 0 @@ -600,8 +671,11 @@ def run(argv=None): def main(argv=None): try: return run(argv) + except (KeyboardInterrupt, EOFError): + ui.warning("Cancelled") + return 130 except (PloadError, OSError) as exc: - print(f"pload-install: error: {exc}", file=sys.stderr) + ui.error(str(exc)) return 1 diff --git a/src/pload/managers/color.py b/src/pload/managers/color.py deleted file mode 100644 index cc653e6..0000000 --- a/src/pload/managers/color.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import sys - - -class Colors: - """Small color helper that stays quiet when output is redirected.""" - - enabled = sys.stdout.isatty() and os.environ.get("NO_COLOR") is None - - @classmethod - def _paint(cls, value, color): - text = str(value) - if not cls.enabled: - return text - return f"\033[1;{color}m{text}\033[0m" - - @classmethod - def green(cls, value): - return cls._paint(value, "32") - - @classmethod - def red(cls, value): - return cls._paint(value, "31") - - @classmethod - def yellow(cls, value): - return cls._paint(value, "33") - - @classmethod - def cyan(cls, value): - return cls._paint(value, "36") - - @classmethod - def blue(cls, value): - return cls._paint(value, "34") - - @classmethod - def bold(cls, value): - text = str(value) - if not cls.enabled: - return text - return f"\033[1m{text}\033[0m" diff --git a/src/pload/managers/dependency.py b/src/pload/managers/dependency.py index 1c5fc00..cae6cbf 100644 --- a/src/pload/managers/dependency.py +++ b/src/pload/managers/dependency.py @@ -1,22 +1,125 @@ import subprocess +from pathlib import Path +from packaging.requirements import InvalidRequirement, Requirement +from packaging.utils import InvalidWheelFilename, canonicalize_name, parse_wheel_filename +from questionary import Choice + +from pload import ui from pload.errors import PloadError -from pload.managers.color import Colors +from pload.managers.platform import ConfigManager + + +def _compatible_tags(venv_path): + """Describe the target runtime, rather than the Python running pload.""" + from pload.declarative import _target_wheel_tags + from pload.snapshots import probe + + return _target_wheel_tags(probe(ConfigManager.venv_python(venv_path))) class DependencyManager: def __init__(self, config): self.config = config - def install_dependencies(self, venv_path, requirements, channel=None): + def install_dependencies(self, venv_path, requirements, channel=None, strategy="auto"): if not requirements: return + if strategy not in {"auto", "custom"}: + raise PloadError(f"unknown package strategy: {strategy}") + + wheel_cache = self.config.home / "cache" / "wheels" + supported_tags = _compatible_tags(venv_path) if wheel_cache.is_dir() else set() + index = channel or self.config.settings.get("pip_index") + failures = [] + installed = [] + + for value in requirements: + try: + requirement = Requirement(value) + except InvalidRequirement: + ui.warning(f"Skipping invalid package requirement: {value}") + failures.append((value, "invalid requirement")) + continue + + routes = self._routes(requirement, wheel_cache, supported_tags, index) + selected = routes[0] + if strategy == "custom" and len(routes) > 1: + selected = ui.select( + f"Source for {value}", + [Choice(route["label"], value=route) for route in routes], + default=routes[0], + ) + + ui.info(f"Installing {value} from {selected['summary']}") + command = self.config.get_pip_command(venv_path) + ["install"] + if selected["kind"] == "cache": + command.append(str(selected["path"])) + command += ["--find-links", str(wheel_cache)] + else: + command.append(value) + if index: + command += ["--index-url", index] + try: + process = subprocess.run(command, check=False) + except OSError as exc: + failures.append((value, str(exc))) + ui.warning(f"Could not install {value}; continuing with the next package") + continue + if process.returncode != 0: + failures.append((value, f"{selected['summary']} failed")) + ui.warning(f"Could not install {value}; continuing with the next package") + continue + installed.append(value) + + if installed: + ui.success( + f"Installed {len(installed)} requested package" + f"{'s' if len(installed) != 1 else ''}", + detail=", ".join(installed), + ) + if failures: + details = "; ".join(f"{name} ({reason})" for name, reason in failures) + raise PloadError( + "some packages were not installed; successful packages were kept: " + details + ) - command = self.config.get_pip_command(venv_path) + ["install"] + list(requirements) - if channel: - command += ["--index-url", channel] + @staticmethod + def _routes(requirement, wheel_cache, supported_tags, index): + matches = [] + # Installing a wheel path would discard URL, marker, or extras semantics. + # Let pip interpret those requirement forms from the configured index. + if (wheel_cache.is_dir() and not requirement.url + and not requirement.marker and not requirement.extras): + for wheel in wheel_cache.glob("*.whl"): + try: + name, version, _, tags = parse_wheel_filename(wheel.name) + except InvalidWheelFilename: + continue + if canonicalize_name(name) != canonicalize_name(requirement.name): + continue + if requirement.specifier and not requirement.specifier.contains( + version, prereleases=True, + ): + continue + if not tags.intersection(supported_tags): + continue + matches.append((version, wheel)) + matches.sort(key=lambda item: (item[0], item[1].name), reverse=True) - print(f"[*] Installing: {' '.join(Colors.green(item) for item in requirements)}") - process = subprocess.run(command, check=False) - if process.returncode != 0: - raise PloadError("dependency installation failed") + routes = [ + { + "kind": "cache", + "path": Path(wheel), + "summary": f"pload cache ({wheel.name})", + "label": f"Cache {wheel.name}", + } + for _, wheel in matches + ] + location = index or "pip/default index" + routes.append({ + "kind": "index", + "summary": f"package index ({location})", + "label": f"Index {location}", + }) + return routes diff --git a/src/pload/managers/pyversion.py b/src/pload/managers/pyversion.py index a9ea2c4..dacf6e1 100644 --- a/src/pload/managers/pyversion.py +++ b/src/pload/managers/pyversion.py @@ -432,7 +432,7 @@ def _python_files(self, root, recursive=False): return [] def _classify_path(self, path, declared): - resolved = str(path.resolve()).replace("\\", "/").lower() + resolved = str(self._resolved_path(path)).replace("\\", "/").lower() configured_uv = str(Path(self.config.python["install_dir"]).expanduser()).replace( "\\", "/" ).lower() @@ -481,7 +481,18 @@ def _is_python_name(cls, name): @staticmethod def _path_key(path): - return os.path.normcase(str(Path(path).expanduser().resolve())) + return os.path.normcase(str(PythonManager._resolved_path(path))) + + @staticmethod + def _resolved_path(path): + path = Path(path).expanduser() + try: + return path.resolve() + except OSError: + # Windows Store app-execution aliases can look like files but reject + # GetFinalPathNameByHandle on older Python releases. They are still + # valid discovery candidates and will be rejected safely by _probe. + return path.absolute() @staticmethod def _version_key(version): @@ -492,7 +503,7 @@ def _unique_paths(paths): unique = [] seen = set() for path in paths: - key = os.path.normcase(str(Path(path).expanduser().resolve())) + key = os.path.normcase(str(PythonManager._resolved_path(path))) if key not in seen: seen.add(key) unique.append(Path(path).expanduser()) diff --git a/src/pload/managers/venv.py b/src/pload/managers/venv.py index ee345ee..a6d9576 100644 --- a/src/pload/managers/venv.py +++ b/src/pload/managers/venv.py @@ -5,11 +5,10 @@ from datetime import datetime, timezone from pathlib import Path -from rich.console import Console from rich.text import Text +from pload import ui from pload.errors import PloadError -from pload.managers.color import Colors class VenvManager: @@ -42,7 +41,7 @@ def create_venv( python_exe = self.config.get_python_path(version) target_path.parent.mkdir(parents=True, exist_ok=True) command = [str(python_exe), "-m", "venv", str(target_path)] - console = Console(highlight=False) + console = ui.console() if console.is_terminal: status = Text.assemble( ("Creating ", "bold cyan"), @@ -55,7 +54,7 @@ def create_venv( command, capture_output=True, text=True, check=False ) else: - print(f"[*] Creating {Colors.green(display_name)} at {Colors.green(target_path)}") + ui.info(f"Creating {display_name} at {target_path}") process = subprocess.run( command, capture_output=True, text=True, check=False ) @@ -71,7 +70,7 @@ def create_venv( (display_name, "bold"), )) else: - print(f"[*] Created {Colors.green(display_name)}") + ui.success(f"Created {display_name}") entry = self.register_environment( target_path, name=display_name, @@ -84,7 +83,7 @@ def create_venv( (entry["id"], "bold cyan"), )) else: - print(f"[*] Assigned {Colors.cyan(entry['id'])}") + ui.success(f"Assigned {entry['id']}") return target_path @property @@ -163,8 +162,8 @@ def register_environment(self, path, name=None, description=None): self._write_registry(data) return entry - def environments(self): - """Return registered environments and import legacy managed environments.""" + def environments(self, register_discovered=True): + """Return known environments, optionally without importing discovered ones.""" data = self._read_registry() registered_paths = { str(Path(entry.get("path", "")).expanduser().resolve()) @@ -172,16 +171,23 @@ def environments(self): if entry.get("path") } root = self.config.venv_path + transient = [] if root.is_dir(): for child in sorted(root.iterdir(), key=lambda item: item.name.lower()): resolved = str(child.resolve()) if self._valid_environment(child) and resolved not in registered_paths: - self.register_environment(child, name=child.name) + if register_discovered: + self.register_environment(child, name=child.name) + else: + transient.append({ + "id": f"path:{child.name}", "name": child.name, + "path": resolved, "description": "", "created_at": "", + }) registered_paths.add(resolved) data = self._read_registry() result = [] - for entry in data["environments"]: + for entry in data["environments"] + transient: path = Path(entry.get("path", "")).expanduser() if self._valid_environment(path): item = dict(entry) @@ -265,7 +271,7 @@ def remove_venv(self, venv_name, project_dir=None): if Path(entry.get("path", "")).expanduser().resolve() != target_path ] self._write_registry(data) - print(f"[*] Removed {Colors.green(target_path)}") + ui.success(f"Removed {target_path}") def activation_script(self, venv_name, shell=None, project_dir=None): path = self.resolve_existing(venv_name, project_dir=project_dir) diff --git a/src/pload/metadata_resolver.py b/src/pload/metadata_resolver.py new file mode 100644 index 0000000..2e666ff --- /dev/null +++ b/src/pload/metadata_resolver.py @@ -0,0 +1,310 @@ +"""Resolve exact wheel graphs using index metadata, never distribution bodies.""" + +import hashlib +import json +import platform +import sys +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from email.parser import BytesParser +from email.policy import compat32 +from html.parser import HTMLParser +from urllib.error import HTTPError, URLError +from urllib.parse import unquote, urljoin, urlsplit +from urllib.request import Request, urlopen + +from packaging.markers import default_environment +from packaging.requirements import InvalidRequirement, Requirement +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.utils import InvalidWheelFilename, canonicalize_name, parse_wheel_filename +from packaging.version import InvalidVersion, Version +from resolvelib import BaseReporter, Resolver +from resolvelib.resolvers import ResolutionImpossible + +from pload.errors import PloadError + +MAX_INDEX_BYTES = 16 * 1024 * 1024 +MAX_METADATA_BYTES = 8 * 1024 * 1024 + + +def _read_url(url, accept, maximum): + request = Request(url, headers={"Accept": accept, "User-Agent": "pload-metadata/1"}) + try: + with urlopen(request, timeout=30) as response: + length = response.headers.get("Content-Length") + if length and int(length) > maximum: + raise PloadError(f"metadata response is too large: {url}") + content = response.read(maximum + 1) + if len(content) > maximum: + raise PloadError(f"metadata response is too large: {url}") + return content, response.headers.get_content_type() + except (HTTPError, URLError, OSError, ValueError) as exc: + raise PloadError(f"cannot read package metadata from {url}: {exc}") from exc + + +class _SimpleHTML(HTMLParser): + def __init__(self): + super().__init__() + self.files = [] + + def handle_starttag(self, tag, attrs): + if tag.lower() != "a": + return + values = dict(attrs) + href = values.get("href") + if not href: + return + metadata = values.get("data-core-metadata") + if metadata is None: + metadata = values.get("data-dist-info-metadata") + self.files.append({ + "url": href, + "filename": unquote(urlsplit(href).path.rsplit("/", 1)[-1]), + "hashes": _fragment_hashes(urlsplit(href).fragment), + "requires-python": values.get("data-requires-python"), + "core-metadata": _metadata_hashes(metadata), + "yanked": "data-yanked" in values, + "size": None, + }) + + +def _fragment_hashes(fragment): + if "=" not in fragment: + return {} + algorithm, value = fragment.split("=", 1) + return {algorithm.lower(): value} + + +def _metadata_hashes(value): + if value is None or value is False: + return None + if value is True or str(value).lower() == "true": + return {} + if isinstance(value, dict): + return value + return _fragment_hashes(str(value)) + + +@dataclass(frozen=True) +class MetadataCandidate: + name: str + version: Version + source: str + artifact_url: str + filename: str + sha256: str + size: int + metadata_sha256: str + + +class MetadataIndex: + def __init__(self, sources, supported_tags, python_version, fetch=None): + self.sources = sources + self.supported_tags = set(supported_tags) + self.python_version = Version(python_version) + self.fetch = fetch or _read_url + self.project_cache = {} + self.dependency_cache = {} + self.source_metadata_capability = {} + + def prefetch(self, requirements): + names = sorted({canonicalize_name(item.name) for item in requirements}) + missing = [name for name in names if name not in self.project_cache] + if not missing: + return + with ThreadPoolExecutor(max_workers=min(8, len(missing))) as pool: + list(pool.map(self.candidates, missing)) + + def candidates(self, name): + key = canonicalize_name(name) + if key in self.project_cache: + return self.project_cache[key] + by_version = {} + metadata_missing = [] + source_errors = [] + for source_name, base_url in self.sources: + if self.source_metadata_capability.get(source_name) is False: + continue + project_url = base_url.rstrip("/") + "/" + key + "/" + try: + body, content_type = self.fetch( + project_url, + "application/vnd.pypi.simple.v1+json, text/html;q=0.2", + MAX_INDEX_BYTES, + ) + except PloadError as exc: + source_errors.append(f"{source_name}: {exc}") + continue + if content_type == "application/vnd.pypi.simple.v1+json" or body.lstrip().startswith(b"{"): + try: + files = json.loads(body.decode("utf-8"))["files"] + except (KeyError, TypeError, ValueError, UnicodeDecodeError) as exc: + raise PloadError(f"invalid Simple API response for {name}") from exc + else: + parser = _SimpleHTML() + try: + parser.feed(body.decode("utf-8")) + except UnicodeDecodeError as exc: + raise PloadError(f"invalid Simple API HTML response for {name}") from exc + files = parser.files + source_versions = {} + source_has_metadata = False + for record in files: + filename = record.get("filename", "") + try: + project, version, _, tags = parse_wheel_filename(filename) + except (InvalidWheelFilename, InvalidVersion): + continue + if canonicalize_name(project) != key or not tags.intersection(self.supported_tags): + continue + if record.get("yanked"): + continue + requires_python = record.get("requires-python") + if requires_python: + try: + if not SpecifierSet(requires_python).contains(self.python_version): + continue + except InvalidSpecifier: + continue + sha256 = (record.get("hashes") or {}).get("sha256") + raw_metadata = record.get("core-metadata") + if raw_metadata is None: + raw_metadata = record.get("dist-info-metadata") + metadata_hashes = _metadata_hashes(raw_metadata) + if not sha256: + continue + if metadata_hashes is None: + metadata_missing.append(filename) + continue + source_has_metadata = True + url = urljoin(project_url, record["url"]) + candidate = MetadataCandidate( + name=key, version=version, source=source_name, + artifact_url=url.split("#", 1)[0], filename=filename, + sha256=sha256, size=int(record.get("size") or 0), + metadata_sha256=metadata_hashes.get("sha256", ""), + ) + current = source_versions.get(version) + if current is None or (candidate.size or sys.maxsize) < (current.size or sys.maxsize): + source_versions[version] = candidate + self.source_metadata_capability[source_name] = source_has_metadata + if source_versions: + # Preserve source priority for duplicate versions while still + # retaining versions that exist only on a later fallback index. + for version, candidate in source_versions.items(): + by_version.setdefault(version, candidate) + if not by_version and metadata_missing: + raise PloadError( + f"index has compatible files for {name}, but does not expose independent " + "core metadata; refusing to download a wheel during plan. Add a " + "metadata-capable source such as https://pypi.org/simple" + ) + if not by_version and source_errors: + raise PloadError( + f"no configured index could provide metadata for {name}: " + + "; ".join(source_errors) + ) + result = sorted(by_version.values(), key=lambda item: item.version, reverse=True) + self.project_cache[key] = result + return result + + def dependencies(self, candidate, marker_environment): + if candidate in self.dependency_cache: + return self.dependency_cache[candidate] + url = candidate.artifact_url + ".metadata" + body, _ = self.fetch(url, "application/octet-stream", MAX_METADATA_BYTES) + if candidate.metadata_sha256: + actual = hashlib.sha256(body).hexdigest() + if actual != candidate.metadata_sha256: + raise PloadError(f"core metadata checksum mismatch for {candidate.filename}") + message = BytesParser(policy=compat32).parsebytes(body) + if canonicalize_name(message.get("Name", "")) != candidate.name: + raise PloadError(f"core metadata name mismatch for {candidate.filename}") + if message.get("Version") != str(candidate.version): + raise PloadError(f"core metadata version mismatch for {candidate.filename}") + requirements = [] + for value in message.get_all("Requires-Dist", []): + try: + requirement = Requirement(value) + except InvalidRequirement as exc: + raise PloadError(f"invalid Requires-Dist in {candidate.filename}: {value}") from exc + if requirement.url: + raise PloadError( + f"{candidate.filename} uses a direct-URL dependency that cannot be planned" + ) + if requirement.marker and not requirement.marker.evaluate(marker_environment): + continue + requirements.append(requirement) + self.dependency_cache[candidate] = requirements + self.prefetch(requirements) + return requirements + + +class _Provider: + def __init__(self, index, marker_environment): + self.index = index + self.marker_environment = marker_environment + + def identify(self, requirement_or_candidate): + return canonicalize_name(requirement_or_candidate.name) + + def get_preference(self, identifier, resolutions, candidates, information, backtrack_causes): + return (identifier not in resolutions, identifier) + + def find_matches(self, identifier, requirements, incompatibilities): + constraints = list(requirements[identifier]) + rejected = set(incompatibilities[identifier]) + return [ + candidate for candidate in self.index.candidates(identifier) + if candidate not in rejected + and all(requirement.specifier.contains(candidate.version) for requirement in constraints) + ] + + @staticmethod + def is_satisfied_by(requirement, candidate): + return requirement.specifier.contains(candidate.version) + + def get_dependencies(self, candidate): + return self.index.dependencies(candidate, self.marker_environment) + + @staticmethod + def narrow_requirement_selection( + identifiers, resolutions, candidates, information, backtrack_causes, + ): + """Keep every active criterion; ordering remains get_preference's job.""" + return identifiers + + +def resolve_metadata(requirements, sources, environment, supported_tags, fetch=None): + """Return exact packages and wheel metadata without fetching any wheel body.""" + parsed = [Requirement(value) for value in requirements] + marker_environment = default_environment() + python = environment["python"] + marker_environment.update({ + "python_full_version": python, + "python_version": ".".join(python.split(".")[:2]), + "implementation_name": environment.get("implementation", "cpython"), + "platform_system": environment.get("system") or platform.system(), + "platform_machine": environment.get("machine") or platform.machine(), + "extra": "", + }) + index = MetadataIndex(sources, supported_tags, python, fetch=fetch) + index.prefetch(parsed) + try: + result = Resolver(_Provider(index, marker_environment), BaseReporter()).resolve(parsed) + except ResolutionImpossible as exc: + raise PloadError("dependency constraints cannot be resolved from index metadata") from exc + packages = [] + for name, candidate in sorted(result.mapping.items()): + packages.append({ + "name": name, "version": str(candidate.version), + "sources": [candidate.source], + "artifact": [{ + "filename": candidate.filename, "sha256": candidate.sha256, + "tags": [str(tag) for tag in parse_wheel_filename(candidate.filename)[3]], + "repositories": [], "url": candidate.artifact_url, + "size": candidate.size, + "metadata_sha256": candidate.metadata_sha256, + }], + }) + return packages diff --git a/src/pload/resource_plan.py b/src/pload/resource_plan.py new file mode 100644 index 0000000..94e74c1 --- /dev/null +++ b/src/pload/resource_plan.py @@ -0,0 +1,165 @@ +"""Persistent, auditable acquisition plans produced without acquiring packages.""" + +import hashlib +import json +from copy import deepcopy +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - Python 3.8 + import tomli as tomllib + +from pload.errors import PloadError + + +class PlanSelection: + """Mutable route selection with deterministic reset and multi-step undo.""" + + def __init__(self, plan): + self.plan = plan + self._original = deepcopy(plan.get("packages", [])) + self._history = [] + + def routes(self, package_index): + package = self.plan["packages"][package_index] + return ([package["selected"]] if package.get("selected") else []) + list( + package.get("alternatives", []) + ) + + def decision_indices(self): + """Only direct user requirements need interactive route decisions.""" + return [ + index for index, package in enumerate(self.plan["packages"]) + if package.get("direct", True) + ] + + def select(self, package_index, route_index): + routes = self.routes(package_index) + if route_index < 0 or route_index >= len(routes): + raise IndexError("route index out of range") + self._history.append(deepcopy(self.plan["packages"])) + package = self.plan["packages"][package_index] + package["selected"] = routes[route_index] + package["alternatives"] = [ + route for index, route in enumerate(routes) if index != route_index + ] + self.plan["ready"] = all( + item.get("selected") for item in self.plan["packages"] + ) and self.plan["python"].get("status") != "unavailable" + + def undo(self): + if not self._history: + return False + self.plan["packages"] = self._history.pop() + return True + + def reset(self): + self._history.append(deepcopy(self.plan["packages"])) + self.plan["packages"] = deepcopy(self._original) + + +def plan_path(configuration_path): + return Path(configuration_path).parent / ".pload_plan.toml" + + +def file_digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def _quoted(value): + return json.dumps(str(value), ensure_ascii=False) + + +def dump_plan(configuration_path, configuration_sha256, lock_sha256, plan): + python = plan["python"] + lines = [ + "schema = 1", + f"configuration = {_quoted(Path(configuration_path).name)}", + f"configuration_sha256 = {_quoted(configuration_sha256)}", + f"lock_sha256 = {_quoted(lock_sha256)}", + f"name = {_quoted(plan['name'])}", + f"ready = {'true' if plan['ready'] else 'false'}", + "", + "[python]", + f"method = {_quoted(python['method'])}", + f"location = {_quoted(python['location'])}", + f"status = {_quoted(python['status'])}", + ] + for package in plan.get("packages", []): + selected = package.get("selected") or {} + rejection = (package.get("rejections") or [{}])[0] + reason = rejection.get("reason", "no resource satisfies the configuration") + lines += [ + "", + "[[package]]", + f"name = {_quoted(package['name'])}", + f"version = {_quoted(package['version'])}", + f"direct = {'true' if package.get('direct', True) else 'false'}", + f"method = {_quoted(selected.get('method', 'unavailable'))}", + f"location = {_quoted(selected.get('location', ''))}", + f"status = {_quoted(selected.get('status', 'unavailable'))}", + ] + if selected.get("measurement"): + lines.append(f"measurement = {_quoted(selected['measurement'])}") + if selected.get("estimated_seconds") is not None: + lines.append( + f"estimated_seconds = {float(selected['estimated_seconds'])}" + ) + if not selected: + lines.append(f"reason = {_quoted(reason)}") + if selected.get("fingerprint"): + lines.append(f"fingerprint = {_quoted(selected['fingerprint'])}") + artifact = selected.get("artifact") + if artifact: + lines += [ + "", + "[package.artifact]", + f"filename = {_quoted(artifact['filename'])}", + f"sha256 = {_quoted(artifact['sha256'])}", + ] + if artifact.get("url"): + lines.append(f"url = {_quoted(artifact['url'])}") + if artifact.get("size") is not None: + lines.append(f"size = {int(artifact['size'])}") + if artifact.get("metadata_sha256"): + lines.append( + f"metadata_sha256 = {_quoted(artifact['metadata_sha256'])}" + ) + return "\n".join(lines) + "\n" + + +def write_plan(configuration_path, configuration_sha256, lock_sha256, plan): + destination = plan_path(configuration_path) + temporary = destination.with_name(destination.name + ".tmp") + temporary.write_text( + dump_plan(configuration_path, configuration_sha256, lock_sha256, plan), + encoding="utf-8", + ) + temporary.replace(destination) + return destination + + +def load_plan(configuration_path, configuration_sha256, lock_sha256): + path = plan_path(configuration_path) + try: + with path.open("rb") as stream: + data = tomllib.load(stream) + except FileNotFoundError as exc: + raise PloadError( + f"acquisition plan is missing; run 'pload plan {Path(configuration_path).name}'" + ) from exc + except (OSError, tomllib.TOMLDecodeError) as exc: + raise PloadError(f"cannot read acquisition plan {path}: {exc}") from exc + if data.get("schema") != 1: + raise PloadError("unsupported acquisition plan schema") + if ( + data.get("configuration") != Path(configuration_path).name + or data.get("configuration_sha256") != configuration_sha256 + or data.get("lock_sha256") != lock_sha256 + ): + raise PloadError("acquisition plan is stale; run pload plan again") + packages = data.get("package", []) + if not isinstance(packages, list): + raise PloadError("invalid acquisition plan packages") + return path, data diff --git a/src/pload/snapshots.py b/src/pload/snapshots.py new file mode 100644 index 0000000..6aacb5f --- /dev/null +++ b/src/pload/snapshots.py @@ -0,0 +1,517 @@ +"""Portable package recipes, checked wheel bundles and optional remote storage. + +Only Python's standard library and the target interpreter's pip are required +locally. SSH remotes use OpenSSH; Git remotes use the user's existing Git. +""" + +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import tempfile +import uuid +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +from pload.errors import PloadError +from pload.managers.venv import VenvManager +from pload.settings import save_settings + +PROBE = """ +import hashlib, json, platform, sys, os, shutil, subprocess, sysconfig +from importlib import metadata +packages = [] +library_roots = [os.path.realpath(sysconfig.get_path(key)) + for key in ('purelib', 'platlib') if sysconfig.get_path(key)] +for dist in metadata.distributions(): + name = dist.metadata.get('Name') + if name and name.lower() not in ('pip', 'setuptools', 'wheel'): + record = dist.read_text('RECORD') or '' + locations = [os.path.realpath(str(dist.locate_file(item))) + for item in (dist.files or [])] + copyable = bool(locations) and all(any( + location == root or location.startswith(root + os.sep) + for root in library_roots) for location in locations) + packages.append({'name': name, 'version': dist.version, + 'direct_url': dist.read_text('direct_url.json'), + 'record_sha256': hashlib.sha256(record.encode()).hexdigest(), + 'copyable': copyable}) +accelerators = [] +nvidia_smi = shutil.which('nvidia-smi') +if nvidia_smi: + try: + result = subprocess.run([nvidia_smi, '--query-gpu=driver_version', + '--format=csv,noheader'], capture_output=True, text=True, + timeout=5) + if result.returncode == 0: + versions = sorted(set(line.strip() for line in result.stdout.splitlines() + if line.strip())) + accelerators.append({'kind': 'nvidia', 'driver_versions': versions}) + except (OSError, subprocess.SubprocessError): + pass +print(json.dumps({'python': platform.python_version(), + 'implementation': sys.implementation.name, 'system': platform.system(), + 'machine': platform.machine(), 'libc': list(platform.libc_ver()), + 'conda': os.path.isdir(os.path.join(sys.prefix, 'conda-meta')), + 'accelerators': accelerators, 'packages': packages})) +""" + +RESOURCE_PROBE = r""" +import hashlib, json, os, platform, sys +from importlib import metadata +requested = json.loads(sys.argv[1]) +packages = [] +for requested_name in requested: + try: + dist = metadata.distribution(requested_name) + except metadata.PackageNotFoundError: + continue + name = dist.metadata.get('Name') or requested_name + record = dist.read_text('RECORD') or '' + paths = [line.split(',', 1)[0].replace('\\', '/') for line in record.splitlines() + if line.strip()] + copyable = bool(paths) and all( + path and not path.startswith('/') and + os.path.normpath(path).replace('\\', '/') != '..' and + not os.path.normpath(path).replace('\\', '/').startswith('../') + for path in paths + ) + packages.append({'name': name, 'version': dist.version, + 'record_sha256': hashlib.sha256(record.encode()).hexdigest(), + 'copyable': copyable}) +print(json.dumps({'python': platform.python_version(), + 'implementation': sys.implementation.name, 'system': platform.system(), + 'machine': platform.machine(), 'packages': packages})) +""" + + +def execute(command, capture=True, cwd=None, timeout=None, input_text=None): + try: + result = subprocess.run( + command, capture_output=capture, text=True, cwd=cwd, check=False, + timeout=timeout, input=input_text, + ) + except subprocess.TimeoutExpired as exc: + raise PloadError(f"{command[0]} timed out after {timeout} seconds") from exc + except OSError as exc: + raise PloadError(f"cannot run {command[0]}: {exc}") from exc + if result.returncode: + detail = (result.stderr or result.stdout or "").strip() if capture else "" + raise PloadError(f"{command[0]} failed: {detail or result.returncode}") + return result.stdout.strip() if capture else "" + + +def safe_name(value): + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,99}", value or ""): + raise PloadError("use a name of 1–100 letters, numbers, dots, underscores or hyphens") + return value + + +def digest(path): + checksum = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + checksum.update(chunk) + return checksum.hexdigest() + + +def write_json(path, data): + Path(path).write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def read_json(path): + try: + return json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise PloadError(f"cannot read {path}: {exc}") from exc + + +def probe(python): + return json.loads(execute([str(python), "-c", PROBE])) + + +def probe_resources(python, package_names): + """Inspect only locked packages needed by planning, without hashing file bodies.""" + return json.loads(execute([ + str(python), "-c", RESOURCE_PROBE, + json.dumps(sorted(set(package_names))), + ])) + + +def public_index(url): + """Store useful provenance without passwords, tokens or URL query strings.""" + if not url: + return None + parsed = urlsplit(url) + return urlunsplit((parsed.scheme, parsed.netloc.rsplit("@", 1)[-1], parsed.path, "", "")) + + +def validate_bundle(path): + path = Path(path) + if path.is_symlink() or (path / "snapshot.json").is_symlink(): + raise PloadError("snapshot must not be a symbolic link") + data = read_json(path / "snapshot.json") + if not isinstance(data, dict) or data.get("schema") != 1: + raise PloadError("unsupported snapshot schema") + files = data.get("files") + if not isinstance(files, dict) or "requirements.txt" not in files: + raise PloadError("snapshot has no requirements checksum") + if not isinstance(data.get("runtime"), dict): + raise PloadError("snapshot has no runtime metadata") + for name, checksum in files.items(): + if name != "requirements.txt" and not re.fullmatch( + r"wheels/[A-Za-z0-9][A-Za-z0-9_.+!-]*\.whl", name + ): + raise PloadError(f"invalid snapshot file: {name}") + item = path / name + if any(part.is_symlink() for part in (item, item.parent)): + raise PloadError(f"snapshot contains a symbolic link: {name}") + if not item.is_file() or digest(item) != checksum: + raise PloadError(f"snapshot checksum mismatch or missing file: {name}") + # Snapshots contain only exact pins, never executable requirements directives. + for line in (path / "requirements.txt").read_text(encoding="utf-8").splitlines(): + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*==[A-Za-z0-9][A-Za-z0-9_.+!-]*", line): + raise PloadError(f"snapshot requires an exact package pin: {line!r}") + return data + + +class SnapshotManager: + def __init__(self, config): + self.config = config + self.root = config.home / "snapshots" + self.wheels = config.home / "cache" / "wheels" + + def path(self, name): + if name == "objects": + raise PloadError("'objects' is reserved for repository wheel storage") + return self.root / safe_name(name) + + def export(self, name, environment=None, python=None, bundle=None, index=None, links=None): + target = self.path(name) + if target.exists(): + raise PloadError(f"snapshot already exists: {name}; choose a new name") + if python: + candidate = Path(python).expanduser() + # Resolving bin/python's symlink escapes a venv and probes the base Python. + interpreter = candidate.absolute() if candidate.is_file() else self.config.get_python_path(python) + else: + env = VenvManager(self.config).resolve_existing(environment or ".") + interpreter = env / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + runtime = probe(interpreter) + if runtime.get("conda"): + raise PloadError("Conda environments contain native dependencies that pip cannot reproduce; " + "use conda export for the full environment, or export a pip requirements " + "file explicitly and pass it to pload restore") + pins = [] + exact_wheels = {} + for package in sorted(runtime.pop("packages"), key=lambda p: p["name"].lower()): + pin = f"{package['name']}=={package['version']}" + if package["direct_url"]: + provenance = json.loads(package["direct_url"]) + archive = provenance.get("archive_info", {}) + checksum = archive.get("hashes", {}).get("sha256") + if not checksum and archive.get("hash", "").startswith("sha256="): + checksum = archive["hash"].split("=", 1)[1] + normalized = re.sub(r"[-_.]+", "-", package["name"].lower()) + selected = {re.sub(r"[-_.]+", "-", item.lower()) + for item in (bundle or "").split(",")} + if checksum and (bundle == "all" or normalized in selected): + for directory in [self.wheels] + [Path(p).expanduser() for p in links or []]: + for wheel in directory.glob("*.whl"): + wheel_name = re.sub(r"[-_.]+", "-", wheel.name.split("-")[0].lower()) + if wheel_name != normalized: + continue + if digest(wheel) == checksum: + exact_wheels[pin] = str(wheel.absolute()) + break + if pin in exact_wheels: + break + if pin not in exact_wheels: + raise PloadError( + f"{package['name']} is a direct/local install: bundle its original wheel " + "using --bundle and --find-links (matching installation SHA-256 required). " + "Editable/VCS sources must first be built and installed as wheels." + ) + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*==[A-Za-z0-9][A-Za-z0-9_.+!-]*", pin): + raise PloadError(f"invalid installed package metadata: {pin!r}") + pins.append(pin) + self.root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".export-", dir=str(self.root)) as temporary: + stage = Path(temporary) / name + stage.mkdir() + requirements = stage / "requirements.txt" + requirements.write_text("".join(pin + "\n" for pin in pins), encoding="utf-8") + files = {"requirements.txt": digest(requirements)} + if bundle: + selected = {re.sub(r"[-_.]+", "-", p.lower()) for p in bundle.split(",")} + download = pins if bundle == "all" else [ + pin for pin in pins + if re.sub(r"[-_.]+", "-", pin.split("==")[0].lower()) in selected + ] + found = {re.sub(r"[-_.]+", "-", pin.split("==")[0].lower()) for pin in download} + if bundle != "all" and selected - found: + raise PloadError("bundled packages not installed: " + ", ".join(selected - found)) + wheel_dir = stage / "wheels" + wheel_dir.mkdir() + self.wheels.mkdir(parents=True, exist_ok=True) + if download: + download = [exact_wheels.get(pin, pin) for pin in download] + command = [str(interpreter), "-m", "pip", "download", "--only-binary=:all:", + "--no-deps", "--dest", str(wheel_dir), + "--find-links", str(self.wheels)] + for link in links or []: + command.extend(["--find-links", str(Path(link).expanduser().resolve())]) + # First try without networking; fall back to pip's normal HTTP/wheel cache. + try: + execute(command + ["--no-index"] + download) + except PloadError: + if index or self.config.settings.get("pip_index"): + command += ["--index-url", index or self.config.settings["pip_index"]] + execute(command + download, capture=False) + for wheel in sorted(wheel_dir.glob("*.whl")): + files["wheels/" + wheel.name] = digest(wheel) + cached = self.wheels / wheel.name + if cached.exists() and digest(cached) != digest(wheel): + raise PloadError(f"different wheel with same filename in cache: {wheel.name}") + if not cached.exists(): + shutil.copyfile(wheel, cached) + write_json(stage / "snapshot.json", { + "schema": 1, "name": name, "runtime": runtime, "files": files, + "bundle": bundle or "none", + "index_url": public_index(index or self.config.settings.get("pip_index")), + }) + validate_bundle(stage) + stage.rename(target) + return target + + def restore(self, source, name, version=None, offline=False, portable=False, index=None): + candidate = Path(source).expanduser() + path = candidate if candidate.exists() else self.path(source) + is_recipe = path.is_file() + data = None if is_recipe else validate_bundle(path) + if offline and is_recipe: + raise PloadError("offline mode requires a validated snapshot, not an arbitrary requirements file") + if data: + runtime = data["runtime"] + version = version or ".".join(runtime.get("python", "").split(".")[:2]) + interpreter = self.config.get_python_path(version) + if data: + actual = probe(interpreter) + for key in ("implementation", "system", "machine"): + if actual.get(key) != runtime.get(key) and not portable: + raise PloadError(f"{key} mismatch: {runtime.get(key)} vs {actual.get(key)}; " + "use --portable for a fresh platform-specific resolution") + if actual["python"].split(".")[:2] != runtime.get("python", "").split(".")[:2]: + raise PloadError("Python major/minor must match the snapshot") + if portable and offline: + raise PloadError("--portable cannot be combined with --offline") + if offline and data.get("bundle") != "all": + raise PloadError("offline restoration requires a snapshot exported with --bundle all") + requirements = path if is_recipe else path / "requirements.txt" + self.root.mkdir(parents=True, exist_ok=True) + env = VenvManager(self.config).create_venv(version=str(interpreter), name=name) + command = self.config.get_pip_command(env) + ["install", "-r", str(requirements.resolve())] + if data and not portable: + # Only checksum-verified artifacts may be used from a snapshot. + with tempfile.TemporaryDirectory(prefix=".restore-", dir=str(self.root)) as stage: + artifacts = [] + for filename in data["files"]: + if filename.startswith("wheels/"): + wheel = Path(stage) / Path(filename).name + shutil.copyfile(path / filename, wheel) + if digest(wheel) != data["files"][filename]: + raise PloadError("snapshot changed during restoration") + # Old pip records hashes only when supplied in the file URL. + artifacts.append(wheel.resolve().as_uri() + "#sha256=" + + data["files"][filename]) + if artifacts: + # Install exact verified files first so an index cannot replace them. + execute(self.config.get_pip_command(env) + + ["install", "--no-index", "--no-deps"] + artifacts, capture=False) + command += ["--find-links", stage] + self._install(command, offline, index) + else: + self._install(command, offline, index) + execute(self.config.get_pip_command(env) + ["check"], capture=False) + return env + + def _install(self, command, offline, index): + if self.wheels.is_dir(): + command += ["--find-links", str(self.wheels)] + if offline: + command += ["--no-index", "--no-deps"] + elif index or self.config.settings.get("pip_index"): + command += ["--index-url", index or self.config.settings["pip_index"]] + try: + if not offline: + try: + execute(command + ["--no-index", "--only-binary=:all:"]) + return + except PloadError: + pass + execute(command, capture=False) + except PloadError as exc: + raise PloadError(f"restore failed; the new environment is kept for inspection: {exc}") + + +class RepositoryManager: + def __init__(self, config): + self.config = config + self.snapshots = SnapshotManager(config) + + def repositories(self): + return self.config.settings.get("repositories", {}) + + def add(self, name, location, kind): + safe_name(name) + repositories = dict(self.repositories()) + if name in repositories: + raise PloadError("repository exists; remove its configuration before replacing it") + if kind == "ssh": + self.ssh_location(location) + elif kind == "local": + location = str(Path(location).expanduser().resolve()) + elif location.startswith("-") or not location: + raise PloadError("invalid Git location") + repositories[name] = {"kind": kind, "location": location} + self.config.settings["repositories"] = repositories + save_settings(self.config.home, self.config.settings) + + def remove(self, name): + repositories = dict(self.repositories()) + if name not in repositories: + raise PloadError(f"unknown repository: {name}") + del repositories[name] + self.config.settings["repositories"] = repositories + save_settings(self.config.home, self.config.settings) + + @staticmethod + def ssh_location(location): + host, separator, path = location.partition(":") + if (not separator or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.@-]*", host) + or not re.fullmatch(r"/[A-Za-z0-9_./-]+", path) + or ".." in path.split("/") or path == "/"): + raise PloadError("SSH location must be HOST:/absolute/directory (no spaces or '..')") + return host, path.rstrip("/") + + def transfer(self, repo, name, push=False): + safe_name(name) + configured = self.repositories().get(repo) + if not configured: + raise PloadError(f"unknown repository: {repo}") + target = self.snapshots.path(name) + if not push and target.exists(): + raise PloadError("local snapshot exists; immutable snapshots are never overwritten") + if push: + data = validate_bundle(target) + self.snapshots.root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".transfer-", dir=str(self.snapshots.root)) as temp: + temp = Path(temp) + kind, location = configured["kind"], configured["location"] + if kind == "ssh": + host, root = self.ssh_location(location) + remote = root + "/" + name + if push: + # Publish a complete staging directory atomically; refuse overwrites. + stage = root + "/.upload-" + uuid.uuid4().hex + execute(["ssh", "-o", "BatchMode=yes", host, + f"mkdir -p {shlex.quote(stage + '/wheels')}"]) + for filename in ["snapshot.json"] + list(data["files"]): + if filename.startswith("wheels/"): + obj = root + "/objects/" + data["files"][filename] + execute(["ssh", "-o", "BatchMode=yes", host, + f"mkdir -p {shlex.quote(root + '/objects')}"]) + exists = execute(["ssh", "-o", "BatchMode=yes", host, + f"if test -f {shlex.quote(obj)}; then printf yes; fi"]) + if exists != "yes": + pending = obj + "." + uuid.uuid4().hex + execute(["scp", "-q", "-o", "BatchMode=yes", str(target / filename), + host + ":" + pending]) + execute(["ssh", "-o", "BatchMode=yes", host, + f"mv -n {shlex.quote(pending)} {shlex.quote(obj)}"]) + remote_hash = execute(["ssh", "-o", "BatchMode=yes", host, + f"sha256sum {shlex.quote(obj)}"]) + if remote_hash.split()[0] != data["files"][filename]: + raise PloadError("remote wheel checksum mismatch; snapshot not published") + execute(["ssh", "-o", "BatchMode=yes", host, + f"ln {shlex.quote(obj)} {shlex.quote(stage + '/' + filename)}"]) + else: + execute(["scp", "-q", "-o", "BatchMode=yes", str(target / filename), + host + ":" + stage + "/" + filename]) + execute(["ssh", "-o", "BatchMode=yes", host, + (f"test ! -e {shlex.quote(remote)} && " + f"mv -T {shlex.quote(stage)} {shlex.quote(remote)}")]) + else: + stage = temp / name + stage.mkdir() + execute(["scp", "-q", "-o", "BatchMode=yes", + host + ":" + remote + "/snapshot.json", str(stage / "snapshot.json")]) + data = read_json(stage / "snapshot.json") + files = data.get("files", {}) if isinstance(data, dict) else {} + if not isinstance(files, dict): + raise PloadError("invalid remote file list") + for filename in files: + if filename != "requirements.txt" and not re.fullmatch( + r"wheels/[A-Za-z0-9][A-Za-z0-9_.+!-]*\.whl", filename + ): + raise PloadError("invalid remote filename") + (stage / filename).parent.mkdir(parents=True, exist_ok=True) + cached = self.snapshots.wheels / Path(filename).name + if filename.startswith("wheels/") and cached.is_file() and digest(cached) == files[filename]: + shutil.copyfile(cached, stage / filename) + else: + execute(["scp", "-q", "-o", "BatchMode=yes", + host + ":" + remote + "/" + filename, str(stage / filename)]) + validate_bundle(stage) + self.cache_bundle(stage, data) + stage.rename(target) + return + if kind == "git": + if push and any(f.startswith("wheels/") for f in data["files"]): + raise PloadError("Git repositories store recipes only; use SSH/local for wheels") + checkout = temp / "git" + execute(["git", "clone", "--", location, str(checkout)]) + remote = checkout / "snapshots" / name + elif kind == "local": + remote = Path(location) / name + else: + raise PloadError(f"unsupported repository kind: {kind}") + if push: + if remote.exists(): + raise PloadError("remote snapshot already exists; choose a new snapshot name") + remote.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".publish-", dir=str(remote.parent)) as pending: + stage = Path(pending) / name + stage.mkdir() + for filename in ["snapshot.json"] + list(data["files"]): + (stage / filename).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(target / filename, stage / filename) + validate_bundle(stage) + stage.rename(remote) + if kind == "git": + execute(["git", "add", "--", "snapshots/" + name], cwd=checkout) + execute(["git", "commit", "-m", "Add pload snapshot " + name], cwd=checkout) + execute(["git", "push", "origin", "HEAD"], cwd=checkout) + else: + data = validate_bundle(remote) + stage = temp / name + stage.mkdir() + for filename in ["snapshot.json"] + list(data["files"]): + (stage / filename).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(remote / filename, stage / filename) + validate_bundle(stage) + self.cache_bundle(stage, data) + stage.rename(target) + + def cache_bundle(self, path, data): + self.snapshots.wheels.mkdir(parents=True, exist_ok=True) + for filename, checksum in data["files"].items(): + if filename.startswith("wheels/"): + cached = self.snapshots.wheels / Path(filename).name + if cached.exists() and digest(cached) != checksum: + raise PloadError(f"cache filename collision: {cached.name}") + if not cached.exists(): + shutil.copyfile(path / filename, cached) diff --git a/src/pload/ui.py b/src/pload/ui.py new file mode 100644 index 0000000..9ba5320 --- /dev/null +++ b/src/pload/ui.py @@ -0,0 +1,248 @@ +"""One terminal language for every pload command. + +Keep presentation decisions here so commands cannot invent their own colours, +symbols, tables, or prompt behaviour. Rich owns output; Questionary owns +interactive input. Both automatically degrade when output is redirected. +""" + +from __future__ import annotations + +import argparse +import difflib +import os +import re +import sys +from contextlib import contextmanager + +import questionary +from prompt_toolkit.styles import Style as PromptStyle +from rich import box +from rich.console import Console +from rich.table import Table +from rich.text import Text + +ACCENT = "cyan" +SUCCESS = "green" +WARNING = "yellow" +DANGER = "red" +MUTED = "dim" + +PROMPT_STYLE = PromptStyle.from_dict( + { + "qmark": "fg:#22d3ee bold", + "question": "bold", + "answer": "fg:#34d399 bold", + "pointer": "fg:#22d3ee bold", + "highlighted": "fg:#22d3ee bold", + "selected": "fg:#34d399", + "instruction": "fg:#7c8499", + "text": "", + "disabled": "fg:#7c8499 italic", + } +) +PLAIN_PROMPT_STYLE = PromptStyle.from_dict( + {name: "" for name in ( + "qmark", "question", "answer", "pointer", "highlighted", + "selected", "instruction", "text", "disabled", + )} +) + + +def _prompt_style(): + return PLAIN_PROMPT_STYLE if os.environ.get("NO_COLOR") is not None else PROMPT_STYLE + + +def _color_system(): + return None if os.environ.get("NO_COLOR") is not None else "auto" + + +def glyph(value, fallback, *, encoding=None): + """Use Unicode only when the active terminal can actually encode it.""" + encoding = encoding or getattr(sys.stdout, "encoding", None) or "utf-8" + try: + value.encode(encoding) + except (LookupError, UnicodeEncodeError): + return fallback + return value + + +def navigation_hint(action="select"): + return f"{glyph('↑/↓', 'up/down')} move • enter {action}" + + +class ArgumentParser(argparse.ArgumentParser): + """Argparse with the same compact error language as the rest of pload.""" + + def error(self, message): + suggestion = None + if "invalid choice:" in message and " (choose from " in message: + summary, choices = message.split(" (choose from ", 1) + candidates = [ + item.strip().strip("'\"") + for item in choices.rstrip(")").split(",") + ] + invalid = re.findall(r"invalid choice: '([^']+)'", summary) + if invalid: + matches = difflib.get_close_matches( + invalid[0], candidates, n=1, cutoff=0.5, + ) + suggestion = matches[0] if matches else None + message = summary + error(message) + if suggestion: + console(stderr=True).print(f"[dim]Did you mean '{suggestion}'?[/]") + console(stderr=True).print(f"[dim]Try '{self.prog} -h' for help.[/]") + raise SystemExit(2) + + +def console(*, stderr=False): + return Console(stderr=stderr, highlight=False, color_system=_color_system()) + + +def is_interactive(): + return sys.stdin.isatty() and sys.stdout.isatty() + + +def table(*columns, expand=False): + """Build the compact table used throughout pload.""" + result = Table( + box=box.SIMPLE_HEAD, + header_style="bold cyan", + show_edge=False, + pad_edge=False, + expand=expand, + ) + for heading, style, kwargs in columns: + result.add_column(heading, style=style, **kwargs) + return result + + +def heading(title, subtitle=None, *, output=None): + output = output or console() + output.print() + output.print(Text(title, style="bold cyan")) + if subtitle: + output.print(Text(subtitle, style="dim")) + + +def logo(*, output=None): + """Render the single pload wordmark used by landing and setup screens.""" + output = output or console() + output.print(Text( + """ ____ _ _ + | _ \\| | ___ __ _ __| | + | |_) | |/ _ \\ / _` |/ _` | + | __/| | (_) | (_| | (_| | + |_| |_|\\___/ \\__,_|\\__,_|""", + style="bold cyan", + )) + + +def success(message, *, detail=None, output=None): + output = output or console() + mark = glyph("✓ ", "[ok] ", encoding=output.encoding) + output.print(Text.assemble((mark, "bold green"), (message, "white"))) + if detail: + output.print(f" [dim]{detail}[/]") + + +def info(message, *, output=None): + (output or console()).print(Text.assemble(("• ", "cyan"), (message, "white"))) + + +def warning(message, *, output=None): + (output or console(stderr=True)).print( + Text.assemble(("! ", "bold yellow"), (message, "yellow")) + ) + + +def error(message, *, output=None): + (output or console(stderr=True)).print( + Text.assemble(("error: ", "bold red"), (message, "red")) + ) + + +@contextmanager +def status(message): + output = console(stderr=True) + if output.is_terminal and os.environ.get("PLOAD_NO_PROGRESS") is None: + with output.status(f"[cyan]{message}[/]", spinner="dots", spinner_style="cyan") as live: + yield lambda value: live.update(f"[cyan]{value}[/]") + else: + output.print(f"[dim]{message}[/]") + yield lambda value: None + + +def select(message, choices, *, default=None, instruction=None, hotkeys=None): + """Choose with normal cursor keys, with a numbered fallback for tests/pipes.""" + hotkeys = {str(key).lower(): value for key, value in (hotkeys or {}).items()} + normalized = [ + item if isinstance(item, questionary.Choice) else questionary.Choice(str(item), value=item) + for item in choices + ] + if is_interactive(): + instruction = instruction or navigation_hint() + question = questionary.select( + message, + choices=normalized, + default=default, + instruction=instruction, + pointer=glyph("❯", ">"), + qmark="?", + style=_prompt_style(), + use_shortcuts=False, + use_arrow_keys=True, + ) + for key, value in hotkeys.items(): + def register(binding, result): + @question.application.key_bindings.add(binding, eager=True) + def choose_action(event): + event.app.exit(result=result) + + register(key, value) + return question.unsafe_ask() + + default_index = 0 + values = [item.value for item in normalized] + if default in values: + default_index = values.index(default) + print(f"\n{message}") + for index, item in enumerate(normalized, 1): + suffix = " (default)" if index - 1 == default_index else "" + print(f" {index}) {item.title}{suffix}") + while True: + answer = input(f"Choose [1-{len(normalized)}] [{default_index + 1}]: ").strip() + if not answer: + return normalized[default_index].value + if answer.lower() in hotkeys: + return hotkeys[answer.lower()] + if answer.isdigit() and 1 <= int(answer) <= len(normalized): + return normalized[int(answer) - 1].value + + +def text(message, *, default=""): + if is_interactive(): + return questionary.text( + message, + default=str(default), + qmark="?", + style=_prompt_style(), + ).unsafe_ask().strip() + suffix = f" [{default}]" if default else "" + value = input(f"{message}{suffix}: ").strip() + return value or str(default) + + +def confirm(message, *, default=False): + if is_interactive(): + return questionary.confirm( + message, + default=default, + qmark="?", + style=_prompt_style(), + ).unsafe_ask() + suffix = "Y/n" if default else "y/N" + answer = input(f"{message} [{suffix}]: ").strip().lower() + if not answer: + return default + return answer in {"y", "yes"} diff --git a/tests/test_cli.py b/tests/test_cli.py index 9171c87..cee7cfa 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,10 @@ +import io + import pytest +from rich.console import Console +import pload.cli as cli_module +import pload.declarative as declarative_module from pload.cli import build_parser, main, shell_script @@ -124,6 +129,19 @@ def test_simple_commands_have_no_alias_but_long_options_have_short_forms(): raise AssertionError("new must not have a command alias") +def test_parser_errors_are_compact_and_actionable(capsys): + with pytest.raises(SystemExit) as error: + build_parser().parse_args(["lisst"]) + + output = capsys.readouterr().err + assert error.value.code == 2 + assert "error:" in output + assert "invalid choice" in output + assert "Did you mean 'list'?" in output + assert "Try 'pload -h' for help." in output + assert not output.startswith("usage:") + + def test_description_short_option_does_not_conflict_with_detailed_help(): args = build_parser().parse_args(["new", "-m", "Data tools"]) @@ -152,7 +170,7 @@ def test_brief_and_detailed_help_are_distinct(capsys): assert "$ pload new -n data" in detailed assert "What changes on disk" in detailed assert "PLOAD_HOME/runtime" in detailed - assert "Related options" in detailed + assert "Options" in detailed def test_command_specific_detailed_help_uses_d_flag(capsys): @@ -219,6 +237,109 @@ def test_no_arguments_show_welcome_and_simple_usage(capsys): assert output.index("Simple usage") < output.index("Global options") +def test_apply_prints_saved_plan_before_package_progress(tmp_path, monkeypatch, capsys): + events = [] + + class FakeManager: + def __init__(self, config): + pass + + def saved_plan(self, path): + events.append("loaded") + return {"name": "demo"} + + def apply(self, path, name, offline, progress, target=None): + events.append("apply") + progress("PACKAGE\t1/1\tindex-exact\tnumpy==2.0.2\t16777216") + progress( + "SOURCE\tnumpy==2.0.2\t" + "downloading from https://files.invalid/numpy.whl" + ) + progress("DOWNLOAD\tnumpy==2.0.2\t8388608\t16777216") + progress("PACKAGE_DONE\tnumpy==2.0.2\tverified") + return tmp_path / "demo" + + monkeypatch.setattr(declarative_module, "DeclarativeEnvironmentManager", FakeManager) + monkeypatch.setattr( + cli_module, "print_declarative_plan", + lambda plan: events.append("printed-plan"), + ) + + assert main(["apply", str(tmp_path / "pload.toml")]) == 0 + + assert events == ["loaded", "printed-plan", "apply"] + output = capsys.readouterr().out + assert "[index-exact]" in output + assert "numpy==2.0.2" in output + assert "downloading from https://files.invalid/numpy.whl" not in output + assert "8.0 MiB / 16.0 MiB" in output + + +def test_apply_progress_uses_one_adaptive_rich_row_per_package(): + stream = io.StringIO() + console = Console(file=stream, force_terminal=True, color_system=None, width=160) + + with cli_module.ApplyProgress(console) as progress: + progress("PACKAGE\t1/1\tindex-exact\tnumpy==2.0.2\t16777216") + progress( + "SOURCE\tnumpy==2.0.2\t" + "downloading from https://files.invalid/numpy.whl" + ) + progress("DOWNLOAD\tnumpy==2.0.2\t8388608\t16777216") + + rendered = stream.getvalue() + assert "[index-exact]" in rendered + assert "numpy==2.0.2" in rendered + progress_line = next(line for line in rendered.splitlines() if "8.0 MiB" in line) + assert "[" in progress_line and "]" in progress_line + # Rich falls back to ASCII on legacy Windows consoles even when a terminal + # is forced, while Unix terminals use the Unicode heavy bar. + assert "━" in progress_line or "-" in progress_line + assert "░" not in rendered + assert "8.0 MiB / 16.0 MiB" in rendered + assert "downloading from" not in rendered + + +def test_environment_copy_progress_finishes_green(): + stream = io.StringIO() + console = Console( + file=stream, force_terminal=True, color_system="standard", no_color=False, + width=120, height=5, _environ={}, + ) + + with cli_module.ApplyProgress(console) as progress: + progress("PACKAGE\t1/1\tenvironment-copy\tdemo==1.0\t0") + progress("DOWNLOAD\tdemo==1.0\t0\t100") + progress("DOWNLOAD\tdemo==1.0\t100\t100") + progress("PACKAGE_DONE\tdemo==1.0\tcopied") + + assert "\x1b[32m" in stream.getvalue() + + +def test_apply_local_targets_venv_beside_manifest(tmp_path, monkeypatch): + captured = {} + manifest = tmp_path / "project" / "pload.toml" + + class FakeManager: + def __init__(self, config): + pass + + def saved_plan(self, path): + return {"packages": []} + + def apply(self, path, name, offline, progress, target=None): + captured["target"] = target + return target + + monkeypatch.setattr( + declarative_module, "DeclarativeEnvironmentManager", FakeManager + ) + monkeypatch.setattr(cli_module, "print_declarative_plan", lambda plan: None) + + assert main(["apply", str(manifest), "--local"]) == 0 + assert captured["target"] == manifest.parent / ".venv" + + def test_new_without_options_uses_guided_creation(tmp_path, monkeypatch): captured = {} @@ -229,6 +350,7 @@ def test_new_without_options_uses_guided_creation(tmp_path, monkeypatch): "name": "guided", "description": "Created interactively", "requirements": None, + "package_strategy": None, }, ) monkeypatch.setattr( @@ -237,7 +359,7 @@ def test_new_without_options_uses_guided_creation(tmp_path, monkeypatch): ) monkeypatch.setattr( "pload.cli.DependencyManager.install_dependencies", - lambda self, path, requirements, channel: None, + lambda self, path, requirements, channel, strategy: None, ) assert main(["-H", str(tmp_path / "home"), "new"]) == 0 diff --git a/tests/test_declarative.py b/tests/test_declarative.py new file mode 100644 index 0000000..c9d7923 --- /dev/null +++ b/tests/test_declarative.py @@ -0,0 +1,1232 @@ +import hashlib +import io +import platform +import shutil +import sys +import zipfile +from pathlib import Path + +import pytest +from packaging.tags import platform_tags + +import pload.declarative as declarative_module +from pload.cli import build_parser, main, shell_script +from pload.declarative import ( + ArtifactRepository, + DeclarativeEnvironmentManager, + dump_lock, + dump_manifest, + load_lock, + load_manifest, + lock_path, + validate_manifest, +) +from pload.errors import PloadError +from pload.managers.platform import ConfigManager +from pload.managers.venv import VenvManager +from pload.resource_plan import file_digest, write_plan +from pload.settings import save_settings +from pload.snapshots import digest, execute + + +def tiny_wheel(directory, tag="py3-none-any", distribution="pload_demo", version="1.0"): + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{distribution}-{version}-{tag}.whl" + module = distribution.replace("-", "_") + project = distribution.replace("_", "-") + with zipfile.ZipFile(path, "w") as wheel: + wheel.writestr(f"{module}.py", "answer = 42\n") + wheel.writestr( + f"{distribution}-{version}.dist-info/METADATA", + f"Metadata-Version: 2.1\nName: {project}\nVersion: {version}\n", + ) + wheel.writestr( + f"{distribution}-{version}.dist-info/WHEEL", + f"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: {tag}\n", + ) + wheel.writestr(f"{distribution}-{version}.dist-info/RECORD", "") + return path + + +def simple_manifest(mode="compatible"): + return { + "schema": 1, + "name": "demo", + "environment": { + "python": "3.12.1", "implementation": "cpython", + "system": "Linux", "machine": "x86_64", + "dependencies": ["demo==1.0"], + }, + "policy": { + "reproducibility": mode, "network": "allow", + "source_build": "fallback", "publish_missing_artifacts": True, + "repositories": [], + }, + "sources": {"default": {"kind": "index", "url": "https://pypi.org/simple"}}, + "repositories": {}, + "capabilities": {}, + "package": [{ + "name": "demo", "version": "1.0", "sources": ["default"], "artifact": [], + }], + } + + +def write_configuration(path, data, with_lock=True): + path.write_text(dump_manifest(data), encoding="utf-8") + if with_lock and data.get("package"): + locked = dict(data) + locked["resolved_dependencies"] = [ + f"{item['name']}=={item['version']}" for item in data["package"] + ] + lock_path(path).write_text(dump_lock(path, locked), encoding="utf-8") + + +def plan_and_save(manager, path, **kwargs): + plan = manager.plan(path, **kwargs) + manager.save_plan(path, plan) + return plan + + +def test_manifest_roundtrip(tmp_path): + data = simple_manifest() + path = tmp_path / "pload.toml" + write_configuration(path, data) + loaded_path, loaded = load_manifest(path) + assert loaded_path == path + assert loaded["environment"] == data["environment"] + assert loaded["package"] == [] + _, locked, current = load_lock(path, loaded) + assert current is True + assert locked["package"] == data["package"] + + +def test_manifest_roundtrip_with_automatic_plan_and_package_override(tmp_path): + data = simple_manifest() + data["plan"] = { + "mode": "auto", + "packages": { + "numpy": {"method": "index-exact", "location": "default"}, + }, + } + path = tmp_path / "pload.toml" + write_configuration(path, data) + + _, loaded = load_manifest(path) + + assert loaded["plan"] == data["plan"] + + +def test_manifest_rejects_invalid_planning_mode_and_route(): + data = simple_manifest() + data["plan"] = {"mode": "always-guess"} + with pytest.raises(PloadError, match="plan.mode"): + validate_manifest(data) + + data["plan"] = {"mode": "auto", "packages": {"demo": {"method": "magic"}}} + with pytest.raises(PloadError, match="plan.packages.demo.method"): + validate_manifest(data) + + +def test_lock_migrates_legacy_embedded_lock_without_resolving(tmp_path, monkeypatch): + data = simple_manifest("exact") + path = tmp_path / "pload.toml" + legacy = dump_manifest(data) + """ +[[package]] +name = "demo" +version = "1.0" +sources = ["default"] +""" + path.write_text(legacy, encoding="utf-8") + monkeypatch.setattr( + declarative_module, "execute", + lambda *args, **kwargs: pytest.fail("legacy migration invoked the resolver"), + ) + + result = DeclarativeEnvironmentManager(ConfigManager(home=tmp_path / "home")).lock(path) + + assert result["migrated"] is True + assert "[[package]]" not in path.read_text(encoding="utf-8") + _, configuration = load_manifest(path) + _, locked, current = load_lock(path, configuration) + assert current is True + assert locked["package"][0]["name"] == "demo" + + +def test_manifest_accepts_unpinned_requirements_but_rejects_direct_urls(): + data = simple_manifest() + data["environment"]["dependencies"] = ["demo>=1"] + validate_manifest(data) + data["environment"]["dependencies"] = ["demo @ https://example.com/demo.whl"] + with pytest.raises(PloadError, match="direct-URL"): + validate_manifest(data) + + +def test_manifest_rejects_secrets_in_shareable_sources(): + data = simple_manifest() + data["sources"]["default"]["url"] = "https://user:secret@example.com/simple?token=x" + with pytest.raises(PloadError, match="credentials"): + validate_manifest(data) + + +def test_manifest_rejects_dangling_provider_references(): + data = simple_manifest() + data["package"][0]["sources"] = ["missing"] + with pytest.raises(PloadError, match="unknown source"): + validate_manifest(data) + + +def test_manifest_allows_dependency_edits_to_make_the_package_lock_stale(): + data = simple_manifest() + data["environment"]["dependencies"] = ["other==1.0"] + validate_manifest(data) + + +def test_manifest_rejects_unknown_policy_repository(): + data = simple_manifest() + data["policy"]["repositories"] = ["missing"] + with pytest.raises(PloadError, match="unknown repositories"): + validate_manifest(data) + + +def test_compatible_plan_uses_index_and_offline_reports_missing(tmp_path): + path = tmp_path / "pload.toml" + write_configuration(path, simple_manifest()) + manager = DeclarativeEnvironmentManager(ConfigManager(home=tmp_path / "home")) + assert manager.plan(path)["packages"][0]["selected"]["method"] == "index-resolve" + assert manager.plan(path, offline=True)["packages"][0]["selected"] is None + + +def test_exact_configuration_requires_an_artifact_route(tmp_path): + data = simple_manifest("exact") + path = tmp_path / "pload.toml" + write_configuration(path, data) + manager = DeclarativeEnvironmentManager(ConfigManager(home=tmp_path / "home")) + assert manager.plan(path)["ready"] is False + + +def test_plan_analysis_does_not_create_or_overwrite_saved_choices(tmp_path): + path = tmp_path / "pload.toml" + write_configuration(path, simple_manifest()) + manager = DeclarativeEnvironmentManager(ConfigManager(home=tmp_path / "home")) + + plan = manager.plan(path) + saved_path = Path(plan["plan_path"]) + assert not saved_path.exists() + + manager.save_plan(path, plan) + before = saved_path.read_bytes() + manager.plan(path) + assert saved_path.read_bytes() == before + + +def test_plan_uses_measurements_instead_of_fixed_fake_times(tmp_path, monkeypatch): + path = tmp_path / "pload.toml" + write_configuration(path, simple_manifest()) + monkeypatch.setattr( + DeclarativeEnvironmentManager, "_network_rtt", staticmethod(lambda url: 0.123), + ) + + route = DeclarativeEnvironmentManager( + ConfigManager(home=tmp_path / "home") + ).plan(path)["packages"][0]["selected"] + + assert route["measurement"] == "RTT 123 ms" + assert "estimated_seconds" not in route + + +def test_automatic_plan_honors_package_route_override(tmp_path): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + }) + data["package"][0]["artifact"] = [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + "url": "https://files.invalid/" + wheel.name, + }] + data["plan"] = { + "mode": "auto", + "packages": {"demo": {"method": "index-exact", "location": "default"}}, + } + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + + result = manager.plan(manifest) + + assert result["planning_mode"] == "auto" + assert result["packages"][0]["selected"]["method"] == "index-exact" + assert result["packages"][0]["alternatives"] == [] + + +def test_configured_route_must_exist_in_automatic_plan(tmp_path): + data = simple_manifest() + data["plan"] = { + "mode": "auto", + "packages": {"demo": {"method": "repository", "location": "missing"}}, + } + path = tmp_path / "pload.toml" + write_configuration(path, data) + + package = DeclarativeEnvironmentManager( + ConfigManager(home=tmp_path / "home") + ).plan(path)["packages"][0] + + assert package["selected"] is None + assert package["rejections"][0]["reason"].endswith("repository:missing") + + +def test_changed_python_rejects_incompatible_cached_wheel_before_apply(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + incompatible_tag = f"cp37-cp37m-{next(iter(platform_tags()))}" + wheel = tiny_wheel(tmp_path / "fixture", incompatible_tag) + data = simple_manifest("exact") + data["name"] = "changed-python" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": [incompatible_tag], "repositories": [], + }], + }] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + + plan = plan_and_save(manager, manifest) + package = plan["packages"][0] + assert package["selected"] is None + assert package["rejections"][0]["artifact"] == wheel.name + assert "incompatible with" in package["rejections"][0]["reason"] + assert plan["ready"] is False + + monkeypatch.setattr( + VenvManager, "create_venv", + lambda *args, **kwargs: pytest.fail("apply created an environment for an invalid plan"), + ) + monkeypatch.setattr( + manager, "_acquire", + lambda *args, **kwargs: pytest.fail("apply downloaded an incompatible artifact"), + ) + with pytest.raises(PloadError, match="incompatible with"): + manager.apply(manifest) + + data["policy"]["reproducibility"] = "compatible" + write_configuration(manifest, data) + compatible = manager.plan(manifest)["packages"][0] + assert compatible["selected"]["method"] == "index-resolve" + assert compatible["selected"]["status"] == "network" + + +def test_cache_plan_applies_without_package_network_access(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + data = simple_manifest("exact") + data["name"] = "cache-only" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + data["policy"]["publish_missing_artifacts"] = False + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + }], + }] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + assert plan_and_save(manager, manifest)["packages"][0]["selected"]["method"] == "cache" + + original_execute = declarative_module.execute + + def reject_package_network(command, *args, **kwargs): + if "download" in command: + pytest.fail("a cache plan attempted a package download") + if "install" in command and "--no-index" not in command: + pytest.fail("a cache plan allowed pip index access") + return original_execute(command, *args, **kwargs) + + monkeypatch.setattr(declarative_module, "execute", reject_package_network) + progress = [] + restored = manager.apply(manifest, progress=progress.append) + output = original_execute([ + config.get_pip_command(restored)[0], "-c", "import pload_demo; print(pload_demo.answer)", + ]) + assert output == "42" + assert any( + event.startswith("DOWNLOAD\tpload-demo==1.0\t") for event in progress + ) + + +def test_apply_can_materialize_project_local_environment(tmp_path): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": [], + }) + data["package"] = [] + project = tmp_path / "project" + project.mkdir() + manifest = project / "pload.toml" + write_configuration(manifest, data, with_lock=False) + manager = DeclarativeEnvironmentManager(config) + plan_and_save(manager, manifest) + + restored = manager.apply(manifest, target=project / ".venv") + + assert restored == project / ".venv" + assert (restored / "pyvenv.cfg").is_file() + assert manager.apply(manifest, target=project / ".venv") == restored + + +def test_plan_discovers_and_apply_uses_exact_external_cache(tmp_path, monkeypatch): + external_cache = tmp_path / "pip-cache" + wheel = tiny_wheel(external_cache) + config_home = tmp_path / "home" + save_settings(config_home, {"resource_cache_dirs": [str(external_cache)]}) + config = ConfigManager(home=config_home) + config.get_python_path = lambda version=None: Path(sys.executable) + data = simple_manifest("exact") + data["name"] = "external-cache" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + }], + }] + manifest = tmp_path / "project" / "pload.toml" + manifest.parent.mkdir() + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + + selected = plan_and_save(manager, manifest)["packages"][0]["selected"] + assert selected["method"] == "external-cache" + assert selected["location"] == str(wheel.resolve()) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + + original_execute = declarative_module.execute + original_acquire = manager._acquire + acquisition_methods = [] + + def record_acquisition(*args, **kwargs): + acquisition_methods.append(args[3]["method"]) + return original_acquire(*args, **kwargs) + + def reject_download(command, *args, **kwargs): + if "download" in command: + pytest.fail("external-cache route attempted a download") + return original_execute(command, *args, **kwargs) + + monkeypatch.setattr(declarative_module, "execute", reject_download) + monkeypatch.setattr(manager, "_acquire", record_acquisition) + restored = manager.apply(manifest) + assert acquisition_methods == ["external-cache"] + assert digest(manager.cache / wheel.name) == digest(wheel) + assert original_execute([ + config.get_pip_command(restored)[0], "-c", + "import pload_demo; print(pload_demo.answer)", + ]) == "42" + + +def test_plan_reuses_package_from_compatible_pload_environment(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + source = VenvManager(config).create_venv(name="package-source") + original_execute = declarative_module.execute + original_execute(config.get_pip_command(source) + [ + "install", "--no-index", "--find-links", str(wheel.parent), "pload-demo==1.0", + ]) + data = simple_manifest("exact") + data["name"] = "environment-copy" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [], + }] + manifest = tmp_path / "project" / "pload.toml" + manifest.parent.mkdir() + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + + selected = plan_and_save(manager, manifest)["packages"][0]["selected"] + assert selected["method"] == "environment-copy" + assert selected["location"] == str(source) + assert len(selected["fingerprint"]) == 64 + + def reject_package_network(command, *args, **kwargs): + if "download" in command or ("install" in command and "--no-index" not in command): + pytest.fail("environment-copy route attempted package network access") + return original_execute(command, *args, **kwargs) + + monkeypatch.setattr(declarative_module, "execute", reject_package_network) + progress = [] + restored = manager.apply(manifest, progress=progress.append) + assert original_execute([ + config.get_pip_command(restored)[0], "-c", + "import pload_demo; print(pload_demo.answer)", + ]) == "42" + copy_events = [ + event for event in progress + if event.startswith("DOWNLOAD\tpload-demo==1.0\t") + ] + assert copy_events + _, _, received, total = copy_events[-1].split("\t") + assert int(received) == int(total) > 0 + completed = next( + index for index, event in enumerate(progress) + if event.startswith("PACKAGE_DONE\tpload-demo==1.0\t") + ) + creates_environment = next( + index for index, event in enumerate(progress) + if event.startswith("STAGE creating environment") + ) + assert completed < creates_environment + + +def test_plan_resolves_metadata_once_without_downloading_wheels(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + data = simple_manifest("exact") + data["name"] = "auto-locked" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo>=1"], + }) + data["policy"]["publish_missing_artifacts"] = False + data["package"] = [] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data, with_lock=False) + manager = DeclarativeEnvironmentManager(config) + original_execute = declarative_module.execute + resolver_calls = [] + + def fake_metadata(requirements, sources, environment, supported_tags): + resolver_calls.append(list(requirements)) + return [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + "url": "https://example.invalid/" + wheel.name, + "size": wheel.stat().st_size, "metadata_sha256": "", + }], + }] + + monkeypatch.setattr(declarative_module, "resolve_metadata", fake_metadata) + before = manifest.read_bytes() + first = manager.plan(manifest) + assert first["lock"]["status"] == "generated" + assert first["lock"]["required"] is False + assert first["packages"][0]["selected"]["method"] == "index-exact" + assert resolver_calls == [["pload-demo>=1"]] + assert manifest.read_bytes() == before + assert lock_path(manifest).is_file() + _, configuration = load_manifest(manifest) + assert configuration["environment"]["dependencies"] == ["pload-demo>=1"] + _, locked, current = load_lock(manifest, configuration) + assert current is True + assert locked["environment"]["dependencies"] == ["pload-demo==1.0"] + assert locked["package"][0]["artifact"][0]["filename"] == wheel.name + + second = manager.plan(manifest) + assert second["lock"]["status"] == "current" + assert len(resolver_calls) == 1 + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + plan_and_save(manager, manifest) + restored = manager.apply(manifest) + assert len(resolver_calls) == 1 + output = original_execute([ + config.get_pip_command(restored)[0], "-c", "import pload_demo; print(pload_demo.answer)", + ]) + assert output == "42" + + +def test_plan_marks_only_user_declared_packages_as_direct(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + root_wheel = tiny_wheel(tmp_path / "fixture", distribution="root_demo") + child_wheel = tiny_wheel(tmp_path / "fixture", distribution="child_demo") + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["root-demo"], + }) + data["package"] = [] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data, with_lock=False) + + def fake_metadata(requirements, sources, environment, supported_tags): + return [ + {"name": name, "version": "1.0", "sources": ["default"], + "artifact": [{"filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": []}]} + for name, wheel in (("child-demo", child_wheel), ("root-demo", root_wheel)) + ] + + monkeypatch.setattr(declarative_module, "resolve_metadata", fake_metadata) + plan = DeclarativeEnvironmentManager(config).plan(manifest) + + assert {item["name"]: item["direct"] for item in plan["packages"]} == { + "child-demo": False, "root-demo": True, + } + + +def test_plan_refreshes_stale_lock_using_only_metadata(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + demo = tiny_wheel(tmp_path / "fixture") + extra = tiny_wheel(tmp_path / "fixture", distribution="extra_demo") + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + data["environment"]["dependencies"].append("extra-demo") + write_configuration(manifest, data, with_lock=False) + manager = DeclarativeEnvironmentManager(config) + calls = [] + + def fake_metadata(requirements, sources, environment, supported_tags): + calls.append(list(requirements)) + return [ + {"name": name, "version": "1.0", "sources": ["default"], + "artifact": [{"filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": []}]} + for name, wheel in (("pload-demo", demo), ("extra-demo", extra)) + ] + + monkeypatch.setattr(declarative_module, "resolve_metadata", fake_metadata) + before_manifest = manifest.read_bytes() + before_lock = lock_path(manifest).read_bytes() + plan = manager.plan(manifest) + assert plan["lock"]["status"] == "generated" + assert plan["lock"]["required"] is False + assert calls == [["pload-demo==1.0", "extra-demo"]] + assert manifest.read_bytes() == before_manifest + assert lock_path(manifest).read_bytes() != before_lock + _, configuration = load_manifest(manifest) + assert configuration["environment"]["dependencies"] == [ + "pload-demo==1.0", "extra-demo", + ] + _, locked, current = load_lock(manifest, configuration) + assert current is True + assert {item["name"] for item in locked["package"]} == {"extra-demo", "pload-demo"} + + +def test_describe_plan_apply_through_content_repository(tmp_path, monkeypatch): + source_config = ConfigManager(home=tmp_path / "source-home") + repository = tmp_path / "artifact-store" + save_settings(source_config.home, { + "repositories": {"lab": {"kind": "local", "location": str(repository)}}, + "pip_index": "https://user:secret@example.invalid/simple?token=x", + }) + source_config = ConfigManager(home=tmp_path / "source-home") + wheel = tiny_wheel(tmp_path / "fixture") + source = VenvManager(source_config).create_venv(name="source") + execute(source_config.get_pip_command(source) + [ + "install", "--no-index", "--find-links", str(wheel.parent), "pload-demo==1.0", + ]) + manager = DeclarativeEnvironmentManager(source_config) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + describe_progress = [] + manifest = manager.describe( + "source", tmp_path / "pload.toml", name="restored", + sources=["pload-demo=https://download.example.invalid/cu121"], + progress=describe_progress.append, + ) + _, data = load_manifest(manifest) + _, locked, current = load_lock(manifest, data) + assert current is True + artifact = locked["package"][0]["artifact"][0] + assert data["sources"]["default"]["url"] == "https://example.invalid/simple" + assert locked["package"][0]["sources"] == ["package-pload-demo"] + assert data["sources"]["package-pload-demo"]["url"].endswith("/cu121") + assert artifact["repositories"] == [] + assert not (repository / "objects" / artifact["sha256"]).exists() + manager.publish_package("pload-demo", "source", "lab") + assert (repository / "objects" / artifact["sha256"]).is_file() + assert any("Locking pload-demo==1.0" in item for item in describe_progress) + assert not any("Publishing" in item for item in describe_progress) + + shutil.rmtree(source_config.home / "cache") + target_config = ConfigManager(home=tmp_path / "target-home") + target_config.get_python_path = lambda version=None: Path(sys.executable) + target = DeclarativeEnvironmentManager(target_config) + plan = plan_and_save(target, manifest) + assert plan["packages"][0]["selected"]["method"] == "repository" + target.cache.mkdir(parents=True) + shutil.copyfile(wheel, target.cache / artifact["filename"]) + original_fetch = declarative_module.ArtifactRepository.fetch.__func__ + fetches = [] + + def record_fetch(cls, *args, **kwargs): + fetches.append(args[1]) + return original_fetch(cls, *args, **kwargs) + + monkeypatch.setattr( + declarative_module.ArtifactRepository, "fetch", classmethod(record_fetch), + ) + apply_progress = [] + restored = target.apply(manifest, progress=apply_progress.append) + assert fetches == [artifact["sha256"]] + output = execute([target_config.get_pip_command(restored)[0], "-c", + "import pload_demo; print(pload_demo.answer)"]) + assert output == "42" + assert target.apply(manifest) == restored + assert digest(target.cache / artifact["filename"]) == artifact["sha256"] + assert any( + item.startswith("PACKAGE\t1/1\trepository\tpload-demo==1.0") + for item in apply_progress + ) + assert any("DONE verified exact requirements" in item for item in apply_progress) + + +def test_apply_rolls_back_a_new_environment_after_install_failure(tmp_path): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + artifact_dir = tmp_path / "artifacts" + artifact_dir.mkdir() + broken = artifact_dir / "demo-1.0-py3-none-any.whl" + broken.write_bytes(b"not a wheel") + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + }) + data["package"][0]["artifact"] = [{ + "filename": broken.name, "sha256": digest(broken), + "tags": ["py3-none-any"], "repositories": [], + }] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + with pytest.raises(PloadError): + DeclarativeEnvironmentManager(config).apply(manifest) + with pytest.raises(PloadError): + VenvManager(config).resolve_existing("demo") + + +def test_apply_requires_a_saved_plan_and_never_auto_publishes(tmp_path): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "artifacts") + checksum = digest(wheel) + repository = tmp_path / "store" + data = simple_manifest("exact") + data["name"] = "published" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + data["policy"]["repositories"] = ["lab"] + data["repositories"] = { + "lab": {"kind": "local", "location": str(repository)}, + } + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": checksum, + "tags": ["py3-none-any"], "repositories": [], + }], + }] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + with pytest.raises(PloadError, match="plan is missing"): + manager.apply(manifest) + plan_and_save(manager, manifest) + manager.apply(manifest) + assert not (repository / "objects" / checksum).exists() + + +def test_remote_add_explicitly_publishes_one_installed_package(tmp_path): + repository = tmp_path / "remote" + config_home = tmp_path / "home" + save_settings(config_home, { + "repositories": {"lab": {"kind": "local", "location": str(repository)}}, + }) + config = ConfigManager(home=config_home) + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + source = VenvManager(config).create_venv(name="remote-source") + execute(config.get_pip_command(source) + [ + "install", "--no-index", "--find-links", str(wheel.parent), "pload-demo==1.0", + ]) + manager = DeclarativeEnvironmentManager(config) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + + result = manager.publish_package("pload-demo", "remote-source", "lab") + + assert result["sha256"] == digest(wheel) + assert (repository / "objects" / digest(wheel)).is_file() + record = ( + repository / "packages" / "pload-demo" / "1.0" / f"{wheel.name}.json" + ) + assert record.is_file() + assert '"sha256": "' + digest(wheel) + '"' in record.read_text(encoding="utf-8") + + +def test_apply_obeys_selected_route_without_fallback(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "artifacts") + data = simple_manifest("exact") + data["name"] = "strict-route" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + }], + }] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + wheel.unlink() + plan_and_save(manager, manifest) + (manager.cache / wheel.name).unlink() + + with pytest.raises(PloadError, match="planned route failed.*cache"): + manager.apply(manifest) + + +def test_apply_downloads_the_exact_url_saved_by_plan(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + data = simple_manifest("exact") + data["name"] = "exact-url" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + artifact_url = "https://files.example.invalid/" + wheel.name + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + "url": artifact_url, "size": wheel.stat().st_size, + }], + }] + manifest = tmp_path / "project" / "pload.toml" + manifest.parent.mkdir() + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + assert plan_and_save(manager, manifest)["packages"][0]["selected"]["method"] == "index-exact" + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + requested = [] + + def fake_urlopen(request, timeout): + requested.append(request.full_url) + return io.BytesIO(wheel.read_bytes()) + + monkeypatch.setattr(declarative_module, "urlopen", fake_urlopen) + progress = [] + restored = manager.apply(manifest, progress=progress.append) + assert requested == [artifact_url] + assert any( + item.startswith("PACKAGE\t1/1\tindex-exact\tpload-demo==1.0") + for item in progress + ) + assert any( + item.startswith(f"SOURCE\tpload-demo==1.0\tdownloading from {artifact_url}") + for item in progress + ) + assert any(item.startswith("DOWNLOAD\tpload-demo==1.0\t") for item in progress) + assert execute([ + config.get_pip_command(restored)[0], "-c", + "import pload_demo; print(pload_demo.answer)", + ]) == "42" + + +def test_exact_download_reports_frequent_real_byte_progress(tmp_path, monkeypatch): + payload = b"x" * (declarative_module.DOWNLOAD_CHUNK_SIZE * 4) + source = tmp_path / "source.whl" + source.write_bytes(payload) + response = io.BytesIO(payload) + response.headers = {"Content-Length": str(len(payload))} + monkeypatch.setattr( + declarative_module, "urlopen", lambda request, timeout: response + ) + ticks = iter((0, 0.04, 0.08, 0.12, 0.16)) + monkeypatch.setattr(declarative_module.time, "monotonic", lambda: next(ticks)) + config = ConfigManager(home=tmp_path / "home") + manager = DeclarativeEnvironmentManager(config) + stage = tmp_path / "stage" + stage.mkdir() + destination = tmp_path / "output" / "demo.whl" + destination.parent.mkdir() + artifact = { + "filename": "demo.whl", + "sha256": digest(source), + "size": len(payload), + "url": "https://files.example.invalid/demo.whl", + } + package = {"name": "demo", "version": "1.0"} + selected = {"method": "index-exact", "location": "default", "artifact": artifact} + events = [] + + manager._acquire( + tmp_path / "pload.toml", + {"sources": {"default": {"url": "https://example.invalid/simple"}}}, + package, + selected, + destination, + stage, + Path(sys.executable), + progress=events.append, + ) + + downloads = [event for event in events if event.startswith("DOWNLOAD\t")] + assert len(downloads) >= 3 + assert downloads[-1].split("\t")[2:] == [str(len(payload)), str(len(payload))] + assert destination.read_bytes() == payload + + +def test_local_repository_fetch_streams_real_byte_progress(tmp_path, monkeypatch): + payload = b"r" * (declarative_module.DOWNLOAD_CHUNK_SIZE * 4) + checksum = hashlib.sha256(payload).hexdigest() + repository = tmp_path / "repository" + source = repository / "objects" / checksum + source.parent.mkdir(parents=True) + source.write_bytes(payload) + ticks = iter((0, 0.04, 0.08, 0.12, 0.16)) + monkeypatch.setattr(declarative_module.time, "monotonic", lambda: next(ticks)) + destination = tmp_path / "downloaded.whl" + events = [] + + ArtifactRepository.fetch( + {"kind": "local", "location": str(repository)}, + checksum, + destination, + tmp_path, + progress=lambda received, total: events.append((received, total)), + expected_size=0, + ) + + assert len(events) >= 3 + assert events[-1] == (len(payload), len(payload)) + assert destination.read_bytes() == payload + + +def test_ssh_repository_fetch_streams_real_bytes(tmp_path, monkeypatch): + chunk = b"s" * declarative_module.DOWNLOAD_CHUNK_SIZE + payload = chunk * 4 + checksum = hashlib.sha256(payload).hexdigest() + + class StreamingOutput: + def __init__(self): + self.parts = 0 + + def readline(self, size): + return f"{len(payload)}\n".encode() + + def read1(self, size): + if self.parts < 4: + self.parts += 1 + return chunk + return b"" + + class StreamingSsh: + def __init__(self, command, stdout, stderr): + self.stdout = StreamingOutput() + self.stderr = io.BytesIO() + self.returncode = 0 + + def poll(self): + return 0 + + def kill(self): + self.returncode = -1 + + def wait(self, timeout=None): + return self.returncode + + monkeypatch.setattr(declarative_module.subprocess, "Popen", StreamingSsh) + ticks = iter((0, 0.04, 0.08, 0.12, 0.16, 0.20)) + monkeypatch.setattr(declarative_module.time, "monotonic", lambda: next(ticks)) + events = [] + destination = tmp_path / "ssh-download.whl" + + ArtifactRepository.fetch( + {"kind": "ssh", "location": "example:/repository"}, + checksum, + destination, + tmp_path, + progress=lambda received, total: events.append((received, total)), + expected_size=0, + ) + + assert [received for received, _ in events[:4]] == [ + len(chunk), len(chunk) * 2, len(chunk) * 3, len(chunk) * 4, + ] + assert events[-1] == (len(payload), len(payload)) + assert destination.read_bytes() == payload + + +def test_offline_apply_rejects_saved_network_route(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + }) + data["package"][0]["artifact"] = [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + "url": "https://example.invalid/" + wheel.name, + }] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + plan_and_save(manager, manifest) + monkeypatch.setattr( + declarative_module, "urlopen", + lambda *args, **kwargs: pytest.fail("offline apply opened the network"), + ) + + with pytest.raises(PloadError, match="offline apply forbids.*network route"): + manager.apply(manifest, offline=True) + + +def test_apply_refuses_to_replace_missing_planned_python(tmp_path): + planned_python = tmp_path / "planned-python" + planned_python.write_text("placeholder", encoding="utf-8") + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: planned_python + manifest = tmp_path / "pload.toml" + data = simple_manifest("compatible") + data["environment"]["dependencies"] = [] + data["package"] = [] + write_configuration(manifest, data, with_lock=False) + manager = DeclarativeEnvironmentManager(config) + plan_and_save(manager, manifest) + planned_python.unlink() + + with pytest.raises(PloadError, match="planned Python interpreter.*disappeared"): + manager.apply(manifest) + + +def test_compatible_index_route_disables_pip_cache(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + manifest = tmp_path / "pload.toml" + write_configuration(manifest, simple_manifest("compatible")) + manager = DeclarativeEnvironmentManager(config) + plan_and_save(manager, manifest) + monkeypatch.setattr(manager, "_check_runtime", lambda *args: None) + environment = tmp_path / "environment" + + def create_environment(*args, **kwargs): + environment.mkdir() + return environment + + monkeypatch.setattr(VenvManager, "create_venv", create_environment) + commands = [] + monkeypatch.setattr( + declarative_module, "execute", + lambda command, *args, **kwargs: commands.append(command) or "", + ) + + manager.apply(manifest) + + route_command = next(command for command in commands if "demo==1.0" in command) + assert "--no-cache-dir" in route_command + + +def test_apply_rejects_plan_that_omits_a_locked_package(tmp_path): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + }) + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + generated = manager.plan(manifest) + generated["packages"] = [] + generated["ready"] = True + write_plan( + manifest, declarative_module.configuration_digest(data), + file_digest(lock_path(manifest)), generated, + ) + with pytest.raises(PloadError, match="do not match the current lock"): + manager.apply(manifest) + + +def test_apply_rejects_tampered_route_status_and_artifact_url(tmp_path): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + }) + wheel = tiny_wheel(tmp_path / "fixture") + data["package"][0]["artifact"] = [{ + "filename": wheel.name, + "sha256": digest(wheel), + "tags": ["py3-none-any"], + "repositories": [], + "url": "https://locked.invalid/demo.whl", + }] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + generated = manager.plan(manifest) + + generated["packages"][0]["selected"]["status"] = "ready" + manager.save_plan(manifest, generated) + with pytest.raises(PloadError, match="route status does not match"): + manager.apply(manifest) + + generated = manager.plan(manifest) + generated["packages"][0]["selected"]["artifact"]["url"] = ( + "https://changed.invalid/demo.whl" + ) + manager.save_plan(manifest, generated) + with pytest.raises(PloadError, match="artifact URL does not match"): + manager.apply(manifest) + + +def test_legacy_exact_download_uses_planned_python_and_disables_pip_cache( + tmp_path, monkeypatch, +): + config = ConfigManager(home=tmp_path / "home") + manager = DeclarativeEnvironmentManager(config) + wheel = tiny_wheel(tmp_path / "fixture") + planned_python = tmp_path / "planned-python" + command_seen = [] + + def fake_execute(command, *args, **kwargs): + command_seen.append(command) + destination = Path(command[command.index("--dest") + 1]) + shutil.copyfile(wheel, destination / wheel.name) + return "" + + monkeypatch.setattr(declarative_module, "execute", fake_execute) + output = tmp_path / "result.whl" + manager._acquire( + tmp_path / "pload.toml", + {"sources": {"default": {"url": "https://index.invalid/simple"}}}, + {"name": "pload-demo", "version": "1.0"}, + { + "method": "index-exact", "location": "default", + "artifact": {"filename": wheel.name, "sha256": digest(wheel)}, + }, + output, tmp_path, interpreter=planned_python, + ) + + assert command_seen[0][:3] == [str(planned_python), "-m", "pip"] + assert "--no-cache-dir" in command_seen[0] + assert output.read_bytes() == wheel.read_bytes() + + +def test_declarative_commands_and_shell_integration(capsys): + parser = build_parser() + assert parser.parse_args(["describe", "v1", "-o", "env.toml"]).output == "env.toml" + parsed = parser.parse_args([ + "describe", "v1", "-s", "torch=https://download.pytorch.org/whl/cu121", + ]) + assert parsed.package_sources == ["torch=https://download.pytorch.org/whl/cu121"] + assert parser.parse_args(["lock"]).file == "pload.toml" + assert parser.parse_args(["apply"]).file == "pload.toml" + assert parser.parse_args(["plan", "--no-ui"]).no_ui is True + remote = parser.parse_args(["remote", "add", "numpy", "-f", "v10", "-r", "lab"]) + assert (remote.package, remote.environment, remote.repository) == ("numpy", "v10", "lab") + for command in ("describe", "lock", "plan", "apply"): + assert main([command, "-h"]) == 0 + for shell in ("bash", "zsh", "fish", "powershell"): + assert command in shell_script(shell) + assert "--mode" in capsys.readouterr().out diff --git a/tests/test_dependency.py b/tests/test_dependency.py new file mode 100644 index 0000000..741237c --- /dev/null +++ b/tests/test_dependency.py @@ -0,0 +1,85 @@ +from subprocess import CompletedProcess + +import pytest +from packaging.tags import Tag + +from pload.errors import PloadError +from pload.managers.dependency import DependencyManager +from pload.managers.platform import ConfigManager + + +def test_failed_package_does_not_stop_later_requirements(tmp_path, monkeypatch): + manager = DependencyManager(ConfigManager(home=tmp_path / "home")) + commands = [] + return_codes = iter([1, 0]) + + def run(command, check): + commands.append(command) + return CompletedProcess(command, next(return_codes)) + + monkeypatch.setattr("pload.managers.dependency.subprocess.run", run) + + with pytest.raises(PloadError, match="successful packages were kept"): + manager.install_dependencies( + tmp_path / "environment", + ["not a requirement ???", "missing-package", "working-package"], + ) + + assert len(commands) == 2 + assert "missing-package" in commands[0] + assert "working-package" in commands[1] + + +def test_automatic_strategy_prefers_compatible_pload_cache(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + cache = config.home / "cache" / "wheels" + cache.mkdir(parents=True) + older = cache / "demo-1.0-py3-none-any.whl" + newest = cache / "demo-2.0-py3-none-any.whl" + older.touch() + newest.touch() + captured = [] + + monkeypatch.setattr( + "pload.managers.dependency._compatible_tags", + lambda path: {Tag("py3", "none", "any")}, + ) + monkeypatch.setattr( + "pload.managers.dependency.subprocess.run", + lambda command, check: captured.append(command) or CompletedProcess(command, 0), + ) + + DependencyManager(config).install_dependencies( + tmp_path / "environment", ["demo>=1"], strategy="auto", + ) + + assert str(newest) in captured[0] + assert str(older) not in captured[0] + + +def test_custom_strategy_can_choose_index_instead_of_cache(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + cache = config.home / "cache" / "wheels" + cache.mkdir(parents=True) + (cache / "demo-1.0-py3-none-any.whl").touch() + captured = [] + + monkeypatch.setattr( + "pload.managers.dependency._compatible_tags", + lambda path: {Tag("py3", "none", "any")}, + ) + monkeypatch.setattr( + "pload.managers.dependency.ui.select", + lambda message, choices, **kwargs: choices[-1].value, + ) + monkeypatch.setattr( + "pload.managers.dependency.subprocess.run", + lambda command, check: captured.append(command) or CompletedProcess(command, 0), + ) + + DependencyManager(config).install_dependencies( + tmp_path / "environment", ["demo>=1"], strategy="custom", + ) + + assert "demo>=1" in captured[0] + assert not any(item.endswith(".whl") for item in captured[0]) diff --git a/tests/test_installer.py b/tests/test_installer.py index a979c3c..6ee9d44 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -60,6 +60,19 @@ def test_official_pip_source_is_explicit(tmp_path): assert settings["pip_index"] == PYPI_OFFICIAL_INDEX +def test_setup_preserves_repositories_and_custom_state(tmp_path): + from pload.settings import save_settings + + original = { + "repositories": {"lab": {"kind": "ssh", "location": "host:/srv/pload"}}, + "state_dir": str(tmp_path / "custom-state"), + } + save_settings(tmp_path / "pload", original) + collected = collect_settings(installer_args(tmp_path)) + assert collected["repositories"] == original["repositories"] + assert collected["state_dir"] == original["state_dir"] + + def test_additional_pip_source_presets(tmp_path): ustc = collect_settings(installer_args(tmp_path, pip_source="ustc")) aliyun = collect_settings(installer_args(tmp_path, pip_source="aliyun")) @@ -80,6 +93,21 @@ def test_guided_choice_accepts_numeric_selection(monkeypatch, capsys): assert "Alibaba Cloud" in output +def test_settings_screen_can_revisit_and_edit_any_value(tmp_path, monkeypatch): + answers = iter(["bin_dir", "__save__"]) + monkeypatch.setattr(installer.ui, "select", lambda *args, **kwargs: next(answers)) + monkeypatch.setattr( + installer.ui, + "text", + lambda message, default="": str(tmp_path / "custom-bin"), + ) + + args = installer.review_settings(installer_args(tmp_path, yes=False)) + + assert args.bin_dir == str(tmp_path / "custom-bin") + assert args.yes is True + + def test_config_only_install_writes_reusable_configuration(tmp_path): result = run([ "--yes", @@ -168,7 +196,7 @@ def test_shell_configuration_updates_existing_managed_block(monkeypatch, tmp_pat def test_installer_has_brief_and_detailed_help(capsys): assert run(["-h"]) == 0 brief = capsys.readouterr().out - assert "colored, numbered guided setup" in brief + assert "cursor-key guided setup" in brief assert "pload-install -h -d" in brief assert "--downloads-json-url" in brief assert "--home, -H" in brief @@ -177,7 +205,7 @@ def test_installer_has_brief_and_detailed_help(capsys): assert run(["-h", "-d"]) == 0 detailed = capsys.readouterr().out assert "What an interactive run looks like" in detailed - assert "Enter a number" in detailed + assert "↑/↓ move" in detailed assert "Resulting layout" in detailed assert "What the installer changes" in detailed assert "--downloads-json-url" in detailed diff --git a/tests/test_metadata_resolver.py b/tests/test_metadata_resolver.py new file mode 100644 index 0000000..8bdcc55 --- /dev/null +++ b/tests/test_metadata_resolver.py @@ -0,0 +1,163 @@ +import hashlib +import json + +import pytest +from packaging.tags import Tag + +from pload.errors import PloadError +from pload.metadata_resolver import resolve_metadata + + +def _project(filename, checksum, metadata_checksum, size=100): + return json.dumps({ + "meta": {"api-version": "1.1"}, + "name": filename.split("-", 1)[0], + "files": [{ + "filename": filename, + "url": "https://files.invalid/" + filename, + "hashes": {"sha256": checksum}, + "requires-python": ">=3.8", + "core-metadata": {"sha256": metadata_checksum}, + "yanked": False, + "size": size, + }], + "versions": [filename.split("-")[1]], + }).encode() + + +def test_resolver_reads_only_project_pages_and_core_metadata(): + root_metadata = ( + b"Metadata-Version: 2.1\nName: root-demo\nVersion: 1.0\n" + b"Requires-Dist: child-demo>=2\n\n" + ) + child_metadata = b"Metadata-Version: 2.1\nName: child-demo\nVersion: 2.1\n\n" + root_wheel = "root_demo-1.0-py3-none-any.whl" + child_wheel = "child_demo-2.1-py3-none-any.whl" + responses = { + "https://index.invalid/simple/root-demo/": _project( + root_wheel, "a" * 64, hashlib.sha256(root_metadata).hexdigest(), + ), + "https://index.invalid/simple/child-demo/": _project( + child_wheel, "b" * 64, hashlib.sha256(child_metadata).hexdigest(), + ), + "https://files.invalid/" + root_wheel + ".metadata": root_metadata, + "https://files.invalid/" + child_wheel + ".metadata": child_metadata, + } + requested = [] + + def fetch(url, accept, maximum): + requested.append(url) + return responses[url], ( + "application/vnd.pypi.simple.v1+json" + if url.endswith("/") else "application/octet-stream" + ) + + packages = resolve_metadata( + ["root-demo>=1"], [("default", "https://index.invalid/simple")], + {"python": "3.12.1", "implementation": "cpython", + "system": "Linux", "machine": "x86_64"}, + {Tag("py3", "none", "any")}, fetch=fetch, + ) + + assert [(item["name"], item["version"]) for item in packages] == [ + ("child-demo", "2.1"), ("root-demo", "1.0"), + ] + assert all(url.endswith(("/", ".metadata")) for url in requested) + assert not any(url.endswith(".whl") for url in requested) + + +def test_resolver_refuses_to_download_wheel_when_metadata_is_unavailable(): + wheel = "root_demo-1.0-py3-none-any.whl" + project = json.loads(_project(wheel, "a" * 64, "b" * 64)) + project["files"][0].pop("core-metadata") + requested = [] + + def fetch(url, accept, maximum): + requested.append(url) + return json.dumps(project).encode(), "application/vnd.pypi.simple.v1+json" + + with pytest.raises(PloadError, match="refusing to download a wheel during plan"): + resolve_metadata( + ["root-demo"], [("default", "https://index.invalid/simple")], + {"python": "3.12.1", "implementation": "cpython", + "system": "Linux", "machine": "x86_64"}, + {Tag("py3", "none", "any")}, fetch=fetch, + ) + assert requested == ["https://index.invalid/simple/root-demo/"] + + +def test_resolver_supports_multiple_root_requirements(): + requested = [] + + def fetch(url, accept, maximum): + requested.append(url) + name = url.rstrip("/").rsplit("/", 1)[-1] + if url.endswith(".metadata"): + version = "1.0" if "first_demo" in url else "2.0" + project = "first-demo" if "first_demo" in url else "second-demo" + return ( + f"Metadata-Version: 2.1\nName: {project}\nVersion: {version}\n\n".encode(), + "application/octet-stream", + ) + distribution = name.replace("-", "_") + version = "1.0" if name == "first-demo" else "2.0" + filename = f"{distribution}-{version}-py3-none-any.whl" + metadata = ( + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n\n".encode() + ) + return ( + _project(filename, ("a" if name == "first-demo" else "b") * 64, + hashlib.sha256(metadata).hexdigest()), + "application/vnd.pypi.simple.v1+json", + ) + + packages = resolve_metadata( + ["first-demo", "second-demo"], + [("default", "https://index.invalid/simple")], + {"python": "3.12.1", "implementation": "cpython", + "system": "Linux", "machine": "x86_64"}, + {Tag("py3", "none", "any")}, fetch=fetch, + ) + + assert [(item["name"], item["version"]) for item in packages] == [ + ("first-demo", "1.0"), ("second-demo", "2.0"), + ] + assert not any(url.endswith(".whl") for url in requested) + + +def test_later_source_can_supply_version_missing_from_preferred_source(): + first_metadata = b"Metadata-Version: 2.1\nName: root-demo\nVersion: 1.0\n\n" + later_metadata = b"Metadata-Version: 2.1\nName: root-demo\nVersion: 2.0\n\n" + first_wheel = "root_demo-1.0-py3-none-any.whl" + later_wheel = "root_demo-2.0-py3-none-any.whl" + responses = { + "https://first.invalid/simple/root-demo/": _project( + first_wheel, "a" * 64, hashlib.sha256(first_metadata).hexdigest(), + ), + "https://later.invalid/simple/root-demo/": _project( + later_wheel, "b" * 64, hashlib.sha256(later_metadata).hexdigest(), + ), + "https://files.invalid/" + first_wheel + ".metadata": first_metadata, + "https://files.invalid/" + later_wheel + ".metadata": later_metadata, + } + + def fetch(url, accept, maximum): + return responses[url], ( + "application/vnd.pypi.simple.v1+json" + if url.endswith("/") else "application/octet-stream" + ) + + packages = resolve_metadata( + ["root-demo>=2"], + [ + ("preferred", "https://first.invalid/simple"), + ("fallback", "https://later.invalid/simple"), + ], + {"python": "3.12.1", "implementation": "cpython", + "system": "Linux", "machine": "x86_64"}, + {Tag("py3", "none", "any")}, fetch=fetch, + ) + + assert [(item["version"], item["sources"]) for item in packages] == [ + ("2.0", ["fallback"]), + ] diff --git a/tests/test_pyversion.py b/tests/test_pyversion.py index b73ab00..c4dde5e 100644 --- a/tests/test_pyversion.py +++ b/tests/test_pyversion.py @@ -1,3 +1,4 @@ +import os from dataclasses import replace from pathlib import Path @@ -19,6 +20,20 @@ def test_source_filter_rejects_unknown_types(): PythonManager.parse_sources(["mystery"]) +def test_unresolvable_windows_alias_uses_absolute_path(monkeypatch, tmp_path): + alias = tmp_path / "python.exe" + original_resolve = Path.resolve + + def inaccessible(self): + if self == alias: + raise OSError("inaccessible app execution alias") + return original_resolve(self) + + monkeypatch.setattr(Path, "resolve", inaccessible) + assert PythonManager._resolved_path(alias) == alias.absolute() + assert PythonManager._path_key(alias) == os.path.normcase(str(alias.absolute())) + + def test_discovery_deduplicates_resolved_paths_and_filters(monkeypatch, tmp_path): python = tmp_path / "python3.12" python.touch() diff --git a/tests/test_resource_plan.py b/tests/test_resource_plan.py new file mode 100644 index 0000000..81f9249 --- /dev/null +++ b/tests/test_resource_plan.py @@ -0,0 +1,52 @@ +from pload.resource_plan import PlanSelection + + +def _route(method): + return { + "method": method, "location": "/" + method, "status": "ready", + "cost": [0], "estimated_seconds": 1, + } + + +def test_route_selection_supports_navigation_state_undo_and_reset(): + plan = { + "python": {"status": "ready"}, "ready": True, + "packages": [{ + "name": "demo", "version": "1", "selected": _route("cache"), + "alternatives": [_route("environment-copy"), _route("index-exact")], + }], + } + selection = PlanSelection(plan) + + selection.select(0, 2) + assert plan["packages"][0]["selected"]["method"] == "index-exact" + assert [item["method"] for item in plan["packages"][0]["alternatives"]] == [ + "cache", "environment-copy", + ] + assert selection.undo() is True + assert plan["packages"][0]["selected"]["method"] == "cache" + assert selection.undo() is False + + selection.select(0, 1) + selection.reset() + assert plan["packages"][0]["selected"]["method"] == "cache" + assert selection.undo() is True + assert plan["packages"][0]["selected"]["method"] == "environment-copy" + + +def test_only_direct_requirements_are_interactive_decisions(): + plan = { + "python": {"status": "ready"}, "ready": True, + "packages": [ + {"name": "torch", "direct": True, "selected": _route("cache"), + "alternatives": []}, + {"name": "filelock", "direct": False, "selected": _route("cache"), + "alternatives": [_route("index-exact")]}, + {"name": "numpy", "direct": True, "selected": _route("cache"), + "alternatives": []}, + ], + } + + selection = PlanSelection(plan) + + assert selection.decision_indices() == [0, 2] diff --git a/tests/test_snapshots.py b/tests/test_snapshots.py new file mode 100644 index 0000000..30f83aa --- /dev/null +++ b/tests/test_snapshots.py @@ -0,0 +1,187 @@ +import json +import shutil +import sys +import zipfile + +import pytest + +from pload.cli import main, shell_script +from pload.errors import PloadError +from pload.managers.platform import ConfigManager +from pload.managers.venv import VenvManager +from pload.snapshots import ( + RepositoryManager, + SnapshotManager, + digest, + execute, + public_index, + validate_bundle, + write_json, +) + + +def tiny_wheel(directory): + directory.mkdir(parents=True, exist_ok=True) + path = directory / "pload_demo-1.0-py3-none-any.whl" + with zipfile.ZipFile(path, "w") as wheel: + wheel.writestr("pload_demo.py", "answer = 42\n") + wheel.writestr("pload_demo-1.0.dist-info/METADATA", + "Metadata-Version: 2.1\nName: pload-demo\nVersion: 1.0\n") + wheel.writestr("pload_demo-1.0.dist-info/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n") + wheel.writestr("pload_demo-1.0.dist-info/RECORD", "") + return path + + +def recipe(path): + path.mkdir(parents=True) + (path / "requirements.txt").write_text("", encoding="utf-8") + write_json(path / "snapshot.json", { + "schema": 1, "runtime": {}, + "files": {"requirements.txt": digest(path / "requirements.txt")}, + }) + return path + + +def test_offline_roundtrip_external_interpreter_and_shared_cache(tmp_path): + config = ConfigManager(home=tmp_path / "home") + wheels = tmp_path / "existing-wheels" + tiny_wheel(wheels) + source = VenvManager(config).create_venv(name="source") + execute(config.get_pip_command(source) + ["install", "--no-index", "--find-links", + str(wheels), "pload-demo==1.0"]) + manager = SnapshotManager(config) + python = config.get_pip_command(source)[0] + snapshot = manager.export("demo", python=python, bundle="all", links=[str(wheels)]) + assert (snapshot / "requirements.txt").read_text().strip() == "pload-demo==1.0" + assert (manager.wheels / "pload_demo-1.0-py3-none-any.whl").is_file() + restored = manager.restore("demo", "restored", version=sys.executable, offline=True) + output = execute([config.get_pip_command(restored)[0], "-c", + "import pload_demo; print(pload_demo.answer)"]) + assert output == "42" + # A restored wheel has direct_url metadata; exporting it again must reuse + # the exact original archive from the shared cache, not require an index. + reexport = manager.export("restored-again", environment="restored", bundle="all") + assert validate_bundle(reexport)["files"] == validate_bundle(snapshot)["files"] + with pytest.raises(PloadError, match="already exists"): + manager.export("demo", python=python) + repos = RepositoryManager(config) + repos.add("local", str(tmp_path / "remote"), "local") + repos.transfer("local", "demo", push=True) + second = ConfigManager(home=tmp_path / "second") + reader = RepositoryManager(second) + reader.add("local", str(tmp_path / "remote"), "local") + reader.transfer("local", "demo") + validate_bundle(reader.snapshots.path("demo")) + assert (reader.snapshots.wheels / "pload_demo-1.0-py3-none-any.whl").exists() + + +def test_tampering_is_rejected_before_creating_environment(tmp_path): + manager = SnapshotManager(ConfigManager(home=tmp_path)) + snapshot = recipe(manager.path("demo")) + (snapshot / "requirements.txt").write_text("unexpected==1\n", encoding="utf-8") + with pytest.raises(PloadError, match="checksum"): + manager.restore("demo", "nope") + assert not manager.config.venv_path.exists() + + +@pytest.mark.parametrize("filename", ["../escape.whl", "/tmp/escape.whl", "wheels/../../escape.whl"]) +def test_unsafe_manifest_paths_rejected(tmp_path, filename): + snapshot = recipe(tmp_path / "demo") + data = json.loads((snapshot / "snapshot.json").read_text()) + data["files"][filename] = "0" * 64 + write_json(snapshot / "snapshot.json", data) + with pytest.raises(PloadError, match="invalid snapshot file"): + validate_bundle(snapshot) + + +def test_runtime_mismatch_does_not_create_environment(tmp_path): + manager = SnapshotManager(ConfigManager(home=tmp_path)) + recipe(manager.path("demo")) + with pytest.raises(PloadError, match="mismatch"): + manager.restore("demo", "nope", version=sys.executable) + assert not manager.config.venv_path.exists() + + +@pytest.mark.parametrize("location", ["-oProxyCommand:x", "host:relative", "host:/a/../b", + "host:/a;touch-b", "host:/"]) +def test_ssh_location_validation(location): + with pytest.raises(PloadError): + RepositoryManager.ssh_location(location) + + +def test_git_recipe_roundtrip(tmp_path, monkeypatch): + if not shutil.which("git"): + pytest.skip("Git is optional") + for key, value in {"GIT_AUTHOR_NAME": "pload test", "GIT_COMMITTER_NAME": "pload test", + "GIT_AUTHOR_EMAIL": "test@example.invalid", + "GIT_COMMITTER_EMAIL": "test@example.invalid"}.items(): + monkeypatch.setenv(key, value) + bare = tmp_path / "remote.git" + execute(["git", "init", "--bare", str(bare)]) + writer = RepositoryManager(ConfigManager(home=tmp_path / "writer")) + recipe(writer.snapshots.path("demo")) + writer.add("git", str(bare), "git") + writer.transfer("git", "demo", push=True) + reader = RepositoryManager(ConfigManager(home=tmp_path / "reader")) + reader.add("git", str(bare), "git") + reader.transfer("git", "demo") + validate_bundle(reader.snapshots.path("demo")) + + +def test_commands_and_shell_integration(capsys): + for command in ("repo",): + assert main([command, "-h"]) == 0 + for shell in ("bash", "zsh", "fish", "powershell"): + assert command in shell_script(shell) + assert "artifact providers" in capsys.readouterr().out + + +def test_repository_list_alias(tmp_path, capsys): + assert main(["-H", str(tmp_path), "repo", "ls"]) == 0 + assert capsys.readouterr().out.strip() == "{}" + + +def test_source_credentials_are_not_recorded(): + assert public_index("https://user:secret@example.com/simple?token=secret") == ( + "https://example.com/simple" + ) + + +def test_conda_export_is_explicitly_rejected(tmp_path, monkeypatch): + monkeypatch.setattr("pload.snapshots.probe", lambda _: {"conda": True}) + manager = SnapshotManager(ConfigManager(home=tmp_path)) + with pytest.raises(PloadError, match="Conda"): + manager.export("native", python=sys.executable) + assert not manager.path("native").exists() + + +def test_editable_source_is_not_silently_pinned(tmp_path, monkeypatch): + monkeypatch.setattr("pload.snapshots.probe", lambda _: { + "packages": [{"name": "editable", "version": "1.0", "direct_url": + json.dumps({"url": "file:///source", "dir_info": {"editable": True}})}], + }) + manager = SnapshotManager(ConfigManager(home=tmp_path)) + with pytest.raises(PloadError, match="original wheel"): + manager.export("source", python=sys.executable, bundle="all") + assert not manager.path("source").exists() + + +def test_local_repository_refuses_overwrite(tmp_path): + manager = RepositoryManager(ConfigManager(home=tmp_path / "home")) + recipe(manager.snapshots.path("demo")) + manager.add("disk", str(tmp_path / "remote"), "local") + manager.transfer("disk", "demo", push=True) + with pytest.raises(PloadError, match="already exists"): + manager.transfer("disk", "demo", push=True) + manager.remove("disk") + validate_bundle(tmp_path / "remote" / "demo") + + +def test_offline_requirements_cannot_fetch_direct_urls(tmp_path): + requirements = tmp_path / "requirements.txt" + requirements.write_text("package @ https://example.invalid/package.whl\n") + manager = SnapshotManager(ConfigManager(home=tmp_path / "home")) + with pytest.raises(PloadError, match="validated snapshot"): + manager.restore(str(requirements), "offline", offline=True) + assert not manager.config.venv_path.exists() diff --git a/tests/test_ui.py b/tests/test_ui.py new file mode 100644 index 0000000..0ae0ed8 --- /dev/null +++ b/tests/test_ui.py @@ -0,0 +1,98 @@ +from questionary import Choice + +from pload import ui +from pload.cli import choose_declarative_routes + + +def test_select_has_numbered_fallback_when_terminal_ui_is_unavailable(monkeypatch, capsys): + monkeypatch.setattr(ui, "is_interactive", lambda: False) + monkeypatch.setattr("builtins.input", lambda prompt: "2") + + selected = ui.select( + "Choose a source", + [Choice("Local cache", value="cache"), Choice("PyPI", value="index")], + default="cache", + ) + + assert selected == "index" + assert "1) Local cache (default)" in capsys.readouterr().out + + +def test_select_fallback_accepts_action_hotkeys(monkeypatch): + monkeypatch.setattr(ui, "is_interactive", lambda: False) + monkeypatch.setattr("builtins.input", lambda prompt: "p") + + selected = ui.select( + "Choose a source", + [Choice("Local cache", value="cache")], + hotkeys={"p": ("previous", None)}, + ) + + assert selected == ("previous", None) + + +def test_route_chooser_only_visits_direct_packages_and_supports_backtracking(monkeypatch): + plan = { + "python": {"status": "available"}, + "packages": [ + { + "name": "torch", + "version": "2.8.0", + "direct": True, + "selected": { + "method": "index", + "location": "https://pypi.org/simple", + "status": "available", + "estimated_seconds": 10.0, + }, + "alternatives": [{ + "method": "cache", + "location": "/cache/torch.whl", + "status": "available", + "estimated_seconds": 0.1, + }], + }, + { + "name": "typing-extensions", + "version": "4.15.0", + "direct": False, + "selected": { + "method": "cache", + "location": "/cache/typing_extensions.whl", + "status": "available", + "estimated_seconds": 0.1, + }, + "alternatives": [], + }, + ], + } + answers = iter([("route", 1), ("save", None)]) + prompts = [] + + def choose(message, choices, **kwargs): + prompts.append((message, choices)) + return next(answers) + + monkeypatch.setattr("pload.cli.ui.select", choose) + + assert choose_declarative_routes(plan) == "save" + assert plan["packages"][0]["selected"]["method"] == "cache" + assert plan["packages"][1]["selected"]["method"] == "cache" + assert len(prompts) == 2 + assert all("typing-extensions" not in message for message, _ in prompts) + displayed = [choice.title for _, choices in prompts for choice in choices] + assert all("Previous" not in title and "Save" not in title for title in displayed) + + +def test_no_color_disables_rich_ansi_sequences(monkeypatch): + monkeypatch.setenv("NO_COLOR", "1") + output = ui.console() + + assert output.color_system is None + assert ui._prompt_style() is ui.PLAIN_PROMPT_STYLE + + +def test_symbols_fall_back_for_legacy_windows_encodings(): + assert ui.glyph("✓", "[ok]", encoding="cp1252") == "[ok]" + assert ui.glyph("❯", ">", encoding="cp1252") == ">" + assert ui.glyph("✓", "[ok]", encoding="utf-8") == "✓"