From cc329b57a8f1a538b177f6d26cc3bbdc8f8210de Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Thu, 24 Sep 2026 20:24:49 +0800 Subject: [PATCH 01/41] Add reproducible snapshots and configurable artifact repositories --- CHANGELOG.md | 19 ++ README.md | 130 +++++++++ src/pload/cli.py | 78 +++++- src/pload/help_content.py | 69 +++++ src/pload/managers/dependency.py | 3 + src/pload/snapshots.py | 437 +++++++++++++++++++++++++++++++ tests/test_snapshots.py | 183 +++++++++++++ 7 files changed, 913 insertions(+), 6 deletions(-) create mode 100644 src/pload/snapshots.py create mode 100644 tests/test_snapshots.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3734092..fa8e228 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## Unreleased + +### Added + +- Export exact package snapshots from managed and external Python environments, + with optional selected/all wheel bundles and checksums. +- Restore snapshots offline, explicitly re-resolve for another platform, or + import trusted requirements files into new environments. +- Configure local, SSH and Git recipe repositories; push and pull immutable snapshots. +- Reuse local wheel archives and pip caches, deduplicate SSH wheel storage by SHA-256, + and avoid re-downloading identical cached wheels during pull. +- Document interoperability boundaries, GPU/native dependencies, source-wheel + provenance and repository trust requirements. + +### Fixed + +- Keep an external environment's interpreter path intact when probing packages, + instead of following its Python symlink out to the base installation. + ## 1.0.0 - 2026-09-24 ### Added diff --git a/README.md b/README.md index 27b1b54..06076ce 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ complete parameter reference and `-h -d` for practical examples and effects. - [Shell activation](#shell-activation) - [Discover and install Python](#discover-and-install-python) - [Create and activate environments](#create-and-activate-environments) +- [Reproduce and share environments](#reproduce-and-share-environments) - [Help and aliases](#help-and-aliases) - [Isolation and directory layout](#isolation-and-directory-layout) - [Command reference](#command-reference) @@ -74,6 +75,135 @@ python -m pload.installer --yes \ `--yes` accepts explicit values and saved defaults. It only changes a shell profile when `--shell` is supplied or already configured. +## Reproduce and share environments + +The next-version implementation on `feature/reproducible-environments` adds +`export`, `restore` and `repo`. These commands are not in the published 1.0.0. +Install this checkout into a separate development environment to try them: + +```console +python3 -m venv .dev +.dev/bin/python -m pip install -e . +.dev/bin/pload export -h -d +``` + +On Windows, use `.dev\Scripts\python.exe` and `.dev\Scripts\pload.exe`. +If using a shell function installed by 1.0.0, call the new executable directly +or refresh shell integration so it recognizes the new commands. + +### Save a recipe or a wheel bundle + +```console +pload export analysis-01 -e v1 +pload export analysis-offline -e v1 -b all +pload export training-01 -e v2 -b torch -f /mnt/existing-wheels +``` + +Each snapshot is an immutable directory under `PLOAD_HOME/snapshots/NAME`. +`requirements.txt` contains exact installed package versions, and `snapshot.json` +records Python, platform, architecture, libc information and SHA-256 checksums. +`-b all` adds all application package wheels; `-b torch,torchvision` archives only +those packages. Other pinned packages are fetched from the configured index on restore. +For a CUDA wheel index, add `-i https://download.pytorch.org/whl/cuXXX`, replacing +`cuXXX` with the index appropriate for the installed torch build. The supplied +index is recorded without credentials or query parameters; it is not automatically +trusted or selected on restore. Use `restore -i URL` when needed. + +For environments created by venv, virtualenv, uv, Poetry or Pipenv, pass their +Python executable directly (it must support Python 3.8+): + +```console +pload export other-project -p /path/to/project/.venv/bin/python +pload restore other-project -n project-copy +pload restore ./requirements.txt -n imported -v 3.12 +python -m pip install -r ~/.pload/snapshots/analysis-01/requirements.txt +``` + +Requirements exported by another tool are the interchange format. Native uv, +Poetry and Pipenv lockfiles are not parsed: export a requirements file with the +original tool first. A snapshot describes what is installed; it is not a +cross-platform solver lockfile. pip/setuptools/wheel bootstrap tooling is excluded. +Archiving wheels requires pip inside the source environment; an existing uv can +seed it, for example `uv pip install --python /path/to/.venv/bin/python pip`. + +Editable installs and source checkouts must first be built and installed as wheels. +For direct-URL or local-wheel installs, supply the original wheel via `-f DIR` +and include the package in `-b`; its SHA-256 must match the installation metadata. +Environments restored from a bundle can be re-exported with `-b all`, reusing +those exact wheels from the shared cache. +Native Conda environments are also rejected: their native libraries/channels cannot +be reproduced by pip. Keep a native Conda export, or explicitly export only pip +requirements when a pip-only conversion is intended. + +### Restore and understand the limits + +```console +pload restore analysis-offline -n analysis-copy -o +pload restore analysis-01 -n another-platform -p +``` + +By default Python major/minor, implementation, OS and architecture must match. +`-o` requires a complete `-b all` bundle and installs without an index; wheels are +verified before use and pip checks their supported platform tags. The Python +interpreter must already be installed. `-p` explicitly permits another platform +and re-resolves the pins without the bundled wheels. This can fail when that +version has no compatible build; it cannot be combined with `-o`. + +NumPy is platform-dependent too. Wheels cannot restore kernel drivers, the CUDA +driver, system libraries, compiler flags, environment variables, data files or +Conda native dependencies. OS/architecture checks are only a first guard, not a +complete ABI or GPU compatibility guarantee. The first version does not mount +remote site-packages at runtime: it fetches reusable artifacts and installs locally. +Checksums detect corruption, not a malicious repository owner. Restore only trusted +packages/requirements; installation can execute code. If installation fails, the +new environment remains for inspection; existing environments are not modified. + +### Use your own host, a directory or GitHub + +```console +pload repo add lab frpxiaoxin:/home/tibless/pload-cloud -t ssh +pload repo push lab analysis-offline +# On another machine, configure the same remote first: +pload repo pull lab analysis-offline +pload restore analysis-offline -n analysis-copy -o + +pload repo add disk /mnt/shared/pload -t local +pload repo add recipes git@github.com:YOUR-NAME/pload-recipes.git -t git +pload repo push recipes analysis-01 +pload repo list +pload repo remove lab +``` + +SSH uses your existing alias/keys and `ssh`/`scp`, with a Linux filesystem on the +server; no daemon, public HTTP endpoint or new Python dependency is needed. An +SSH alias is preferable to a hardcoded FRP IP/port because SSH already manages +the connection details. Git uses your installed Git and its authentication/identity. +Create the GitHub repository yourself and point `repo add` at its URL. Git stores +recipes only; push large wheel bundles to SSH or a local filesystem instead. +Git pushes a new snapshot commit to the repository's default branch. Pull never +installs packages automatically. Removing a remote removes only its configuration. + +Snapshot names cannot be overwritten locally or remotely: choose a new name for +each revision. Failed SSH uploads can leave `.upload-*` staging directories, which +are not visible as published snapshots. This initial version does not provide +remote garbage collection, interrupted-download resumption or a remote catalogue. + +### Reuse downloads + +`PLOAD_HOME/cache/wheels` is shared by export, restore and package installation +during `pload new`. Export/restore attempt an offline wheel lookup first, then use +pip's normal resolver and cache. `export -f DIR` can reuse existing wheel files; +it is repeatable. Pulled wheels are checksum-checked and added to the shared cache. +Identical filenames with different bytes are rejected rather than silently replaced. +SSH stores identical wheel content once under `objects/SHA256`, with hard links +from snapshots. Pull skips network transfer for identical local cached wheels. + +pip's existing HTTP/wheel cache is left in its configured location and remains +usable. Set `PIP_CACHE_DIR=/your/path` to relocate it; set `PLOAD_HOME` to relocate +pload's snapshot and shared wheel directories. Unpacked uv caches or installed +site-packages are not treated as wheel archives. A cache miss may still require +one download even when the library is already installed somewhere. + ## Upgrade Check the installed version first: diff --git a/src/pload/cli.py b/src/pload/cli.py index 048d8b3..98a6b01 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -28,6 +28,9 @@ "shell-init": ("shell",), "python": ("py",), "config": ("cfg",), + "export": (), + "restore": (), + "repo": (), } PYTHON_ALIASES = { "install": (), @@ -276,6 +279,43 @@ def build_parser(): ) config.add_argument("key", nargs="?", help="setting name for `config set`") config.add_argument("value", nargs="?", help="new value for `config set`") + export = subparsers.add_parser( + "export", description="Save exact package pins and optional platform-specific wheels.", + help="snapshot an existing environment", + ) + export.add_argument("name", help="new immutable snapshot name") + source = export.add_mutually_exclusive_group() + source.add_argument("--environment", "-e", help="environment ID, name or path (default: .venv)") + source.add_argument("--python", "-p", help="external interpreter, including uv/Conda/Poetry") + export.add_argument("--bundle", "-b", help="all or comma-separated packages to archive as wheels") + export.add_argument("--index-url", "-i", help="index used to download wheels, e.g. PyTorch CUDA") + export.add_argument("--find-links", "-f", action="append", help="existing wheel directory; repeatable") + restore = subparsers.add_parser( + "restore", description="Create a new environment from a snapshot or trusted requirements file.", + help="restore a snapshot or requirements.txt", + ) + restore.add_argument("source", help="snapshot name/directory or requirements.txt path") + restore.add_argument("--name", "-n", required=True, help="name of the new environment") + restore.add_argument("--version", "-v", help="Python ID, version or interpreter path") + restore.add_argument("--offline", "-o", action="store_true", help="install only from local wheels") + restore.add_argument("--portable", "-p", action="store_true", + help="resolve pins for another platform; ignore bundled wheels") + restore.add_argument("--index-url", "-i", help="package index for online restoration") + repo = subparsers.add_parser("repo", help="manage snapshot repositories", + description="Manage local, SSH and Git snapshot repositories.") + actions = repo.add_subparsers(dest="repo_command", required=True) + add = actions.add_parser("add", description="Save a repository location; no credentials stored.") + add.add_argument("name", help="repository nickname") + add.add_argument("location", help="directory, HOST:/absolute/path, or Git URL") + add.add_argument("--type", "-t", choices=["local", "ssh", "git"], default="local", + help="transport (default: local)") + actions.add_parser("list", description="Show configured repositories.") + remove = actions.add_parser("remove", description="Remove configuration, keeping all remote files.") + remove.add_argument("name", help="repository nickname") + for verb in ("push", "pull"): + action = actions.add_parser(verb, description=f"{verb.title()} a complete immutable snapshot.") + action.add_argument("repository", help="configured repository nickname") + action.add_argument("snapshot", help="snapshot name") return parser @@ -463,10 +503,13 @@ def _brief_help(parser, command_path, root_parser=None): "path": "Print an environment path", "config": "Show effective configuration", "shell-init": "Print shell activation integration", + "export": "Save a package snapshot and optional wheels", + "restore": "Recreate a snapshot or requirements file", + "repo": "Manage local, SSH and Git repositories", } - for command in ("new", "init", "list", "rm", "python", "path", "config", "shell-init"): + for command, summary in summaries.items(): aliases = COMMAND_ALIASES[command] - table.add_row(command, ", ".join(aliases) if aliases else "—", summaries[command]) + table.add_row(command, ", ".join(aliases) if aliases else "—", summary) console.print(table) console.print("[bold cyan]Quick start[/]") console.print(" [green]pload new[/] [dim]# guided creation[/]") @@ -554,7 +597,7 @@ def shell_script(shell): if shell in {"bash", "zsh"}: return r'''pload() { case "${1:-}" in - ""|new|init|i|rm|remove|del|delete|list|ls|path|p|shell-init|shell|python|py|config|cfg|-*) + ""|new|init|i|rm|remove|del|delete|list|ls|path|p|shell-init|shell|python|py|config|cfg|export|restore|repo|-*) command pload "$@" ;; *) @@ -567,7 +610,7 @@ def shell_script(shell): if shell == "fish": return r'''function pload switch "$argv[1]" - case '' new init i rm remove del delete list ls path p shell-init shell python py config cfg '-*' + case '' new init i rm remove del delete list ls path p shell-init shell python py config cfg export restore repo '-*' command pload $argv case '*' set -l name . @@ -581,7 +624,7 @@ def shell_script(shell): end''' return r'''function pload { param([Parameter(ValueFromRemainingArguments = $true)][string[]]$PloadArgs) - $commands = @('new', 'init', 'i', 'rm', 'remove', 'del', 'delete', 'list', 'ls', 'path', 'p', 'shell-init', 'shell', 'python', 'py', 'config', 'cfg', '-h', '--help', '--version') + $commands = @('new', 'init', 'i', 'rm', 'remove', 'del', 'delete', 'list', 'ls', 'path', 'p', 'shell-init', 'shell', 'python', 'py', 'config', 'cfg', 'export', 'restore', 'repo', '-h', '--help', '--version') $backend = Get-Command -Name @('pload.exe', 'pload.cmd') -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1 if (-not $backend) { throw 'pload executable not found on PATH' } if ($PloadArgs.Count -eq 0) { @@ -647,6 +690,29 @@ def run(argv=None): venvs = VenvManager(config) dependencies = DependencyManager(config) + if command in {"export", "restore", "repo"}: + from pload.snapshots import RepositoryManager, SnapshotManager + + snapshots = SnapshotManager(config) + if command == "export": + print(snapshots.export(args.name, args.environment, args.python, args.bundle, + args.index_url, args.find_links)) + elif command == "restore": + print(snapshots.restore(args.source, args.name, args.version, + args.offline, args.portable, args.index_url)) + else: + repositories = RepositoryManager(config) + if args.repo_command == "add": + repositories.add(args.name, args.location, args.type) + elif args.repo_command == "remove": + repositories.remove(args.name) + elif args.repo_command == "list": + print(json.dumps(repositories.repositories(), indent=2)) + else: + repositories.transfer(args.repository, args.snapshot, args.repo_command == "push") + print(f"[*] {args.repo_command}: {args.snapshot} ({args.repository})") + return 0 + if command == "new": if not any((args.python_version, args.name, args.path, args.description, args.requirements, args.channel)): guided = _guided_new(config) @@ -759,7 +825,7 @@ def run(argv=None): def main(argv=None): try: return run(argv) - except (PloadError, PythonNotFoundError, re.error) as exc: + except (PloadError, PythonNotFoundError, re.error, OSError) as exc: print(f"pload: error: {exc}", file=sys.stderr) return 1 diff --git a/src/pload/help_content.py b/src/pload/help_content.py index 7f677b7..8decd66 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -331,6 +331,75 @@ } +GUIDES[("export",)] = r""" +# Save a reproducible package snapshot + +```console +pload export analysis-01 -e v1 +pload export analysis-offline -p /path/to/project/.venv/bin/python -b all +pload export training-01 -e v2 -b torch -f /mnt/downloaded-wheels +``` + +The first command records exact installed versions and interpreter/platform metadata +under `PLOAD_HOME/snapshots/analysis-01`. Its requirements.txt also works with pip +and uv. The second adds every package's wheel for offline restoration. The third +archives only torch, so other packages still come from your package index. +Use `-i URL` to select a dedicated wheel index, such as your PyTorch CUDA index. +Existing shared wheels are tried offline first; then pip uses its normal caches. + +Names are immutable: export a new name for a new revision. Source checkouts and +editable installs must first be built and installed as wheels. Direct-wheel installs +need their original archive in `-f DIR` or the shared cache and inclusion in `-b`; +the archive hash must match the installation metadata. Native Conda environments require Conda's own +export tools. Python itself, OS libraries and GPU drivers are not bundled. +""" +GUIDES[("restore",)] = r""" +# Restore into a new environment + +```console +pload restore analysis-offline -n analysis-copy -o +pload restore analysis-01 -n linux-copy -p +pload restore ./requirements.txt -n imported -v 3.12 +``` + +`-o` uses a complete wheel snapshot without a package index. SHA-256 checks run +before creating the environment; wheel compatibility is checked by pip. The Python +major/minor version must match and must already be installed. +`-p` deliberately re-resolves the pinned versions for another platform. It is not +a promise of an identical environment and cannot be combined with offline mode. +Only import requirements files and repositories you trust: installing packages +can execute code. Exported index provenance is informational; choose the restore +index explicitly with `-i URL` when your configured index cannot supply a package. + +The target name must be unused. A failed install leaves that new environment for +inspection or removal with `pload rm`. Existing environments are never updated. +""" +GUIDES[("repo",)] = r""" +# Store recipes and reusable wheels on your own infrastructure + +```console +pload repo add lab frpxiaoxin:/home/tibless/pload-cloud -t ssh +pload repo push lab analysis-offline +pload repo pull lab analysis-offline +pload restore analysis-offline -n analysis-copy -o +pload repo add recipes git@github.com:YOUR-NAME/pload-recipes.git -t git +pload repo push recipes analysis-01 +pload repo list +pload repo remove lab +``` + +Pull on another machine (or into a different PLOAD_HOME): existing snapshots are +not overwritten. SSH uses your OpenSSH alias/keys and a Linux directory; no web +server or extra Python package is required. Identical wheels are stored once by +hash on the server and reused from the local cache on pull. +Git repositories store recipes only; use SSH or a local directory for wheel bundles. +Git authentication and commit identity come from your existing Git configuration. +Removing a repository only forgets its configuration; remote data remains intact. +""" +for _command in ("add", "list", "remove", "push", "pull"): + GUIDES[("repo", _command)] = GUIDES[("repo",)] + + def render_detailed_help(parser, command_path): console = Console(highlight=False) content = GUIDES.get(tuple(command_path), GUIDES[()]) diff --git a/src/pload/managers/dependency.py b/src/pload/managers/dependency.py index 1c5fc00..7713767 100644 --- a/src/pload/managers/dependency.py +++ b/src/pload/managers/dependency.py @@ -13,6 +13,9 @@ def install_dependencies(self, venv_path, requirements, channel=None): return command = self.config.get_pip_command(venv_path) + ["install"] + list(requirements) + wheels = self.config.home / "cache" / "wheels" + if wheels.is_dir(): + command += ["--find-links", str(wheels)] if channel: command += ["--index-url", channel] diff --git a/src/pload/snapshots.py b/src/pload/snapshots.py new file mode 100644 index 0000000..61d7119 --- /dev/null +++ b/src/pload/snapshots.py @@ -0,0 +1,437 @@ +"""Portable package recipes, checked wheel bundles and optional remote storage. + +Only Python's standard library and the target interpreter's pip are required +locally. SSH remotes use OpenSSH; Git remotes use the user's existing Git. +""" + +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import tempfile +import uuid +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +from pload.errors import PloadError +from pload.managers.venv import VenvManager +from pload.settings import save_settings + +PROBE = """ +import json, platform, sys, os +from importlib import metadata +packages = [] +for dist in metadata.distributions(): + name = dist.metadata.get('Name') + if name and name.lower() not in ('pip', 'setuptools', 'wheel'): + packages.append({'name': name, 'version': dist.version, + 'direct_url': dist.read_text('direct_url.json')}) +print(json.dumps({'python': platform.python_version(), + 'implementation': sys.implementation.name, 'system': platform.system(), + 'machine': platform.machine(), 'libc': list(platform.libc_ver()), + 'conda': os.path.isdir(os.path.join(sys.prefix, 'conda-meta')), 'packages': packages})) +""" + + +def execute(command, capture=True, cwd=None): + try: + result = subprocess.run(command, capture_output=capture, text=True, cwd=cwd, check=False) + except OSError as exc: + raise PloadError(f"cannot run {command[0]}: {exc}") from exc + if result.returncode: + detail = (result.stderr or result.stdout or "").strip() if capture else "" + raise PloadError(f"{command[0]} failed: {detail or result.returncode}") + return result.stdout.strip() if capture else "" + + +def safe_name(value): + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,99}", value or ""): + raise PloadError("use a name of 1–100 letters, numbers, dots, underscores or hyphens") + return value + + +def digest(path): + checksum = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + checksum.update(chunk) + return checksum.hexdigest() + + +def write_json(path, data): + Path(path).write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def read_json(path): + try: + return json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise PloadError(f"cannot read {path}: {exc}") from exc + + +def probe(python): + return json.loads(execute([str(python), "-c", PROBE])) + + +def public_index(url): + """Store useful provenance without passwords, tokens or URL query strings.""" + if not url: + return None + parsed = urlsplit(url) + return urlunsplit((parsed.scheme, parsed.netloc.rsplit("@", 1)[-1], parsed.path, "", "")) + + +def validate_bundle(path): + path = Path(path) + if path.is_symlink() or (path / "snapshot.json").is_symlink(): + raise PloadError("snapshot must not be a symbolic link") + data = read_json(path / "snapshot.json") + if not isinstance(data, dict) or data.get("schema") != 1: + raise PloadError("unsupported snapshot schema") + files = data.get("files") + if not isinstance(files, dict) or "requirements.txt" not in files: + raise PloadError("snapshot has no requirements checksum") + if not isinstance(data.get("runtime"), dict): + raise PloadError("snapshot has no runtime metadata") + for name, checksum in files.items(): + if name != "requirements.txt" and not re.fullmatch( + r"wheels/[A-Za-z0-9][A-Za-z0-9_.+!-]*\.whl", name + ): + raise PloadError(f"invalid snapshot file: {name}") + item = path / name + if any(part.is_symlink() for part in (item, item.parent)): + raise PloadError(f"snapshot contains a symbolic link: {name}") + if not item.is_file() or digest(item) != checksum: + raise PloadError(f"snapshot checksum mismatch or missing file: {name}") + # Snapshots contain only exact pins, never executable requirements directives. + for line in (path / "requirements.txt").read_text(encoding="utf-8").splitlines(): + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*==[A-Za-z0-9][A-Za-z0-9_.+!-]*", line): + raise PloadError(f"snapshot requires an exact package pin: {line!r}") + return data + + +class SnapshotManager: + def __init__(self, config): + self.config = config + self.root = config.home / "snapshots" + self.wheels = config.home / "cache" / "wheels" + + def path(self, name): + if name == "objects": + raise PloadError("'objects' is reserved for repository wheel storage") + return self.root / safe_name(name) + + def export(self, name, environment=None, python=None, bundle=None, index=None, links=None): + target = self.path(name) + if target.exists(): + raise PloadError(f"snapshot already exists: {name}; choose a new name") + if python: + candidate = Path(python).expanduser() + # Resolving bin/python's symlink escapes a venv and probes the base Python. + interpreter = candidate.absolute() if candidate.is_file() else self.config.get_python_path(python) + else: + env = VenvManager(self.config).resolve_existing(environment or ".") + interpreter = env / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + runtime = probe(interpreter) + if runtime.get("conda"): + raise PloadError("Conda environments contain native dependencies that pip cannot reproduce; " + "use conda export for the full environment, or export a pip requirements " + "file explicitly and pass it to pload restore") + pins = [] + exact_wheels = {} + for package in sorted(runtime.pop("packages"), key=lambda p: p["name"].lower()): + pin = f"{package['name']}=={package['version']}" + if package["direct_url"]: + provenance = json.loads(package["direct_url"]) + archive = provenance.get("archive_info", {}) + checksum = archive.get("hashes", {}).get("sha256") + if not checksum and archive.get("hash", "").startswith("sha256="): + checksum = archive["hash"].split("=", 1)[1] + normalized = re.sub(r"[-_.]+", "-", package["name"].lower()) + selected = {re.sub(r"[-_.]+", "-", item.lower()) + for item in (bundle or "").split(",")} + if checksum and (bundle == "all" or normalized in selected): + for directory in [self.wheels] + [Path(p).expanduser() for p in links or []]: + for wheel in directory.glob("*.whl"): + if digest(wheel) == checksum: + exact_wheels[pin] = str(wheel.absolute()) + break + if pin not in exact_wheels: + raise PloadError( + f"{package['name']} is a direct/local install: bundle its original wheel " + "using --bundle and --find-links (matching installation SHA-256 required). " + "Editable/VCS sources must first be built and installed as wheels." + ) + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*==[A-Za-z0-9][A-Za-z0-9_.+!-]*", pin): + raise PloadError(f"invalid installed package metadata: {pin!r}") + pins.append(pin) + self.root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".export-", dir=str(self.root)) as temporary: + stage = Path(temporary) / name + stage.mkdir() + requirements = stage / "requirements.txt" + requirements.write_text("".join(pin + "\n" for pin in pins), encoding="utf-8") + files = {"requirements.txt": digest(requirements)} + if bundle: + selected = {re.sub(r"[-_.]+", "-", p.lower()) for p in bundle.split(",")} + download = pins if bundle == "all" else [ + pin for pin in pins + if re.sub(r"[-_.]+", "-", pin.split("==")[0].lower()) in selected + ] + found = {re.sub(r"[-_.]+", "-", pin.split("==")[0].lower()) for pin in download} + if bundle != "all" and selected - found: + raise PloadError("bundled packages not installed: " + ", ".join(selected - found)) + wheel_dir = stage / "wheels" + wheel_dir.mkdir() + self.wheels.mkdir(parents=True, exist_ok=True) + if download: + download = [exact_wheels.get(pin, pin) for pin in download] + command = [str(interpreter), "-m", "pip", "download", "--only-binary=:all:", + "--no-deps", "--dest", str(wheel_dir), + "--find-links", str(self.wheels)] + for link in links or []: + command.extend(["--find-links", str(Path(link).expanduser().resolve())]) + # First try without networking; fall back to pip's normal HTTP/wheel cache. + try: + execute(command + ["--no-index"] + download) + except PloadError: + if index or self.config.settings.get("pip_index"): + command += ["--index-url", index or self.config.settings["pip_index"]] + execute(command + download, capture=False) + for wheel in sorted(wheel_dir.glob("*.whl")): + files["wheels/" + wheel.name] = digest(wheel) + cached = self.wheels / wheel.name + if cached.exists() and digest(cached) != digest(wheel): + raise PloadError(f"different wheel with same filename in cache: {wheel.name}") + if not cached.exists(): + shutil.copyfile(wheel, cached) + write_json(stage / "snapshot.json", { + "schema": 1, "name": name, "runtime": runtime, "files": files, + "bundle": bundle or "none", + "index_url": public_index(index or self.config.settings.get("pip_index")), + }) + validate_bundle(stage) + stage.rename(target) + return target + + def restore(self, source, name, version=None, offline=False, portable=False, index=None): + candidate = Path(source).expanduser() + path = candidate if candidate.exists() else self.path(source) + is_recipe = path.is_file() + data = None if is_recipe else validate_bundle(path) + if offline and is_recipe: + raise PloadError("offline mode requires a validated snapshot, not an arbitrary requirements file") + if data: + runtime = data["runtime"] + version = version or ".".join(runtime.get("python", "").split(".")[:2]) + interpreter = self.config.get_python_path(version) + if data: + actual = probe(interpreter) + for key in ("implementation", "system", "machine"): + if actual.get(key) != runtime.get(key) and not portable: + raise PloadError(f"{key} mismatch: {runtime.get(key)} vs {actual.get(key)}; " + "use --portable for a fresh platform-specific resolution") + if actual["python"].split(".")[:2] != runtime.get("python", "").split(".")[:2]: + raise PloadError("Python major/minor must match the snapshot") + if portable and offline: + raise PloadError("--portable cannot be combined with --offline") + if offline and data.get("bundle") != "all": + raise PloadError("offline restoration requires a snapshot exported with --bundle all") + requirements = path if is_recipe else path / "requirements.txt" + self.root.mkdir(parents=True, exist_ok=True) + env = VenvManager(self.config).create_venv(version=str(interpreter), name=name) + command = self.config.get_pip_command(env) + ["install", "-r", str(requirements.resolve())] + if data and not portable: + # Only checksum-verified artifacts may be used from a snapshot. + with tempfile.TemporaryDirectory(prefix=".restore-", dir=str(self.root)) as stage: + artifacts = [] + for filename in data["files"]: + if filename.startswith("wheels/"): + wheel = Path(stage) / Path(filename).name + shutil.copyfile(path / filename, wheel) + if digest(wheel) != data["files"][filename]: + raise PloadError("snapshot changed during restoration") + artifacts.append(str(wheel)) + if artifacts: + # Install exact verified files first so an index cannot replace them. + execute(self.config.get_pip_command(env) + + ["install", "--no-index", "--no-deps"] + artifacts, capture=False) + command += ["--find-links", stage] + self._install(command, offline, index) + else: + self._install(command, offline, index) + execute(self.config.get_pip_command(env) + ["check"], capture=False) + return env + + def _install(self, command, offline, index): + if self.wheels.is_dir(): + command += ["--find-links", str(self.wheels)] + if offline: + command += ["--no-index", "--no-deps"] + elif index or self.config.settings.get("pip_index"): + command += ["--index-url", index or self.config.settings["pip_index"]] + try: + if not offline: + try: + execute(command + ["--no-index", "--only-binary=:all:"]) + return + except PloadError: + pass + execute(command, capture=False) + except PloadError as exc: + raise PloadError(f"restore failed; the new environment is kept for inspection: {exc}") + + +class RepositoryManager: + def __init__(self, config): + self.config = config + self.snapshots = SnapshotManager(config) + + def repositories(self): + return self.config.settings.get("repositories", {}) + + def add(self, name, location, kind): + safe_name(name) + repositories = dict(self.repositories()) + if name in repositories: + raise PloadError("repository exists; remove its configuration before replacing it") + if kind == "ssh": + self.ssh_location(location) + elif kind == "local": + location = str(Path(location).expanduser().resolve()) + elif location.startswith("-") or not location: + raise PloadError("invalid Git location") + repositories[name] = {"kind": kind, "location": location} + self.config.settings["repositories"] = repositories + save_settings(self.config.home, self.config.settings) + + def remove(self, name): + repositories = dict(self.repositories()) + if name not in repositories: + raise PloadError(f"unknown repository: {name}") + del repositories[name] + self.config.settings["repositories"] = repositories + save_settings(self.config.home, self.config.settings) + + @staticmethod + def ssh_location(location): + host, separator, path = location.partition(":") + if (not separator or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.@-]*", host) + or not re.fullmatch(r"/[A-Za-z0-9_./-]+", path) + or ".." in path.split("/") or path == "/"): + raise PloadError("SSH location must be HOST:/absolute/directory (no spaces or '..')") + return host, path.rstrip("/") + + def transfer(self, repo, name, push=False): + safe_name(name) + configured = self.repositories().get(repo) + if not configured: + raise PloadError(f"unknown repository: {repo}") + target = self.snapshots.path(name) + if not push and target.exists(): + raise PloadError("local snapshot exists; immutable snapshots are never overwritten") + if push: + data = validate_bundle(target) + self.snapshots.root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".transfer-", dir=str(self.snapshots.root)) as temp: + temp = Path(temp) + kind, location = configured["kind"], configured["location"] + if kind == "ssh": + host, root = self.ssh_location(location) + remote = root + "/" + name + if push: + # Publish a complete staging directory atomically; refuse overwrites. + stage = root + "/.upload-" + uuid.uuid4().hex + execute(["ssh", "-o", "BatchMode=yes", host, + f"mkdir -p {shlex.quote(stage + '/wheels')}"]) + for filename in ["snapshot.json"] + list(data["files"]): + if filename.startswith("wheels/"): + obj = root + "/objects/" + data["files"][filename] + execute(["ssh", "-o", "BatchMode=yes", host, + f"mkdir -p {shlex.quote(root + '/objects')}"]) + exists = execute(["ssh", "-o", "BatchMode=yes", host, + f"if test -f {shlex.quote(obj)}; then printf yes; fi"]) + if exists != "yes": + pending = obj + "." + uuid.uuid4().hex + execute(["scp", "-q", "-o", "BatchMode=yes", str(target / filename), + host + ":" + pending]) + execute(["ssh", "-o", "BatchMode=yes", host, + f"mv -n {shlex.quote(pending)} {shlex.quote(obj)}"]) + execute(["ssh", "-o", "BatchMode=yes", host, + f"ln {shlex.quote(obj)} {shlex.quote(stage + '/' + filename)}"]) + else: + execute(["scp", "-q", "-o", "BatchMode=yes", str(target / filename), + host + ":" + stage + "/" + filename]) + execute(["ssh", "-o", "BatchMode=yes", host, + (f"test ! -e {shlex.quote(remote)} && " + f"mv -T {shlex.quote(stage)} {shlex.quote(remote)}")]) + else: + stage = temp / name + stage.mkdir() + execute(["scp", "-q", "-o", "BatchMode=yes", + host + ":" + remote + "/snapshot.json", str(stage / "snapshot.json")]) + data = read_json(stage / "snapshot.json") + files = data.get("files", {}) if isinstance(data, dict) else {} + if not isinstance(files, dict): + raise PloadError("invalid remote file list") + for filename in files: + if filename != "requirements.txt" and not re.fullmatch( + r"wheels/[A-Za-z0-9][A-Za-z0-9_.+!-]*\.whl", filename + ): + raise PloadError("invalid remote filename") + (stage / filename).parent.mkdir(parents=True, exist_ok=True) + cached = self.snapshots.wheels / Path(filename).name + if filename.startswith("wheels/") and cached.is_file() and digest(cached) == files[filename]: + shutil.copyfile(cached, stage / filename) + else: + execute(["scp", "-q", "-o", "BatchMode=yes", + host + ":" + remote + "/" + filename, str(stage / filename)]) + validate_bundle(stage) + self.cache_bundle(stage, data) + stage.rename(target) + return + if kind == "git": + if push and any(f.startswith("wheels/") for f in data["files"]): + raise PloadError("Git repositories store recipes only; use SSH/local for wheels") + checkout = temp / "git" + execute(["git", "clone", "--", location, str(checkout)]) + remote = checkout / "snapshots" / name + elif kind == "local": + remote = Path(location) / name + else: + raise PloadError(f"unsupported repository kind: {kind}") + if push: + if remote.exists(): + raise PloadError("remote snapshot already exists; choose a new snapshot name") + remote.parent.mkdir(parents=True, exist_ok=True) + remote.mkdir() + for filename in ["snapshot.json"] + list(data["files"]): + (remote / filename).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(target / filename, remote / filename) + if kind == "git": + execute(["git", "add", "--", "snapshots/" + name], cwd=checkout) + execute(["git", "commit", "-m", "Add pload snapshot " + name], cwd=checkout) + execute(["git", "push", "origin", "HEAD"], cwd=checkout) + else: + data = validate_bundle(remote) + stage = temp / name + stage.mkdir() + for filename in ["snapshot.json"] + list(data["files"]): + (stage / filename).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(remote / filename, stage / filename) + validate_bundle(stage) + self.cache_bundle(stage, data) + stage.rename(target) + + def cache_bundle(self, path, data): + self.snapshots.wheels.mkdir(parents=True, exist_ok=True) + for filename, checksum in data["files"].items(): + if filename.startswith("wheels/"): + cached = self.snapshots.wheels / Path(filename).name + if cached.exists() and digest(cached) != checksum: + raise PloadError(f"cache filename collision: {cached.name}") + if not cached.exists(): + shutil.copyfile(path / filename, cached) diff --git a/tests/test_snapshots.py b/tests/test_snapshots.py new file mode 100644 index 0000000..9b48290 --- /dev/null +++ b/tests/test_snapshots.py @@ -0,0 +1,183 @@ +import json +import shutil +import sys +import zipfile + +import pytest + +from pload.cli import build_parser, main, shell_script +from pload.errors import PloadError +from pload.managers.platform import ConfigManager +from pload.managers.venv import VenvManager +from pload.snapshots import ( + RepositoryManager, + SnapshotManager, + digest, + execute, + public_index, + validate_bundle, + write_json, +) + + +def tiny_wheel(directory): + directory.mkdir(parents=True, exist_ok=True) + path = directory / "pload_demo-1.0-py3-none-any.whl" + with zipfile.ZipFile(path, "w") as wheel: + wheel.writestr("pload_demo.py", "answer = 42\n") + wheel.writestr("pload_demo-1.0.dist-info/METADATA", + "Metadata-Version: 2.1\nName: pload-demo\nVersion: 1.0\n") + wheel.writestr("pload_demo-1.0.dist-info/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n") + wheel.writestr("pload_demo-1.0.dist-info/RECORD", "") + return path + + +def recipe(path): + path.mkdir(parents=True) + (path / "requirements.txt").write_text("", encoding="utf-8") + write_json(path / "snapshot.json", { + "schema": 1, "runtime": {}, + "files": {"requirements.txt": digest(path / "requirements.txt")}, + }) + return path + + +def test_offline_roundtrip_external_interpreter_and_shared_cache(tmp_path): + config = ConfigManager(home=tmp_path / "home") + wheels = tmp_path / "existing-wheels" + tiny_wheel(wheels) + source = VenvManager(config).create_venv(name="source") + execute(config.get_pip_command(source) + ["install", "--no-index", "--find-links", + str(wheels), "pload-demo==1.0"]) + manager = SnapshotManager(config) + python = config.get_pip_command(source)[0] + snapshot = manager.export("demo", python=python, bundle="all", links=[str(wheels)]) + assert (snapshot / "requirements.txt").read_text().strip() == "pload-demo==1.0" + assert (manager.wheels / "pload_demo-1.0-py3-none-any.whl").is_file() + restored = manager.restore("demo", "restored", version=sys.executable, offline=True) + output = execute([config.get_pip_command(restored)[0], "-c", + "import pload_demo; print(pload_demo.answer)"]) + assert output == "42" + # A restored wheel has direct_url metadata; exporting it again must reuse + # the exact original archive from the shared cache, not require an index. + reexport = manager.export("restored-again", environment="restored", bundle="all") + assert validate_bundle(reexport)["files"] == validate_bundle(snapshot)["files"] + with pytest.raises(PloadError, match="already exists"): + manager.export("demo", python=python) + repos = RepositoryManager(config) + repos.add("local", str(tmp_path / "remote"), "local") + repos.transfer("local", "demo", push=True) + second = ConfigManager(home=tmp_path / "second") + reader = RepositoryManager(second) + reader.add("local", str(tmp_path / "remote"), "local") + reader.transfer("local", "demo") + validate_bundle(reader.snapshots.path("demo")) + assert (reader.snapshots.wheels / "pload_demo-1.0-py3-none-any.whl").exists() + + +def test_tampering_is_rejected_before_creating_environment(tmp_path): + manager = SnapshotManager(ConfigManager(home=tmp_path)) + snapshot = recipe(manager.path("demo")) + (snapshot / "requirements.txt").write_text("unexpected==1\n", encoding="utf-8") + with pytest.raises(PloadError, match="checksum"): + manager.restore("demo", "nope") + assert not manager.config.venv_path.exists() + + +@pytest.mark.parametrize("filename", ["../escape.whl", "/tmp/escape.whl", "wheels/../../escape.whl"]) +def test_unsafe_manifest_paths_rejected(tmp_path, filename): + snapshot = recipe(tmp_path / "demo") + data = json.loads((snapshot / "snapshot.json").read_text()) + data["files"][filename] = "0" * 64 + write_json(snapshot / "snapshot.json", data) + with pytest.raises(PloadError, match="invalid snapshot file"): + validate_bundle(snapshot) + + +def test_runtime_mismatch_does_not_create_environment(tmp_path): + manager = SnapshotManager(ConfigManager(home=tmp_path)) + recipe(manager.path("demo")) + with pytest.raises(PloadError, match="mismatch"): + manager.restore("demo", "nope", version=sys.executable) + assert not manager.config.venv_path.exists() + + +@pytest.mark.parametrize("location", ["-oProxyCommand:x", "host:relative", "host:/a/../b", + "host:/a;touch-b", "host:/"]) +def test_ssh_location_validation(location): + with pytest.raises(PloadError): + RepositoryManager.ssh_location(location) + + +def test_git_recipe_roundtrip(tmp_path, monkeypatch): + if not shutil.which("git"): + pytest.skip("Git is optional") + for key, value in {"GIT_AUTHOR_NAME": "pload test", "GIT_COMMITTER_NAME": "pload test", + "GIT_AUTHOR_EMAIL": "test@example.invalid", + "GIT_COMMITTER_EMAIL": "test@example.invalid"}.items(): + monkeypatch.setenv(key, value) + bare = tmp_path / "remote.git" + execute(["git", "init", "--bare", str(bare)]) + writer = RepositoryManager(ConfigManager(home=tmp_path / "writer")) + recipe(writer.snapshots.path("demo")) + writer.add("git", str(bare), "git") + writer.transfer("git", "demo", push=True) + reader = RepositoryManager(ConfigManager(home=tmp_path / "reader")) + reader.add("git", str(bare), "git") + reader.transfer("git", "demo") + validate_bundle(reader.snapshots.path("demo")) + + +def test_commands_and_shell_integration(capsys): + assert build_parser().parse_args(["export", "example", "-b", "all"]).bundle == "all" + for command in ("export", "restore", "repo"): + assert main([command, "-h"]) == 0 + for shell in ("bash", "zsh", "fish", "powershell"): + assert command in shell_script(shell) + assert "--bundle" in capsys.readouterr().out + + +def test_source_credentials_are_not_recorded(): + assert public_index("https://user:secret@example.com/simple?token=secret") == ( + "https://example.com/simple" + ) + + +def test_conda_export_is_explicitly_rejected(tmp_path, monkeypatch): + monkeypatch.setattr("pload.snapshots.probe", lambda _: {"conda": True}) + manager = SnapshotManager(ConfigManager(home=tmp_path)) + with pytest.raises(PloadError, match="Conda"): + manager.export("native", python=sys.executable) + assert not manager.path("native").exists() + + +def test_editable_source_is_not_silently_pinned(tmp_path, monkeypatch): + monkeypatch.setattr("pload.snapshots.probe", lambda _: { + "packages": [{"name": "editable", "version": "1.0", "direct_url": + json.dumps({"url": "file:///source", "dir_info": {"editable": True}})}], + }) + manager = SnapshotManager(ConfigManager(home=tmp_path)) + with pytest.raises(PloadError, match="original wheel"): + manager.export("source", python=sys.executable, bundle="all") + assert not manager.path("source").exists() + + +def test_local_repository_refuses_overwrite(tmp_path): + manager = RepositoryManager(ConfigManager(home=tmp_path / "home")) + recipe(manager.snapshots.path("demo")) + manager.add("disk", str(tmp_path / "remote"), "local") + manager.transfer("disk", "demo", push=True) + with pytest.raises(PloadError, match="already exists"): + manager.transfer("disk", "demo", push=True) + manager.remove("disk") + validate_bundle(tmp_path / "remote" / "demo") + + +def test_offline_requirements_cannot_fetch_direct_urls(tmp_path): + requirements = tmp_path / "requirements.txt" + requirements.write_text("package @ https://example.invalid/package.whl\n") + manager = SnapshotManager(ConfigManager(home=tmp_path / "home")) + with pytest.raises(PloadError, match="validated snapshot"): + manager.restore(str(requirements), "offline", offline=True) + assert not manager.config.venv_path.exists() From 718dd54e19e90b45a38d8e2f8256c25d87950952 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Thu, 24 Sep 2026 20:26:01 +0800 Subject: [PATCH 02/41] Verify remote objects and publish filesystem snapshots atomically --- src/pload/snapshots.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/pload/snapshots.py b/src/pload/snapshots.py index 61d7119..7eaae02 100644 --- a/src/pload/snapshots.py +++ b/src/pload/snapshots.py @@ -360,6 +360,10 @@ def transfer(self, repo, name, push=False): host + ":" + pending]) execute(["ssh", "-o", "BatchMode=yes", host, f"mv -n {shlex.quote(pending)} {shlex.quote(obj)}"]) + remote_hash = execute(["ssh", "-o", "BatchMode=yes", host, + f"sha256sum {shlex.quote(obj)}"]) + if remote_hash.split()[0] != data["files"][filename]: + raise PloadError("remote wheel checksum mismatch; snapshot not published") execute(["ssh", "-o", "BatchMode=yes", host, f"ln {shlex.quote(obj)} {shlex.quote(stage + '/' + filename)}"]) else: @@ -407,10 +411,14 @@ def transfer(self, repo, name, push=False): if remote.exists(): raise PloadError("remote snapshot already exists; choose a new snapshot name") remote.parent.mkdir(parents=True, exist_ok=True) - remote.mkdir() - for filename in ["snapshot.json"] + list(data["files"]): - (remote / filename).parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(target / filename, remote / filename) + with tempfile.TemporaryDirectory(prefix=".publish-", dir=str(remote.parent)) as pending: + stage = Path(pending) / name + stage.mkdir() + for filename in ["snapshot.json"] + list(data["files"]): + (stage / filename).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(target / filename, stage / filename) + validate_bundle(stage) + stage.rename(remote) if kind == "git": execute(["git", "add", "--", "snapshots/" + name], cwd=checkout) execute(["git", "commit", "-m", "Add pload snapshot " + name], cwd=checkout) From 9e59cf4edbb3e7687cff1db310c109be13003b21 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Thu, 24 Sep 2026 20:27:12 +0800 Subject: [PATCH 03/41] Avoid hashing unrelated wheel archives when re-exporting --- src/pload/snapshots.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pload/snapshots.py b/src/pload/snapshots.py index 7eaae02..c785f7c 100644 --- a/src/pload/snapshots.py +++ b/src/pload/snapshots.py @@ -156,9 +156,14 @@ def export(self, name, environment=None, python=None, bundle=None, index=None, l if checksum and (bundle == "all" or normalized in selected): for directory in [self.wheels] + [Path(p).expanduser() for p in links or []]: for wheel in directory.glob("*.whl"): + wheel_name = re.sub(r"[-_.]+", "-", wheel.name.split("-")[0].lower()) + if wheel_name != normalized: + continue if digest(wheel) == checksum: exact_wheels[pin] = str(wheel.absolute()) break + if pin in exact_wheels: + break if pin not in exact_wheels: raise PloadError( f"{package['name']} is a direct/local install: bundle its original wheel " From fe4084ddc37d79cfddc9566ff68c68f40fb753d0 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Thu, 24 Sep 2026 20:27:49 +0800 Subject: [PATCH 04/41] Preserve wheel hashes with older pip during offline restore --- src/pload/snapshots.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pload/snapshots.py b/src/pload/snapshots.py index c785f7c..513231a 100644 --- a/src/pload/snapshots.py +++ b/src/pload/snapshots.py @@ -259,7 +259,9 @@ def restore(self, source, name, version=None, offline=False, portable=False, ind shutil.copyfile(path / filename, wheel) if digest(wheel) != data["files"][filename]: raise PloadError("snapshot changed during restoration") - artifacts.append(str(wheel)) + # Old pip records hashes only when supplied in the file URL. + artifacts.append(wheel.resolve().as_uri() + "#sha256=" + + data["files"][filename]) if artifacts: # Install exact verified files first so an index cannot replace them. execute(self.config.get_pip_command(env) + From 6f6a893fb8f147b6e3c60b65c64e730a5683bd58 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Thu, 24 Sep 2026 20:29:59 +0800 Subject: [PATCH 05/41] Preserve repository and state configuration during setup --- src/pload/installer.py | 3 ++- tests/test_installer.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/pload/installer.py b/src/pload/installer.py index 1d8d276..0cfceba 100644 --- a/src/pload/installer.py +++ b/src/pload/installer.py @@ -367,10 +367,11 @@ def collect_settings(args): pip_index = ask("custom Python package index URL", "https://example.com/simple", False) return { + **existing, "home": str(home), "bin_dir": str(bin_dir), "venvs_dir": str(venvs_dir), - "state_dir": str(home / "state"), + "state_dir": existing.get("state_dir", str(home / "state")), "pip_source": pip_source, "pip_index": pip_index, "package_spec": args.package_spec or existing.get("package_spec") or default_package_spec(), diff --git a/tests/test_installer.py b/tests/test_installer.py index a979c3c..ed13827 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -60,6 +60,19 @@ def test_official_pip_source_is_explicit(tmp_path): assert settings["pip_index"] == PYPI_OFFICIAL_INDEX +def test_setup_preserves_repositories_and_custom_state(tmp_path): + from pload.settings import save_settings + + original = { + "repositories": {"lab": {"kind": "ssh", "location": "host:/srv/pload"}}, + "state_dir": str(tmp_path / "custom-state"), + } + save_settings(tmp_path / "pload", original) + collected = collect_settings(installer_args(tmp_path)) + assert collected["repositories"] == original["repositories"] + assert collected["state_dir"] == original["state_dir"] + + def test_additional_pip_source_presets(tmp_path): ustc = collect_settings(installer_args(tmp_path, pip_source="ustc")) aliyun = collect_settings(installer_args(tmp_path, pip_source="aliyun")) From 7147f8ae8dd69b405074a94670cd6df76a235488 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Thu, 24 Sep 2026 20:30:16 +0800 Subject: [PATCH 06/41] Support repository listing alias advertised in help --- src/pload/cli.py | 4 ++-- tests/test_snapshots.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pload/cli.py b/src/pload/cli.py index 98a6b01..3f22cbd 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -309,7 +309,7 @@ def build_parser(): add.add_argument("location", help="directory, HOST:/absolute/path, or Git URL") add.add_argument("--type", "-t", choices=["local", "ssh", "git"], default="local", help="transport (default: local)") - actions.add_parser("list", description="Show configured repositories.") + actions.add_parser("list", aliases=["ls"], description="Show configured repositories.") remove = actions.add_parser("remove", description="Remove configuration, keeping all remote files.") remove.add_argument("name", help="repository nickname") for verb in ("push", "pull"): @@ -706,7 +706,7 @@ def run(argv=None): repositories.add(args.name, args.location, args.type) elif args.repo_command == "remove": repositories.remove(args.name) - elif args.repo_command == "list": + elif args.repo_command in {"list", "ls"}: print(json.dumps(repositories.repositories(), indent=2)) else: repositories.transfer(args.repository, args.snapshot, args.repo_command == "push") diff --git a/tests/test_snapshots.py b/tests/test_snapshots.py index 9b48290..fbe106d 100644 --- a/tests/test_snapshots.py +++ b/tests/test_snapshots.py @@ -138,6 +138,11 @@ def test_commands_and_shell_integration(capsys): assert "--bundle" in capsys.readouterr().out +def test_repository_list_alias(tmp_path, capsys): + assert main(["-H", str(tmp_path), "repo", "ls"]) == 0 + assert capsys.readouterr().out.strip() == "{}" + + def test_source_credentials_are_not_recorded(): assert public_index("https://user:secret@example.com/simple?token=secret") == ( "https://example.com/simple" From 6901d949a4bc5fc7cfaece3ee5e4d2513ed9b24b Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 03:18:54 +0800 Subject: [PATCH 07/41] Add declarative environment reproduction --- CHANGELOG.md | 26 +- README.md | 141 +------ docs/MAINTENANCE.md | 7 + docs/declarative-environments.md | 201 ++++++++++ pyproject.toml | 14 +- src/pload/__init__.py | 2 +- src/pload/cli.py | 147 ++++--- src/pload/declarative.py | 670 +++++++++++++++++++++++++++++++ src/pload/help_content.py | 109 ++--- src/pload/snapshots.py | 18 +- tests/test_declarative.py | 188 +++++++++ tests/test_snapshots.py | 7 +- 12 files changed, 1288 insertions(+), 242 deletions(-) create mode 100644 docs/declarative-environments.md create mode 100644 src/pload/declarative.py create mode 100644 tests/test_declarative.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fa8e228..0a5cc4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,17 +2,25 @@ ## Unreleased +Pre-release: `1.1.0a2` + ### Added -- Export exact package snapshots from managed and external Python environments, - with optional selected/all wheel bundles and checksums. -- Restore snapshots offline, explicitly re-resolve for another platform, or - import trusted requirements files into new environments. -- Configure local, SSH and Git recipe repositories; push and pull immutable snapshots. -- Reuse local wheel archives and pip caches, deduplicate SSH wheel storage by SHA-256, - and avoid re-downloading identical cached wheels during pull. -- Document interoperability boundaries, GPU/native dependencies, source-wheel - provenance and repository trust requirements. +- Make one portable `pload.toml` the source of truth for desired Python, + exact dependencies, policies, repositories and locked artifact hashes. +- Add `pload describe`, `pload plan` and idempotent `pload apply`; resource + copying, downloading and publication are internal planner/executor steps. +- Inventory the local wheel cache, adjacent artifacts, local/SSH content stores, + configured indexes and Python installations before choosing a route. +- Allow package-specific indexes in `describe`, including CUDA wheel indexes, + while keeping credentials out of the portable configuration. +- Reuse exact wheel archives by SHA-256, deduplicate local/SSH storage, record + detected NVIDIA driver provenance and roll back failed materializations. + +### Removed + +- Drop the unreleased transport-oriented `export`, `restore`, and repository + push/pull interface in favor of the declarative configuration workflow. ### Fixed diff --git a/README.md b/README.md index 06076ce..dc2cf21 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,18 @@ complete parameter reference and `-h -d` for practical examples and effects. ## Contents - [Install](#install) +- [Reproduce and share environments](#reproduce-and-share-environments) - [Upgrade](#upgrade) - [First run](#first-run) - [Configure later](#configure-later) - [Shell activation](#shell-activation) - [Discover and install Python](#discover-and-install-python) - [Create and activate environments](#create-and-activate-environments) -- [Reproduce and share environments](#reproduce-and-share-environments) - [Help and aliases](#help-and-aliases) - [Isolation and directory layout](#isolation-and-directory-layout) - [Command reference](#command-reference) - [Development](#development) +- [License](#license) ## Install @@ -77,132 +78,20 @@ profile when `--shell` is supplied or already configured. ## Reproduce and share environments -The next-version implementation on `feature/reproducible-environments` adds -`export`, `restore` and `repo`. These commands are not in the published 1.0.0. -Install this checkout into a separate development environment to try them: - -```console -python3 -m venv .dev -.dev/bin/python -m pip install -e . -.dev/bin/pload export -h -d -``` - -On Windows, use `.dev\Scripts\python.exe` and `.dev\Scripts\pload.exe`. -If using a shell function installed by 1.0.0, call the new executable directly -or refresh shell integration so it recognizes the new commands. - -### Save a recipe or a wheel bundle - -```console -pload export analysis-01 -e v1 -pload export analysis-offline -e v1 -b all -pload export training-01 -e v2 -b torch -f /mnt/existing-wheels -``` - -Each snapshot is an immutable directory under `PLOAD_HOME/snapshots/NAME`. -`requirements.txt` contains exact installed package versions, and `snapshot.json` -records Python, platform, architecture, libc information and SHA-256 checksums. -`-b all` adds all application package wheels; `-b torch,torchvision` archives only -those packages. Other pinned packages are fetched from the configured index on restore. -For a CUDA wheel index, add `-i https://download.pytorch.org/whl/cuXXX`, replacing -`cuXXX` with the index appropriate for the installed torch build. The supplied -index is recorded without credentials or query parameters; it is not automatically -trusted or selected on restore. Use `restore -i URL` when needed. - -For environments created by venv, virtualenv, uv, Poetry or Pipenv, pass their -Python executable directly (it must support Python 3.8+): - -```console -pload export other-project -p /path/to/project/.venv/bin/python -pload restore other-project -n project-copy -pload restore ./requirements.txt -n imported -v 3.12 -python -m pip install -r ~/.pload/snapshots/analysis-01/requirements.txt -``` - -Requirements exported by another tool are the interchange format. Native uv, -Poetry and Pipenv lockfiles are not parsed: export a requirements file with the -original tool first. A snapshot describes what is installed; it is not a -cross-platform solver lockfile. pip/setuptools/wheel bootstrap tooling is excluded. -Archiving wheels requires pip inside the source environment; an existing uv can -seed it, for example `uv pip install --python /path/to/.venv/bin/python pip`. - -Editable installs and source checkouts must first be built and installed as wheels. -For direct-URL or local-wheel installs, supply the original wheel via `-f DIR` -and include the package in `-b`; its SHA-256 must match the installation metadata. -Environments restored from a bundle can be re-exported with `-b all`, reusing -those exact wheels from the shared cache. -Native Conda environments are also rejected: their native libraries/channels cannot -be reproduced by pip. Keep a native Conda export, or explicitly export only pip -requirements when a pip-only conversion is intended. - -### Restore and understand the limits +The `1.1` pre-release uses one `pload.toml` as the portable source of truth. +The user declares the desired environment; pload inventories local caches, +configured local/SSH content stores, package indexes and Python installations, +then performs transfers and installation internally: ```console -pload restore analysis-offline -n analysis-copy -o -pload restore analysis-01 -n another-platform -p +pload describe v2 -o pload.toml +pload plan pload.toml +pload apply pload.toml ``` -By default Python major/minor, implementation, OS and architecture must match. -`-o` requires a complete `-b all` bundle and installs without an index; wheels are -verified before use and pip checks their supported platform tags. The Python -interpreter must already be installed. `-p` explicitly permits another platform -and re-resolves the pins without the bundled wheels. This can fail when that -version has no compatible build; it cannot be combined with `-o`. - -NumPy is platform-dependent too. Wheels cannot restore kernel drivers, the CUDA -driver, system libraries, compiler flags, environment variables, data files or -Conda native dependencies. OS/architecture checks are only a first guard, not a -complete ABI or GPU compatibility guarantee. The first version does not mount -remote site-packages at runtime: it fetches reusable artifacts and installs locally. -Checksums detect corruption, not a malicious repository owner. Restore only trusted -packages/requirements; installation can execute code. If installation fails, the -new environment remains for inspection; existing environments are not modified. - -### Use your own host, a directory or GitHub - -```console -pload repo add lab frpxiaoxin:/home/tibless/pload-cloud -t ssh -pload repo push lab analysis-offline -# On another machine, configure the same remote first: -pload repo pull lab analysis-offline -pload restore analysis-offline -n analysis-copy -o - -pload repo add disk /mnt/shared/pload -t local -pload repo add recipes git@github.com:YOUR-NAME/pload-recipes.git -t git -pload repo push recipes analysis-01 -pload repo list -pload repo remove lab -``` - -SSH uses your existing alias/keys and `ssh`/`scp`, with a Linux filesystem on the -server; no daemon, public HTTP endpoint or new Python dependency is needed. An -SSH alias is preferable to a hardcoded FRP IP/port because SSH already manages -the connection details. Git uses your installed Git and its authentication/identity. -Create the GitHub repository yourself and point `repo add` at its URL. Git stores -recipes only; push large wheel bundles to SSH or a local filesystem instead. -Git pushes a new snapshot commit to the repository's default branch. Pull never -installs packages automatically. Removing a remote removes only its configuration. - -Snapshot names cannot be overwritten locally or remotely: choose a new name for -each revision. Failed SSH uploads can leave `.upload-*` staging directories, which -are not visible as published snapshots. This initial version does not provide -remote garbage collection, interrupted-download resumption or a remote catalogue. - -### Reuse downloads - -`PLOAD_HOME/cache/wheels` is shared by export, restore and package installation -during `pload new`. Export/restore attempt an offline wheel lookup first, then use -pip's normal resolver and cache. `export -f DIR` can reuse existing wheel files; -it is repeatable. Pulled wheels are checksum-checked and added to the shared cache. -Identical filenames with different bytes are rejected rather than silently replaced. -SSH stores identical wheel content once under `objects/SHA256`, with hard links -from snapshots. Pull skips network transfer for identical local cached wheels. - -pip's existing HTTP/wheel cache is left in its configured location and remains -usable. Set `PIP_CACHE_DIR=/your/path` to relocate it; set `PLOAD_HOME` to relocate -pload's snapshot and shared wheel directories. Unpacked uv caches or installed -site-packages are not treated as wheel archives. A cache miss may still require -one download even when the library is already installed somewhere. +See the [declarative environment guide](docs/declarative-environments.md) for the +schema, exact versus compatible policies, resource-selection algorithm, CUDA +wheel caching, idempotency and current platform boundaries. ## Upgrade @@ -220,6 +109,12 @@ and virtual environments are kept: pload-install --yes --package-spec "pload==1.0.0" ``` +To opt into the declarative-environment pre-release for testing: + +```console +pload-install --yes --package-spec "pload==1.1.0a2" +``` + To always follow the newest published release instead of pinning a version, omit the version constraint: diff --git a/docs/MAINTENANCE.md b/docs/MAINTENANCE.md index 0b3685d..fb30491 100644 --- a/docs/MAINTENANCE.md +++ b/docs/MAINTENANCE.md @@ -3,6 +3,13 @@ Changes are developed on a release branch and merged to `main` only after the release checklist passes. Each stable merge receives a matching annotated tag. +## Pre-release testing + +Every candidate uses a PEP 440 pre-release version. After the cross-platform +matrix passes, install its wheel into the maintainer's private macOS pload runtime +without changing `PLOAD_HOME`, environment state or caches. Promotion to a stable +version requires explicit acceptance after that Mac test. + ## 0.4.x: portable foundation - Centralize filesystem and interpreter resolution. diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md new file mode 100644 index 0000000..a6393e7 --- /dev/null +++ b/docs/declarative-environments.md @@ -0,0 +1,201 @@ +# Declarative environments + +The `1.1` pre-release replaces the transport-oriented snapshot experiment with +one portable environment configuration. Users describe the desired state; pload +discovers resources, chooses a deterministic plan and materializes that state. + +```console +pload describe v2 -o pload.toml +pload apply pload.toml +``` + +`describe` is the bridge for an existing environment. For a new project, the +same file may be written by hand with exact `NAME==VERSION` dependencies and a +`compatible` policy, then locked more strictly after it has been materialized. + +## One file, two responsibilities + +`pload.toml` contains both: + +- **specification** — the Python, dependencies, target platform and policy the + user wants; +- **lock** — exact package versions, wheel identities, SHA-256 hashes, compatible + tags and logical resource locations discovered by pload. + +The file is the control plane. Python runtimes and wheel bytes remain in caches, +indexes or content-addressed repositories and are the data plane. Credentials are +never stored in the file; SSH uses the existing SSH agent/configuration. + +```toml +schema = 1 +name = "llm-training" + +[environment] +python = "3.12.7" +implementation = "cpython" +system = "Linux" +machine = "x86_64" +dependencies = ["numpy==2.1.3", "torch==2.5.0+cu121"] + +[policy] +reproducibility = "exact" +network = "allow" +source_build = "fallback" +publish_missing_artifacts = true +repositories = ["lab"] + +[sources.default] +kind = "index" +url = "https://pypi.org/simple" + +[repositories.lab] +kind = "ssh" +location = "frpxiaoxin:/home/tibless/pload-cloud" + +[[package]] +name = "torch" +version = "2.5.0+cu121" +sources = ["default"] + +[[package.artifact]] +filename = "torch-2.5.0+cu121-cp312-cp312-linux_x86_64.whl" +sha256 = "..." +tags = ["cp312-cp312-linux_x86_64"] +repositories = ["lab"] +``` + +`describe` writes the lock entries automatically. URLs containing embedded +credentials or query-string tokens are rejected; authentication belongs to the +machine, not a shareable environment description. + +## Describe an existing environment + +```console +pload describe v2 +pload describe /project/.venv -o project.pload.toml +pload describe /project/.venv/bin/python -o project.pload.toml +``` + +The default `exact` mode obtains one wheel for every application package, +computes its SHA-256, places it in the shared cache and automatically publishes +it to the first configured local/SSH artifact repository. The repository is +embedded in the resulting configuration as a logical resource provider. + +Use a specific configured repository when necessary: + +```console +pload describe v2 -r lab +``` + +Installed metadata does not always preserve the original package index. Record a +dedicated source explicitly for packages such as CUDA-enabled PyTorch; repeat the +option for multiple packages: + +```console +pload describe v2 \ + -s torch=https://download.pytorch.org/whl/cu121 \ + -s torchvision=https://download.pytorch.org/whl/cu121 +``` + +If a package has no compatible wheel, exact description stops instead of quietly +claiming reproducibility. `--mode compatible` records exact versions but permits +a future machine to re-resolve their artifacts: + +```console +pload describe v2 --mode compatible +``` + +Compatible mode is useful for exploration, but it is weaker than an artifact lock. + +## Plan against available resources + +```console +pload plan +pload plan project.pload.toml +pload plan --offline +pload plan --json +``` + +The planner inventories: + +1. installed and discoverable Python interpreters; +2. the pload wheel cache; +3. an `artifacts/` directory beside the configuration; +4. content-addressed local and SSH repositories named in the file; +5. package indexes named in the file; +6. target OS, architecture and Python implementation. + +It first rejects candidates that violate hard requirements. Remaining candidates +are ordered lexicographically by reproducibility loss, compatibility risk, +network acquisition, execution cost and a deterministic tie-breaker. This avoids +fragile arbitrary totals where a cheap but incompatible resource could win merely +by accumulating points. + +The current first implementation chooses per-package routes and deduplicates +artifacts by SHA-256. A later global optimizer can combine connections, builds and +downloads across packages without changing the file format. + +## Apply desired state + +```console +pload apply +pload apply project.pload.toml +pload apply project.pload.toml -n project-copy +pload apply project.pload.toml --offline +``` + +`apply` performs the internal operations that were previously exposed as export, +push, pull and restore: + +1. resolve or install the declared Python; +2. verify implementation, exact Python version, OS and architecture; +3. reuse exact local artifacts; +4. retrieve missing SHA-addressed artifacts from configured repositories; +5. fall back to an index only when policy permits it; +6. verify every artifact before installation; +7. create and register the environment; +8. install exact packages and run `pip check`; +9. record the applied configuration digest inside the environment. + +Running `apply` again is idempotent when the environment already carries the same +configuration digest. If the name exists with different state, pload refuses to +overwrite it. Destructive replacement is deliberately not implicit. If package +installation or verification fails, the newly created environment and registry +entry are rolled back; an existing environment is never modified. + +## Resource storage + +Local and SSH artifact repositories store bytes at: + +```text +objects/ +``` + +Two configurations referencing the same large CUDA wheel therefore store and +transfer one object. The TOML contains the checksum and logical repository name; +the user does not run upload/download commands. A missing or unreachable provider +is simply removed from the candidate set, and apply tries the next valid route. + +## Compatibility and boundaries + +- Exact locks are platform-specific unless the wheel is universal. +- A different OS, architecture or exact Python patch version currently requires + a separately described configuration. Multi-target locks are planned. +- Detected NVIDIA driver versions are recorded as capability provenance. Drivers + and system libraries are not installed by pload and are not backed up as if + they were Python artifacts. +- CUDA-enabled Python wheels are ordinary locked artifacts and can be cached in + the content repository. This preserves the large, slow-to-download package even + though the host driver remains the machine administrator's responsibility. +- Native Conda packages require a future Conda provider; they are not silently + converted to pip packages. +- Package installation may execute code. Only use configurations and providers + you trust; checksums prove identity, not trustworthiness. + +## Abandoned snapshot experiment + +The earlier `export → repo push/pull → restore` interface is not the public model +for 1.1. Its useful internals—probing, hashing, caches and content-addressed SSH +storage—remain providers behind `describe`, `plan` and `apply`. Before a stable +storage—remain providers behind `describe`, `plan` and `apply`. The experimental +format was never part of a stable release and is intentionally not carried forward. diff --git a/pyproject.toml b/pyproject.toml index 37322c8..2e43b5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,18 +1,19 @@ [build-system] -requires = ["setuptools>=68"] +requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] name = "pload" -version = "1.0.0" +version = "1.1.0a2" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" -license = { file = "LICENSE" } +license = "Apache-2.0" +license-files = ["LICENSE"] authors = [{ name = "Yunming Hu", email = "hugonelsonm3@gmail.com" }] keywords = ["python", "venv", "virtual-environment", "pyenv"] classifiers = [ - "Development Status :: 5 - Production/Stable", + "Development Status :: 3 - Alpha", "Environment :: Console", "Operating System :: MacOS", "Operating System :: Microsoft :: Windows", @@ -20,7 +21,10 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", ] -dependencies = ["rich>=13.7,<15"] +dependencies = [ + "rich>=13.7,<15", + "tomli>=2; python_version < '3.11'", +] [project.scripts] pload = "pload.cli:main" diff --git a/src/pload/__init__.py b/src/pload/__init__.py index 5becc17..5903a87 100644 --- a/src/pload/__init__.py +++ b/src/pload/__init__.py @@ -1 +1 @@ -__version__ = "1.0.0" +__version__ = "1.1.0a2" diff --git a/src/pload/cli.py b/src/pload/cli.py index 3f22cbd..89ea701 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -28,8 +28,9 @@ "shell-init": ("shell",), "python": ("py",), "config": ("cfg",), - "export": (), - "restore": (), + "describe": (), + "plan": (), + "apply": (), "repo": (), } PYTHON_ALIASES = { @@ -279,43 +280,52 @@ def build_parser(): ) config.add_argument("key", nargs="?", help="setting name for `config set`") config.add_argument("value", nargs="?", help="new value for `config set`") - export = subparsers.add_parser( - "export", description="Save exact package pins and optional platform-specific wheels.", - help="snapshot an existing environment", - ) - export.add_argument("name", help="new immutable snapshot name") - source = export.add_mutually_exclusive_group() - source.add_argument("--environment", "-e", help="environment ID, name or path (default: .venv)") - source.add_argument("--python", "-p", help="external interpreter, including uv/Conda/Poetry") - export.add_argument("--bundle", "-b", help="all or comma-separated packages to archive as wheels") - export.add_argument("--index-url", "-i", help="index used to download wheels, e.g. PyTorch CUDA") - export.add_argument("--find-links", "-f", action="append", help="existing wheel directory; repeatable") - restore = subparsers.add_parser( - "restore", description="Create a new environment from a snapshot or trusted requirements file.", - help="restore a snapshot or requirements.txt", - ) - restore.add_argument("source", help="snapshot name/directory or requirements.txt path") - restore.add_argument("--name", "-n", required=True, help="name of the new environment") - restore.add_argument("--version", "-v", help="Python ID, version or interpreter path") - restore.add_argument("--offline", "-o", action="store_true", help="install only from local wheels") - restore.add_argument("--portable", "-p", action="store_true", - help="resolve pins for another platform; ignore bundled wheels") - restore.add_argument("--index-url", "-i", help="package index for online restoration") - repo = subparsers.add_parser("repo", help="manage snapshot repositories", - description="Manage local, SSH and Git snapshot repositories.") + describe = subparsers.add_parser( + "describe", description="Describe an existing environment as one portable configuration.", + help="create pload.toml from an existing environment", + ) + describe.add_argument("source", help="environment ID, name, path, or Python executable") + describe.add_argument("--output", "-o", default="pload.toml", + help="configuration file to write (default: pload.toml)") + describe.add_argument("--name", "-n", help="logical environment name") + describe.add_argument("--mode", "-m", choices=["exact", "compatible"], default="exact", + help="lock exact artifacts or permit re-resolution (default: exact)") + describe.add_argument("--repository", "-r", + help="artifact repository; defaults to the first configured local/SSH repo") + describe.add_argument( + "--source", "-s", dest="package_sources", action="append", default=[], + metavar="PACKAGE=URL", + help="package-specific index; repeat for packages such as CUDA-enabled torch", + ) + plan = subparsers.add_parser( + "plan", description="Compare a declarative environment with all available resources.", + help="explain how pload would satisfy a configuration", + ) + plan.add_argument("file", nargs="?", default="pload.toml", + help="environment configuration (default: pload.toml)") + plan.add_argument("--offline", "-o", action="store_true", + help="plan using only resources available without the internet") + plan.add_argument("--json", "-j", action="store_true", help="emit machine-readable JSON") + apply = subparsers.add_parser( + "apply", description="Materialize the desired environment from available resources.", + help="create or verify an environment from pload.toml", + ) + apply.add_argument("file", nargs="?", default="pload.toml", + help="environment configuration (default: pload.toml)") + apply.add_argument("--name", "-n", help="override the materialized environment name") + apply.add_argument("--offline", "-o", action="store_true", + help="forbid internet access; configured local/SSH repositories remain usable") + repo = subparsers.add_parser("repo", help="manage resource providers", + description="Manage local and SSH artifact providers.") actions = repo.add_subparsers(dest="repo_command", required=True) add = actions.add_parser("add", description="Save a repository location; no credentials stored.") add.add_argument("name", help="repository nickname") - add.add_argument("location", help="directory, HOST:/absolute/path, or Git URL") - add.add_argument("--type", "-t", choices=["local", "ssh", "git"], default="local", + add.add_argument("location", help="directory or HOST:/absolute/path") + add.add_argument("--type", "-t", choices=["local", "ssh"], default="local", help="transport (default: local)") actions.add_parser("list", aliases=["ls"], description="Show configured repositories.") remove = actions.add_parser("remove", description="Remove configuration, keeping all remote files.") remove.add_argument("name", help="repository nickname") - for verb in ("push", "pull"): - action = actions.add_parser(verb, description=f"{verb.title()} a complete immutable snapshot.") - action.add_argument("repository", help="configured repository nickname") - action.add_argument("snapshot", help="snapshot name") return parser @@ -433,6 +443,31 @@ def _global_options_table(root_parser): return table +def print_declarative_plan(plan): + console = Console(highlight=False) + python = plan["python"] + console.print(Panel.fit( + f"[bold]{plan['name']}[/]\nPython: [cyan]{python['method']}[/] · {python['location']}", + title="[bold cyan]Environment plan[/]", + border_style="blue", + )) + table = Table(box=box.ROUNDED, header_style="bold cyan", border_style="blue") + table.add_column("PACKAGE", style="bold green", no_wrap=True) + table.add_column("VERSION", style="yellow", no_wrap=True) + table.add_column("METHOD", style="cyan", no_wrap=True) + table.add_column("STATUS", no_wrap=True) + table.add_column("RESOURCE") + for package in plan["packages"]: + selected = package["selected"] + if selected: + table.add_row(package["name"], package["version"], selected["method"], + selected["status"], selected["location"]) + else: + table.add_row(package["name"], package["version"], "—", "unavailable", + "No resource satisfies the configuration") + console.print(table) + + def _print_spaced_section(console, renderable): """Give a standalone section consistent visual breathing room.""" console.print() @@ -503,9 +538,10 @@ def _brief_help(parser, command_path, root_parser=None): "path": "Print an environment path", "config": "Show effective configuration", "shell-init": "Print shell activation integration", - "export": "Save a package snapshot and optional wheels", - "restore": "Recreate a snapshot or requirements file", - "repo": "Manage local, SSH and Git repositories", + "describe": "Write one portable configuration for an existing environment", + "plan": "Compare a configuration with available resources", + "apply": "Materialize the environment declared by pload.toml", + "repo": "Manage local and SSH artifact providers", } for command, summary in summaries.items(): aliases = COMMAND_ALIASES[command] @@ -584,6 +620,8 @@ def render_landing(): table.add_column("WHAT IT DOES") table.add_row("pload cfg", "Show where pload stores its data") table.add_row("pload python list", "Find every usable Python interpreter") + table.add_row("pload describe v1", "Describe an environment in pload.toml") + table.add_row("pload apply", "Create exactly what pload.toml declares") table.add_row("pload new", "Open guided environment creation") table.add_row("pload new -n data -v 3.12", "Create a named virtual environment") table.add_row("pload list", "List environments, IDs, and descriptions") @@ -597,7 +635,7 @@ def shell_script(shell): if shell in {"bash", "zsh"}: return r'''pload() { case "${1:-}" in - ""|new|init|i|rm|remove|del|delete|list|ls|path|p|shell-init|shell|python|py|config|cfg|export|restore|repo|-*) + ""|new|init|i|rm|remove|del|delete|list|ls|path|p|shell-init|shell|python|py|config|cfg|describe|plan|apply|repo|-*) command pload "$@" ;; *) @@ -610,7 +648,7 @@ def shell_script(shell): if shell == "fish": return r'''function pload switch "$argv[1]" - case '' new init i rm remove del delete list ls path p shell-init shell python py config cfg export restore repo '-*' + case '' new init i rm remove del delete list ls path p shell-init shell python py config cfg describe plan apply repo '-*' command pload $argv case '*' set -l name . @@ -624,7 +662,7 @@ def shell_script(shell): end''' return r'''function pload { param([Parameter(ValueFromRemainingArguments = $true)][string[]]$PloadArgs) - $commands = @('new', 'init', 'i', 'rm', 'remove', 'del', 'delete', 'list', 'ls', 'path', 'p', 'shell-init', 'shell', 'python', 'py', 'config', 'cfg', 'export', 'restore', 'repo', '-h', '--help', '--version') + $commands = @('new', 'init', 'i', 'rm', 'remove', 'del', 'delete', 'list', 'ls', 'path', 'p', 'shell-init', 'shell', 'python', 'py', 'config', 'cfg', 'describe', 'plan', 'apply', 'repo', '-h', '--help', '--version') $backend = Get-Command -Name @('pload.exe', 'pload.cmd') -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1 if (-not $backend) { throw 'pload executable not found on PATH' } if ($PloadArgs.Count -eq 0) { @@ -690,27 +728,34 @@ def run(argv=None): venvs = VenvManager(config) dependencies = DependencyManager(config) - if command in {"export", "restore", "repo"}: - from pload.snapshots import RepositoryManager, SnapshotManager + if command in {"describe", "plan", "apply", "repo"}: + if command == "repo": + from pload.snapshots import RepositoryManager - snapshots = SnapshotManager(config) - if command == "export": - print(snapshots.export(args.name, args.environment, args.python, args.bundle, - args.index_url, args.find_links)) - elif command == "restore": - print(snapshots.restore(args.source, args.name, args.version, - args.offline, args.portable, args.index_url)) - else: repositories = RepositoryManager(config) if args.repo_command == "add": repositories.add(args.name, args.location, args.type) elif args.repo_command == "remove": repositories.remove(args.name) - elif args.repo_command in {"list", "ls"}: + else: print(json.dumps(repositories.repositories(), indent=2)) + return 0 + from pload.declarative import DeclarativeEnvironmentManager + + manager = DeclarativeEnvironmentManager(config) + if command == "describe": + print(manager.describe( + args.source, args.output, args.name, args.mode, args.repository, + args.package_sources, + )) + elif command == "plan": + plan = manager.plan(args.file, args.offline) + if args.json: + print(json.dumps(plan, ensure_ascii=False, indent=2)) else: - repositories.transfer(args.repository, args.snapshot, args.repo_command == "push") - print(f"[*] {args.repo_command}: {args.snapshot} ({args.repository})") + print_declarative_plan(plan) + else: + print(manager.apply(args.file, args.name, args.offline)) return 0 if command == "new": diff --git a/src/pload/declarative.py b/src/pload/declarative.py new file mode 100644 index 0000000..d4e1100 --- /dev/null +++ b/src/pload/declarative.py @@ -0,0 +1,670 @@ +"""Declarative environment manifests and resource-driven materialization.""" + +import hashlib +import json +import os +import re +import shlex +import shutil +import tempfile +import uuid +from pathlib import Path +from urllib.parse import urlsplit + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - exercised by the Python 3.8 CI job + import tomli as tomllib + +from pload.errors import PloadError +from pload.managers.platform import PythonNotFoundError +from pload.managers.pyversion import PythonManager +from pload.managers.venv import VenvManager +from pload.snapshots import RepositoryManager, digest, execute, probe, public_index, safe_name + +PIN = re.compile(r"([A-Za-z0-9][A-Za-z0-9_.-]*)==([A-Za-z0-9][A-Za-z0-9_.+!-]*)") +WHEEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+!-]*\.whl") +SHA256 = re.compile(r"[0-9a-f]{64}") + + +def normalized_name(value): + return re.sub(r"[-_.]+", "-", value).lower() + + +def manifest_digest(data): + encoded = json.dumps(data, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _quoted(value): + return json.dumps(str(value), ensure_ascii=False) + + +def _array(values): + return "[" + ", ".join(_quoted(item) for item in values) + "]" + + +def dump_manifest(data): + """Serialize the deliberately small schema without a TOML writer dependency.""" + environment = data["environment"] + policy = data["policy"] + lines = [ + f"schema = {data['schema']}", + f"name = {_quoted(data['name'])}", + "", + "[environment]", + f"python = {_quoted(environment['python'])}", + f"implementation = {_quoted(environment.get('implementation', 'cpython'))}", + f"system = {_quoted(environment.get('system', ''))}", + f"machine = {_quoted(environment.get('machine', ''))}", + f"dependencies = {_array(environment.get('dependencies', []))}", + "", + "[policy]", + f"reproducibility = {_quoted(policy.get('reproducibility', 'exact'))}", + f"network = {_quoted(policy.get('network', 'allow'))}", + f"source_build = {_quoted(policy.get('source_build', 'fallback'))}", + "publish_missing_artifacts = " + + ("true" if policy.get("publish_missing_artifacts", True) else "false"), + ] + repositories = policy.get("repositories", []) + if repositories: + lines.append(f"repositories = {_array(repositories)}") + capabilities = data.get("capabilities", {}) + if capabilities: + lines += ["", "[capabilities]"] + for key, value in sorted(capabilities.items()): + lines.append(f"{key} = {_array(value)}") + for name, source in sorted(data.get("sources", {}).items()): + lines += [ + "", + f"[sources.{_quoted(name)}]", + f"kind = {_quoted(source['kind'])}", + f"url = {_quoted(source['url'])}", + ] + for name, repository in sorted(data.get("repositories", {}).items()): + lines += [ + "", + f"[repositories.{_quoted(name)}]", + f"kind = {_quoted(repository['kind'])}", + f"location = {_quoted(repository['location'])}", + ] + for package in data.get("package", []): + lines += [ + "", + "[[package]]", + f"name = {_quoted(package['name'])}", + f"version = {_quoted(package['version'])}", + ] + if package.get("sources"): + lines.append(f"sources = {_array(package['sources'])}") + for artifact in package.get("artifact", []): + lines += [ + "", + "[[package.artifact]]", + f"filename = {_quoted(artifact['filename'])}", + f"sha256 = {_quoted(artifact['sha256'])}", + f"tags = {_array(artifact.get('tags', []))}", + f"repositories = {_array(artifact.get('repositories', []))}", + ] + return "\n".join(lines) + "\n" + + +def load_manifest(path): + path = Path(path).expanduser().resolve() + try: + with path.open("rb") as stream: + data = tomllib.load(stream) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise PloadError(f"cannot read environment configuration {path}: {exc}") from exc + data.setdefault("sources", {}) + data.setdefault("repositories", {}) + data.setdefault("capabilities", {}) + data.setdefault("package", []) + for package in data["package"]: + package.setdefault("sources", []) + package.setdefault("artifact", []) + validate_manifest(data) + return path, data + + +def validate_manifest(data): + if not isinstance(data, dict) or data.get("schema") != 1: + raise PloadError("unsupported environment configuration schema") + safe_name(data.get("name")) + environment = data.get("environment") + policy = data.get("policy") + if not isinstance(environment, dict) or not environment.get("python"): + raise PloadError("environment.python is required") + if not isinstance(policy, dict): + raise PloadError("a [policy] section is required") + if policy.get("reproducibility", "exact") not in {"exact", "compatible"}: + raise PloadError("policy.reproducibility must be exact or compatible") + if policy.get("network", "allow") not in {"allow", "offline"}: + raise PloadError("policy.network must be allow or offline") + if policy.get("source_build", "fallback") not in {"fallback", "forbid"}: + raise PloadError("policy.source_build must be fallback or forbid") + dependencies = environment.get("dependencies", []) + if not isinstance(dependencies, list) or any(not PIN.fullmatch(item) for item in dependencies): + raise PloadError("environment.dependencies must contain exact NAME==VERSION pins") + seen = set() + for name, source in data.get("sources", {}).items(): + safe_name(name) + if source.get("kind") != "index" or not source.get("url"): + raise PloadError(f"source {name} must be a Python package index") + parsed = urlsplit(source["url"]) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise PloadError(f"source {name} must use an HTTP(S) URL") + if public_index(source["url"]) != source["url"]: + raise PloadError(f"source {name} must not contain credentials or query parameters") + capabilities = data.get("capabilities", {}) + if not isinstance(capabilities, dict) or any( + not isinstance(values, list) or any(not isinstance(value, str) for value in values) + for values in capabilities.values() + ): + raise PloadError("capabilities must contain arrays of strings") + for package in data.get("package", []): + if not isinstance(package, dict): + raise PloadError("invalid package lock entry") + pin = f"{package.get('name', '')}=={package.get('version', '')}" + if not PIN.fullmatch(pin): + raise PloadError(f"invalid locked package: {pin}") + key = normalized_name(package["name"]) + if key in seen: + raise PloadError(f"duplicate locked package: {package['name']}") + seen.add(key) + for source_name in package.get("sources", []): + if source_name not in data.get("sources", {}): + raise PloadError( + f"locked package {package['name']} references unknown source {source_name}" + ) + for artifact in package.get("artifact", []): + if (not WHEEL.fullmatch(artifact.get("filename", "")) + or not SHA256.fullmatch(artifact.get("sha256", ""))): + raise PloadError(f"invalid artifact for {package['name']}") + for repository_name in artifact.get("repositories", []): + if repository_name not in data.get("repositories", {}): + raise PloadError( + f"artifact {artifact['filename']} references unknown repository " + f"{repository_name}" + ) + for name, repository in data.get("repositories", {}).items(): + safe_name(name) + kind = repository.get("kind") + location = repository.get("location") + if kind == "ssh": + RepositoryManager.ssh_location(location) + elif kind == "local": + if not location: + raise PloadError(f"repository {name} has no location") + else: + raise PloadError(f"repository {name} must be local or ssh") + return data + + +def _wheel_tags(filename): + parts = filename[:-4].rsplit("-", 3) + return ["-".join(parts[-3:])] if len(parts) == 4 else [] + + +def _wheel_identity(filename): + parts = Path(filename).name[:-4].split("-") if str(filename).endswith(".whl") else [] + if len(parts) < 5: + return None, None + return normalized_name(parts[0]), parts[1] + + +class ArtifactRepository: + """Content-addressed local and SSH artifact access.""" + + @staticmethod + def _local_root(spec, base): + root = Path(spec["location"]).expanduser() + if not root.is_absolute(): + root = base / root + return root.resolve() + + @classmethod + def contains(cls, spec, checksum, base): + if spec["kind"] == "local": + artifact = cls._local_root(spec, base) / "objects" / checksum + return artifact.is_file() and digest(artifact) == checksum + host, root = RepositoryManager.ssh_location(spec["location"]) + result = execute([ + "ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", host, + f"if test -f {shlex.quote(root + '/objects/' + checksum)}; then printf yes; fi", + ]) + return result == "yes" + + @classmethod + def publish(cls, spec, source, checksum, base): + source = Path(source) + if digest(source) != checksum: + raise PloadError("artifact changed before publication") + if spec["kind"] == "local": + target = cls._local_root(spec, base) / "objects" / checksum + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and digest(target) != checksum: + raise PloadError("repository object checksum mismatch") + if not target.exists(): + temporary = target.with_name(target.name + "." + uuid.uuid4().hex) + shutil.copyfile(source, temporary) + temporary.replace(target) + return + host, root = RepositoryManager.ssh_location(spec["location"]) + obj = root + "/objects/" + checksum + execute(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", host, + f"mkdir -p {shlex.quote(root + '/objects')}"]) + if not cls.contains(spec, checksum, base): + pending = obj + "." + uuid.uuid4().hex + execute(["scp", "-q", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", + str(source), host + ":" + pending]) + execute(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", host, + f"mv -n {shlex.quote(pending)} {shlex.quote(obj)}"]) + remote_hash = execute(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", host, + f"sha256sum {shlex.quote(obj)}"]).split()[0] + if remote_hash != checksum: + raise PloadError("remote artifact checksum mismatch") + + @classmethod + def fetch(cls, spec, checksum, destination, base): + destination = Path(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(destination.name + "." + uuid.uuid4().hex) + if spec["kind"] == "local": + shutil.copyfile(cls._local_root(spec, base) / "objects" / checksum, temporary) + else: + host, root = RepositoryManager.ssh_location(spec["location"]) + execute(["scp", "-q", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", + host + ":" + root + "/objects/" + checksum, str(temporary)]) + if digest(temporary) != checksum: + temporary.unlink(missing_ok=True) + raise PloadError("retrieved artifact checksum mismatch") + temporary.replace(destination) + + +class DeclarativeEnvironmentManager: + def __init__(self, config): + self.config = config + self.cache = config.home / "cache" / "wheels" + + def describe(self, source, output, name=None, mode="exact", repository=None, sources=None): + venvs = VenvManager(self.config) + candidate = Path(source).expanduser() + if candidate.is_file(): + interpreter = candidate.resolve() + inferred_name = interpreter.parent.parent.name or "environment" + else: + environment = venvs.resolve_existing(source) + interpreter = environment / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + inferred_name = environment.name + runtime = probe(interpreter) + if runtime.get("conda"): + raise PloadError("native Conda environments require a future Conda provider") + packages = sorted(runtime.pop("packages"), key=lambda item: item["name"].lower()) + accelerators = runtime.pop("accelerators", []) + pins = [f"{item['name']}=={item['version']}" for item in packages] + if any(not PIN.fullmatch(pin) for pin in pins): + raise PloadError("the environment contains package metadata that cannot be locked") + output = Path(output).expanduser().resolve() + repositories = { + key: {"kind": value["kind"], "location": value["location"]} + for key, value in self.config.settings.get("repositories", {}).items() + if value.get("kind") in {"local", "ssh"} + } + selected_repository = repository + if selected_repository and selected_repository not in repositories: + raise PloadError(f"unknown artifact repository: {selected_repository}") + if not selected_repository and repositories: + selected_repository = min(repositories) + index = public_index(self.config.settings.get("pip_index") or "https://pypi.org/simple") + parsed_index = urlsplit(index) + if parsed_index.scheme not in {"http", "https"} or not parsed_index.netloc: + raise PloadError("configured package index must use an HTTP(S) URL") + source_overrides = {} + declared_sources = {"default": {"kind": "index", "url": index}} + installed_names = {normalized_name(item["name"]) for item in packages} + for assignment in sources or []: + package_name, separator, url = assignment.partition("=") + package_name = normalized_name(package_name.strip()) + url = public_index(url.strip()) + if not separator or not package_name or not url: + raise PloadError("package sources must use PACKAGE=INDEX_URL") + parsed_source = urlsplit(url) + if parsed_source.scheme not in {"http", "https"} or not parsed_source.netloc: + raise PloadError("package source must use an HTTP(S) URL") + if package_name not in installed_names: + raise PloadError(f"package source does not match an installed package: {package_name}") + source_name = "package-" + package_name + source_overrides[package_name] = (source_name, url) + declared_sources[source_name] = {"kind": "index", "url": url} + data = { + "schema": 1, + "name": name or inferred_name, + "environment": { + "python": runtime.get("python"), + "implementation": runtime.get("implementation", "cpython"), + "system": runtime.get("system", ""), + "machine": runtime.get("machine", ""), + "dependencies": pins, + }, + "policy": { + "reproducibility": mode, + "network": "allow", + "source_build": "fallback", + "publish_missing_artifacts": True, + "repositories": [selected_repository] if selected_repository else [], + }, + "sources": declared_sources, + "repositories": repositories, + "capabilities": { + "nvidia_driver": sorted({ + version for item in accelerators if item.get("kind") == "nvidia" + for version in item.get("driver_versions", []) + }), + }, + "package": [], + } + self.cache.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="pload-describe-") as temporary: + wheel_dir = Path(temporary) + for installed in packages: + source_name, package_index = source_overrides.get( + normalized_name(installed["name"]), ("default", index) + ) + record = { + "name": installed["name"], "version": installed["version"], + "sources": [source_name], "artifact": [], + } + if mode == "exact": + wheel = self._obtain_wheel( + interpreter, installed, wheel_dir, package_index + ) + checksum = digest(wheel) + cached = self.cache / wheel.name + if cached.exists() and digest(cached) != checksum: + raise PloadError(f"conflicting cached artifact: {wheel.name}") + if not cached.exists(): + shutil.copyfile(wheel, cached) + locations = [] + if selected_repository: + ArtifactRepository.publish( + repositories[selected_repository], cached, checksum, output.parent, + ) + locations.append(selected_repository) + record["artifact"].append({ + "filename": wheel.name, "sha256": checksum, + "tags": _wheel_tags(wheel.name), "repositories": locations, + }) + data["package"].append(record) + validate_manifest(data) + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_name(output.name + ".tmp") + temporary.write_text(dump_manifest(data), encoding="utf-8") + temporary.replace(output) + return output + + def _obtain_wheel(self, interpreter, installed, wheel_dir, index): + expected_name = normalized_name(installed["name"]) + expected_version = installed["version"] + direct_checksum = None + if installed.get("direct_url"): + try: + provenance = json.loads(installed["direct_url"]) + archive = provenance.get("archive_info", {}) + direct_checksum = archive.get("hashes", {}).get("sha256") + if not direct_checksum and archive.get("hash", "").startswith("sha256="): + direct_checksum = archive["hash"].split("=", 1)[1] + except (TypeError, ValueError): + pass + for wheel in self.cache.glob("*.whl"): + if (_wheel_identity(wheel.name) == (expected_name, expected_version) + and (not direct_checksum or digest(wheel) == direct_checksum)): + return wheel + if installed.get("direct_url"): + raise PloadError( + f"cannot lock direct installation {installed['name']}=={installed['version']}; " + "place its original SHA-matching wheel in the pload cache" + ) + command = [ + str(interpreter), "-m", "pip", "download", "--only-binary=:all:", + "--no-deps", "--dest", str(wheel_dir), "--index-url", index, + f"{installed['name']}=={installed['version']}", + ] + try: + execute(command) + except PloadError as exc: + raise PloadError( + f"cannot lock an exact wheel for {installed['name']}=={installed['version']}; " + "use --mode compatible only if re-resolution is acceptable" + ) from exc + matches = [ + wheel for wheel in wheel_dir.glob("*.whl") + if _wheel_identity(wheel.name) == (expected_name, expected_version) + ] + if len(matches) != 1: + raise PloadError(f"download did not produce one exact wheel for {installed['name']}") + return matches[0] + + def plan(self, manifest_path, offline=False): + path, data = load_manifest(manifest_path) + policy = data["policy"] + network_allowed = policy.get("network", "allow") == "allow" and not offline + repositories = data.get("repositories", {}) + packages = self._locked_packages(data) + plans = [] + for package in packages: + candidates = [] + for artifact in package.get("artifact", []): + cached = self.cache / artifact["filename"] + adjacent = path.parent / "artifacts" / artifact["filename"] + if cached.is_file() and digest(cached) == artifact["sha256"]: + candidates.append(self._candidate("cache", cached, "ready", (0, 0, 0, 0))) + if adjacent.is_file() and digest(adjacent) == artifact["sha256"]: + candidates.append(self._candidate( + "configuration-artifact", adjacent, "ready", (0, 0, 0, 1) + )) + for repo_name in artifact.get("repositories", []): + spec = repositories.get(repo_name) + if spec: + try: + available = ArtifactRepository.contains( + spec, artifact["sha256"], path.parent + ) + except PloadError: + available = False + if available: + candidates.append(self._candidate( + "repository", repo_name, "remote", (0, 0, 1, repo_name), artifact + )) + if network_allowed: + for source_name in package.get("sources", []): + source = data.get("sources", {}).get(source_name) + if source and source.get("kind") == "index": + candidates.append(self._candidate( + "index-exact", source_name, "network", + (0, 0, 2, source_name), artifact, + )) + if (not package.get("artifact") + and policy.get("reproducibility") == "compatible" and network_allowed): + source_name = (package.get("sources") or ["default"])[0] + candidates.append(self._candidate( + "index-resolve", source_name, "network", (1, 1, 2, source_name) + )) + candidates.sort(key=lambda item: item["cost"]) + plans.append({ + "name": package["name"], "version": package["version"], + "selected": candidates[0] if candidates else None, + "alternatives": candidates[1:], + }) + try: + python = str(self.config.get_python_path(data["environment"]["python"])) + python_action = {"method": "reuse-python", "location": python, "status": "ready"} + except PythonNotFoundError: + python_action = { + "method": "install-python", "location": data["environment"]["python"], + "status": "network" if network_allowed else "unavailable", + } + ready = (python_action["status"] != "unavailable" + and all(item["selected"] for item in plans)) + return {"path": str(path), "name": data["name"], "python": python_action, + "packages": plans, "ready": ready} + + @staticmethod + def _candidate(method, location, status, cost, artifact=None): + result = {"method": method, "location": str(location), "status": status, + "cost": list(cost)} + if artifact: + result["artifact"] = artifact + return result + + @staticmethod + def _locked_packages(data): + if data.get("package"): + return data["package"] + result = [] + for pin in data["environment"].get("dependencies", []): + match = PIN.fullmatch(pin) + result.append({"name": match.group(1), "version": match.group(2), + "sources": ["default"], "artifact": []}) + return result + + def apply(self, manifest_path, name=None, offline=False): + path, data = load_manifest(manifest_path) + plan = self.plan(path, offline=offline) + unavailable = [item for item in plan["packages"] if not item["selected"]] + if unavailable: + names = ", ".join(f"{item['name']}=={item['version']}" for item in unavailable) + raise PloadError(f"no valid reproduction route for: {names}") + if plan["python"]["method"] == "install-python": + if plan["python"]["status"] == "unavailable": + raise PloadError("required Python is unavailable while offline") + PythonManager(self.config).install_python(data["environment"]["python"]) + interpreter = self.config.get_python_path(data["environment"]["python"]) + self._check_runtime(interpreter, data["environment"]) + target_name = name or data["name"] + expected_state = manifest_digest(data) + try: + existing = VenvManager(self.config).resolve_existing(target_name) + except PloadError: + existing = None + if existing: + marker = existing / ".pload-manifest.sha256" + if marker.is_file() and marker.read_text(encoding="utf-8").strip() == expected_state: + return existing + raise PloadError(f"environment {target_name!r} exists with different state") + artifacts = [] + compatible = [] + self.cache.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="pload-apply-") as temporary: + stage = Path(temporary) + for package_plan in plan["packages"]: + selected = package_plan["selected"] + artifact = selected.get("artifact") + if artifact: + routes = [selected] + package_plan["alternatives"] + failures = [] + acquired = None + for route in routes: + route_artifact = route.get("artifact") + if not route_artifact: + continue + cached = self.cache / route_artifact["filename"] + try: + if not (cached.is_file() + and digest(cached) == route_artifact["sha256"]): + self._acquire(path, data, package_plan, route, cached, stage) + if digest(cached) != route_artifact["sha256"]: + raise PloadError("checksum mismatch") + except (OSError, PloadError) as exc: + failures.append(f"{route['method']}: {exc}") + continue + acquired = (cached, route_artifact) + break + if not acquired: + detail = failures[-1] if failures else "no artifact route" + raise PloadError( + f"cannot acquire {package_plan['name']}=={package_plan['version']}: {detail}" + ) + cached, artifact = acquired + artifacts.append(cached.resolve().as_uri() + "#sha256=" + artifact["sha256"]) + else: + compatible.append(package_plan) + venvs = VenvManager(self.config) + env = venvs.create_venv( + version=str(interpreter), name=target_name, + description=f"Applied from {path.name}", + ) + try: + pip = self.config.get_pip_command(env) + if artifacts: + execute(pip + ["install", "--no-index", "--no-deps"] + artifacts, + capture=False) + for package_plan in compatible: + source_name = package_plan["selected"]["location"] + source = data.get("sources", {}).get(source_name, {}) + command = pip + ["install", "--no-deps"] + if data["policy"].get("source_build") == "forbid": + command += ["--only-binary=:all:"] + if source.get("url"): + command += ["--index-url", source["url"]] + execute(command + [ + f"{package_plan['name']}=={package_plan['version']}" + ], capture=False) + requirements = stage / "requirements.txt" + requirements.write_text( + "".join( + f"{item['name']}=={item['version']}\n" for item in plan["packages"] + ), + encoding="utf-8", + ) + execute(pip + ["install", "--no-index", "--no-deps", "-r", + str(requirements)]) + execute(pip + ["check"], capture=False) + (env / ".pload-manifest.sha256").write_text( + expected_state + "\n", encoding="utf-8" + ) + return env + except (OSError, PloadError): + venvs.remove_venv(target_name) + raise + + def _acquire(self, path, data, package, selected, destination, stage): + artifact = selected["artifact"] + if selected["method"] == "configuration-artifact": + shutil.copyfile(selected["location"], destination) + elif selected["method"] == "repository": + ArtifactRepository.fetch( + data["repositories"][selected["location"]], artifact["sha256"], + destination, path.parent, + ) + elif selected["method"] == "index-exact": + source = data["sources"][selected["location"]] + download = stage / normalized_name(package["name"]) + download.mkdir(exist_ok=True) + execute([ + str(self.config.get_python_path(data["environment"]["python"])), + "-m", "pip", "download", "--only-binary=:all:", "--no-deps", + "--dest", str(download), "--index-url", source["url"], + f"{package['name']}=={package['version']}", + ]) + candidate = download / artifact["filename"] + if not candidate.is_file(): + raise PloadError(f"index did not provide locked artifact {artifact['filename']}") + shutil.copyfile(candidate, destination) + else: + raise PloadError(f"unsupported acquisition method: {selected['method']}") + + @staticmethod + def _check_runtime(interpreter, expected): + actual = probe(interpreter) + if actual.get("implementation") != expected.get("implementation"): + raise PloadError("Python implementation does not satisfy the configuration") + if actual.get("python") != expected.get("python"): + raise PloadError( + f"Python version mismatch: need {expected.get('python')}, got {actual.get('python')}" + ) + for key in ("system", "machine"): + if expected.get(key) and actual.get(key) != expected.get(key): + raise PloadError( + f"target {key} mismatch: need {expected.get(key)}, got {actual.get(key)}" + ) diff --git a/src/pload/help_content.py b/src/pload/help_content.py index 8decd66..ffaa1f4 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -50,6 +50,19 @@ - project-local `pload init` environments may live outside the managed root. Use `pload cfg` to see the exact effective paths before creating anything. + +## Reproduce an environment from one file + +```console +$ pload describe v1 -o pload.toml +$ pload plan pload.toml +$ pload apply pload.toml +``` + +The TOML file describes the desired Python, exact packages, platform, policies, +artifact hashes, indexes and logical artifact providers. `apply` inventories the +current machine and automatically reuses, copies or downloads the best valid +resources; users do not perform separate upload, download or migration steps. """, ("new",): r""" # Create a managed environment @@ -331,72 +344,74 @@ } -GUIDES[("export",)] = r""" -# Save a reproducible package snapshot +GUIDES[("describe",)] = r""" +# Turn an existing environment into one portable configuration + +```console +pload describe v2 +pload describe /project/.venv -o project.pload.toml +pload describe /project/.venv/bin/python -n project -r lab +pload describe v2 -s torch=https://download.pytorch.org/whl/cu121 +``` + +The default exact mode locks every installed application package to an actual +wheel and SHA-256. pload reuses its local cache, downloads a missing wheel when +necessary, and publishes it to the selected configured local/SSH repository. +These are internal resource operations: the user receives one TOML file. + +Use `--mode compatible` only when exact wheels do not exist and future +re-resolution is acceptable. Native Conda packages are not silently converted. +Credentials and URL query tokens are never written into the portable file. +Use repeatable `--source PACKAGE=URL` options when an installed package came from +a dedicated index that cannot be inferred reliably from installed metadata. +""" +GUIDES[("plan",)] = r""" +# Explain how the desired state can be satisfied ```console -pload export analysis-01 -e v1 -pload export analysis-offline -p /path/to/project/.venv/bin/python -b all -pload export training-01 -e v2 -b torch -f /mnt/downloaded-wheels +pload plan +pload plan project.pload.toml +pload plan --offline +pload plan --json ``` -The first command records exact installed versions and interpreter/platform metadata -under `PLOAD_HOME/snapshots/analysis-01`. Its requirements.txt also works with pip -and uv. The second adds every package's wheel for offline restoration. The third -archives only torch, so other packages still come from your package index. -Use `-i URL` to select a dedicated wheel index, such as your PyTorch CUDA index. -Existing shared wheels are tried offline first; then pip uses its normal caches. - -Names are immutable: export a new name for a new revision. Source checkouts and -editable installs must first be built and installed as wheels. Direct-wheel installs -need their original archive in `-f DIR` or the shared cache and inclusion in `-b`; -the archive hash must match the installation metadata. Native Conda environments require Conda's own -export tools. Python itself, OS libraries and GPU drivers are not bundled. +Planning reads the configuration, discovers Python interpreters and checks local +caches, adjacent artifacts, content-addressed repositories and declared indexes. +Incompatible resources are rejected first; valid routes are ordered by exactness, +compatibility risk, transfer, execution cost and a deterministic tie-breaker. +Planning is optional transparency: normal users can go directly to `pload apply`. """ -GUIDES[("restore",)] = r""" -# Restore into a new environment +GUIDES[("apply",)] = r""" +# Materialize the configuration ```console -pload restore analysis-offline -n analysis-copy -o -pload restore analysis-01 -n linux-copy -p -pload restore ./requirements.txt -n imported -v 3.12 +pload apply +pload apply project.pload.toml -n project-copy +pload apply --offline ``` -`-o` uses a complete wheel snapshot without a package index. SHA-256 checks run -before creating the environment; wheel compatibility is checked by pip. The Python -major/minor version must match and must already be installed. -`-p` deliberately re-resolves the pinned versions for another platform. It is not -a promise of an identical environment and cannot be combined with offline mode. -Only import requirements files and repositories you trust: installing packages -can execute code. Exported index provenance is informational; choose the restore -index explicitly with `-i URL` when your configured index cannot supply a package. - -The target name must be unused. A failed install leaves that new environment for -inspection or removal with `pload rm`. Existing environments are never updated. +Apply resolves or installs Python, obtains exact artifacts through the selected +resource routes, verifies hashes, creates the environment, installs packages and +runs `pip check`. Upload, download and copying are internal plan steps rather than +separate user commands. Reapplying the same configuration is idempotent; an +existing environment with different state is never silently overwritten. """ GUIDES[("repo",)] = r""" -# Store recipes and reusable wheels on your own infrastructure +# Configure reusable artifact providers ```console pload repo add lab frpxiaoxin:/home/tibless/pload-cloud -t ssh -pload repo push lab analysis-offline -pload repo pull lab analysis-offline -pload restore analysis-offline -n analysis-copy -o -pload repo add recipes git@github.com:YOUR-NAME/pload-recipes.git -t git -pload repo push recipes analysis-01 +pload repo add disk /mnt/shared/pload -t local pload repo list pload repo remove lab ``` -Pull on another machine (or into a different PLOAD_HOME): existing snapshots are -not overwritten. SSH uses your OpenSSH alias/keys and a Linux directory; no web -server or extra Python package is required. Identical wheels are stored once by -hash on the server and reused from the local cache on pull. -Git repositories store recipes only; use SSH or a local directory for wheel bundles. -Git authentication and commit identity come from your existing Git configuration. -Removing a repository only forgets its configuration; remote data remains intact. +`describe` embeds configured local/SSH providers in `pload.toml`; `apply` uses them +automatically. SSH relies on OpenSSH aliases, keys and agents and stores no password. +Objects are addressed by SHA-256 so identical large wheels are stored only once. +Removing a provider forgets its configuration but does not delete remote data. """ -for _command in ("add", "list", "remove", "push", "pull"): +for _command in ("add", "list", "remove"): GUIDES[("repo", _command)] = GUIDES[("repo",)] diff --git a/src/pload/snapshots.py b/src/pload/snapshots.py index 513231a..55ecbb6 100644 --- a/src/pload/snapshots.py +++ b/src/pload/snapshots.py @@ -21,7 +21,7 @@ from pload.settings import save_settings PROBE = """ -import json, platform, sys, os +import json, platform, sys, os, shutil, subprocess from importlib import metadata packages = [] for dist in metadata.distributions(): @@ -29,10 +29,24 @@ if name and name.lower() not in ('pip', 'setuptools', 'wheel'): packages.append({'name': name, 'version': dist.version, 'direct_url': dist.read_text('direct_url.json')}) +accelerators = [] +nvidia_smi = shutil.which('nvidia-smi') +if nvidia_smi: + try: + result = subprocess.run([nvidia_smi, '--query-gpu=driver_version', + '--format=csv,noheader'], capture_output=True, text=True, + timeout=5) + if result.returncode == 0: + versions = sorted(set(line.strip() for line in result.stdout.splitlines() + if line.strip())) + accelerators.append({'kind': 'nvidia', 'driver_versions': versions}) + except (OSError, subprocess.SubprocessError): + pass print(json.dumps({'python': platform.python_version(), 'implementation': sys.implementation.name, 'system': platform.system(), 'machine': platform.machine(), 'libc': list(platform.libc_ver()), - 'conda': os.path.isdir(os.path.join(sys.prefix, 'conda-meta')), 'packages': packages})) + 'conda': os.path.isdir(os.path.join(sys.prefix, 'conda-meta')), + 'accelerators': accelerators, 'packages': packages})) """ diff --git a/tests/test_declarative.py b/tests/test_declarative.py new file mode 100644 index 0000000..3d25bd1 --- /dev/null +++ b/tests/test_declarative.py @@ -0,0 +1,188 @@ +import platform +import shutil +import sys +import zipfile +from pathlib import Path + +import pytest + +from pload.cli import build_parser, main, shell_script +from pload.declarative import ( + DeclarativeEnvironmentManager, + dump_manifest, + load_manifest, + validate_manifest, +) +from pload.errors import PloadError +from pload.managers.platform import ConfigManager +from pload.managers.venv import VenvManager +from pload.settings import save_settings +from pload.snapshots import digest, execute + + +def tiny_wheel(directory): + directory.mkdir(parents=True, exist_ok=True) + path = directory / "pload_demo-1.0-py3-none-any.whl" + with zipfile.ZipFile(path, "w") as wheel: + wheel.writestr("pload_demo.py", "answer = 42\n") + wheel.writestr( + "pload_demo-1.0.dist-info/METADATA", + "Metadata-Version: 2.1\nName: pload-demo\nVersion: 1.0\n", + ) + wheel.writestr( + "pload_demo-1.0.dist-info/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + wheel.writestr("pload_demo-1.0.dist-info/RECORD", "") + return path + + +def simple_manifest(mode="compatible"): + return { + "schema": 1, + "name": "demo", + "environment": { + "python": "3.12.1", "implementation": "cpython", + "system": "Linux", "machine": "x86_64", + "dependencies": ["demo==1.0"], + }, + "policy": { + "reproducibility": mode, "network": "allow", + "source_build": "fallback", "publish_missing_artifacts": True, + }, + "sources": {"default": {"kind": "index", "url": "https://pypi.org/simple"}}, + "repositories": {}, + "capabilities": {}, + "package": [{ + "name": "demo", "version": "1.0", "sources": ["default"], "artifact": [], + }], + } + + +def test_manifest_roundtrip(tmp_path): + data = simple_manifest() + path = tmp_path / "pload.toml" + path.write_text(dump_manifest(data), encoding="utf-8") + loaded_path, loaded = load_manifest(path) + assert loaded_path == path + assert loaded == data + + +def test_manifest_requires_exact_pins(): + data = simple_manifest() + data["environment"]["dependencies"] = ["demo>=1"] + with pytest.raises(PloadError, match="exact"): + validate_manifest(data) + + +def test_manifest_rejects_secrets_in_shareable_sources(): + data = simple_manifest() + data["sources"]["default"]["url"] = "https://user:secret@example.com/simple?token=x" + with pytest.raises(PloadError, match="credentials"): + validate_manifest(data) + + +def test_manifest_rejects_dangling_provider_references(): + data = simple_manifest() + data["package"][0]["sources"] = ["missing"] + with pytest.raises(PloadError, match="unknown source"): + validate_manifest(data) + + +def test_compatible_plan_uses_index_and_offline_reports_missing(tmp_path): + path = tmp_path / "pload.toml" + path.write_text(dump_manifest(simple_manifest()), encoding="utf-8") + manager = DeclarativeEnvironmentManager(ConfigManager(home=tmp_path / "home")) + assert manager.plan(path)["packages"][0]["selected"]["method"] == "index-resolve" + assert manager.plan(path, offline=True)["packages"][0]["selected"] is None + + +def test_exact_configuration_requires_an_artifact_route(tmp_path): + data = simple_manifest("exact") + path = tmp_path / "pload.toml" + path.write_text(dump_manifest(data), encoding="utf-8") + manager = DeclarativeEnvironmentManager(ConfigManager(home=tmp_path / "home")) + assert manager.plan(path)["ready"] is False + + +def test_describe_plan_apply_through_content_repository(tmp_path): + source_config = ConfigManager(home=tmp_path / "source-home") + repository = tmp_path / "artifact-store" + save_settings(source_config.home, { + "repositories": {"lab": {"kind": "local", "location": str(repository)}}, + "pip_index": "https://user:secret@example.invalid/simple?token=x", + }) + source_config = ConfigManager(home=tmp_path / "source-home") + wheel = tiny_wheel(tmp_path / "fixture") + source = VenvManager(source_config).create_venv(name="source") + execute(source_config.get_pip_command(source) + [ + "install", "--no-index", "--find-links", str(wheel.parent), "pload-demo==1.0", + ]) + manager = DeclarativeEnvironmentManager(source_config) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + manifest = manager.describe( + "source", tmp_path / "pload.toml", name="restored", + sources=["pload-demo=https://download.example.invalid/cu121"], + ) + _, data = load_manifest(manifest) + artifact = data["package"][0]["artifact"][0] + assert data["sources"]["default"]["url"] == "https://example.invalid/simple" + assert data["package"][0]["sources"] == ["package-pload-demo"] + assert data["sources"]["package-pload-demo"]["url"].endswith("/cu121") + assert artifact["repositories"] == ["lab"] + assert (repository / "objects" / artifact["sha256"]).is_file() + + shutil.rmtree(source_config.home / "cache") + target_config = ConfigManager(home=tmp_path / "target-home") + target_config.get_python_path = lambda version=None: Path(sys.executable) + target = DeclarativeEnvironmentManager(target_config) + plan = target.plan(manifest) + assert plan["packages"][0]["selected"]["method"] == "repository" + restored = target.apply(manifest) + output = execute([target_config.get_pip_command(restored)[0], "-c", + "import pload_demo; print(pload_demo.answer)"]) + assert output == "42" + assert target.apply(manifest) == restored + assert digest(target.cache / artifact["filename"]) == artifact["sha256"] + + +def test_apply_rolls_back_a_new_environment_after_install_failure(tmp_path): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + artifact_dir = tmp_path / "artifacts" + artifact_dir.mkdir() + broken = artifact_dir / "demo-1.0-py3-none-any.whl" + broken.write_bytes(b"not a wheel") + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + }) + data["package"][0]["artifact"] = [{ + "filename": broken.name, "sha256": digest(broken), + "tags": ["py3-none-any"], "repositories": [], + }] + manifest = tmp_path / "pload.toml" + manifest.write_text(dump_manifest(data), encoding="utf-8") + with pytest.raises(PloadError): + DeclarativeEnvironmentManager(config).apply(manifest) + with pytest.raises(PloadError): + VenvManager(config).resolve_existing("demo") + + +def test_declarative_commands_and_shell_integration(capsys): + parser = build_parser() + assert parser.parse_args(["describe", "v1", "-o", "env.toml"]).output == "env.toml" + parsed = parser.parse_args([ + "describe", "v1", "-s", "torch=https://download.pytorch.org/whl/cu121", + ]) + assert parsed.package_sources == ["torch=https://download.pytorch.org/whl/cu121"] + assert parser.parse_args(["apply"]).file == "pload.toml" + for command in ("describe", "plan", "apply"): + assert main([command, "-h"]) == 0 + for shell in ("bash", "zsh", "fish", "powershell"): + assert command in shell_script(shell) + assert "--mode" in capsys.readouterr().out diff --git a/tests/test_snapshots.py b/tests/test_snapshots.py index fbe106d..30f83aa 100644 --- a/tests/test_snapshots.py +++ b/tests/test_snapshots.py @@ -5,7 +5,7 @@ import pytest -from pload.cli import build_parser, main, shell_script +from pload.cli import main, shell_script from pload.errors import PloadError from pload.managers.platform import ConfigManager from pload.managers.venv import VenvManager @@ -130,12 +130,11 @@ def test_git_recipe_roundtrip(tmp_path, monkeypatch): def test_commands_and_shell_integration(capsys): - assert build_parser().parse_args(["export", "example", "-b", "all"]).bundle == "all" - for command in ("export", "restore", "repo"): + for command in ("repo",): assert main([command, "-h"]) == 0 for shell in ("bash", "zsh", "fish", "powershell"): assert command in shell_script(shell) - assert "--bundle" in capsys.readouterr().out + assert "artifact providers" in capsys.readouterr().out def test_repository_list_alias(tmp_path, capsys): From 8bdb0da294420bce7cad7d2a1694d7a23d2b2571 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 03:20:09 +0800 Subject: [PATCH 08/41] Keep Python 3.8 build compatibility --- pyproject.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2e43b5f..30a5b56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=77"] +requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" [project] @@ -8,8 +8,7 @@ version = "1.1.0a2" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" -license = "Apache-2.0" -license-files = ["LICENSE"] +license = { file = "LICENSE" } authors = [{ name = "Yunming Hu", email = "hugonelsonm3@gmail.com" }] keywords = ["python", "venv", "virtual-environment", "pyenv"] classifiers = [ From 877666a8c830aa6028da238ab5a56052e78fe8f1 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 03:23:17 +0800 Subject: [PATCH 09/41] Handle inaccessible Windows Python aliases --- src/pload/managers/pyversion.py | 17 ++++++++++++++--- tests/test_pyversion.py | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/pload/managers/pyversion.py b/src/pload/managers/pyversion.py index a9ea2c4..dacf6e1 100644 --- a/src/pload/managers/pyversion.py +++ b/src/pload/managers/pyversion.py @@ -432,7 +432,7 @@ def _python_files(self, root, recursive=False): return [] def _classify_path(self, path, declared): - resolved = str(path.resolve()).replace("\\", "/").lower() + resolved = str(self._resolved_path(path)).replace("\\", "/").lower() configured_uv = str(Path(self.config.python["install_dir"]).expanduser()).replace( "\\", "/" ).lower() @@ -481,7 +481,18 @@ def _is_python_name(cls, name): @staticmethod def _path_key(path): - return os.path.normcase(str(Path(path).expanduser().resolve())) + return os.path.normcase(str(PythonManager._resolved_path(path))) + + @staticmethod + def _resolved_path(path): + path = Path(path).expanduser() + try: + return path.resolve() + except OSError: + # Windows Store app-execution aliases can look like files but reject + # GetFinalPathNameByHandle on older Python releases. They are still + # valid discovery candidates and will be rejected safely by _probe. + return path.absolute() @staticmethod def _version_key(version): @@ -492,7 +503,7 @@ def _unique_paths(paths): unique = [] seen = set() for path in paths: - key = os.path.normcase(str(Path(path).expanduser().resolve())) + key = os.path.normcase(str(PythonManager._resolved_path(path))) if key not in seen: seen.add(key) unique.append(Path(path).expanduser()) diff --git a/tests/test_pyversion.py b/tests/test_pyversion.py index b73ab00..c4dde5e 100644 --- a/tests/test_pyversion.py +++ b/tests/test_pyversion.py @@ -1,3 +1,4 @@ +import os from dataclasses import replace from pathlib import Path @@ -19,6 +20,20 @@ def test_source_filter_rejects_unknown_types(): PythonManager.parse_sources(["mystery"]) +def test_unresolvable_windows_alias_uses_absolute_path(monkeypatch, tmp_path): + alias = tmp_path / "python.exe" + original_resolve = Path.resolve + + def inaccessible(self): + if self == alias: + raise OSError("inaccessible app execution alias") + return original_resolve(self) + + monkeypatch.setattr(Path, "resolve", inaccessible) + assert PythonManager._resolved_path(alias) == alias.absolute() + assert PythonManager._path_key(alias) == os.path.normcase(str(alias.absolute())) + + def test_discovery_deduplicates_resolved_paths_and_filters(monkeypatch, tmp_path): python = tmp_path / "python3.12" python.touch() From 1df64cc587961020fb156bc6f7d7f0021395c11f Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 03:58:48 +0800 Subject: [PATCH 10/41] Batch remote artifacts and show reproduction progress --- CHANGELOG.md | 6 +- README.md | 2 +- pyproject.toml | 2 +- src/pload/__init__.py | 2 +- src/pload/cli.py | 12 +- src/pload/declarative.py | 287 +++++++++++++++++++++++++++++--------- src/pload/snapshots.py | 9 +- tests/test_declarative.py | 9 +- 8 files changed, 256 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a5cc4e..5d33581 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -Pre-release: `1.1.0a2` +Pre-release: `1.1.0a3` ### Added @@ -26,6 +26,10 @@ Pre-release: `1.1.0a2` - Keep an external environment's interpreter path intact when probing packages, instead of following its Python symlink out to the base installation. +- Batch SSH repository inventory, publication and retrieval so environments with + several packages do not spend most of their time reconnecting. Show package + and transfer progress, enforce bounded SSH/SFTP operations, and report actual + transfer timeouts instead of appearing frozen. ## 1.0.0 - 2026-09-24 diff --git a/README.md b/README.md index dc2cf21..155ca1a 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ pload-install --yes --package-spec "pload==1.0.0" To opt into the declarative-environment pre-release for testing: ```console -pload-install --yes --package-spec "pload==1.1.0a2" +pload-install --yes --package-spec "pload==1.1.0a3" ``` To always follow the newest published release instead of pinning a version, diff --git a/pyproject.toml b/pyproject.toml index 30a5b56..96e9c0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pload" -version = "1.1.0a2" +version = "1.1.0a3" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" diff --git a/src/pload/__init__.py b/src/pload/__init__.py index 5903a87..170ef9f 100644 --- a/src/pload/__init__.py +++ b/src/pload/__init__.py @@ -1 +1 @@ -__version__ = "1.1.0a2" +__version__ = "1.1.0a3" diff --git a/src/pload/cli.py b/src/pload/cli.py index 89ea701..bf382bc 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -744,9 +744,13 @@ def run(argv=None): manager = DeclarativeEnvironmentManager(config) if command == "describe": + progress_console = Console(highlight=False) print(manager.describe( args.source, args.output, args.name, args.mode, args.repository, args.package_sources, + progress=lambda message: progress_console.print( + Text("• " + message, style="cyan") + ), )) elif command == "plan": plan = manager.plan(args.file, args.offline) @@ -755,7 +759,13 @@ def run(argv=None): else: print_declarative_plan(plan) else: - print(manager.apply(args.file, args.name, args.offline)) + progress_console = Console(highlight=False) + print(manager.apply( + args.file, args.name, args.offline, + progress=lambda message: progress_console.print( + Text("• " + message, style="cyan") + ), + )) return 0 if command == "new": diff --git a/src/pload/declarative.py b/src/pload/declarative.py index d4e1100..51741c7 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -216,6 +216,11 @@ def _wheel_identity(filename): class ArtifactRepository: """Content-addressed local and SSH artifact access.""" + SSH_OPTIONS = ( + "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", + "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=4", + ) + @staticmethod def _local_root(spec, base): root = Path(spec["location"]).expanduser() @@ -225,69 +230,166 @@ def _local_root(spec, base): @classmethod def contains(cls, spec, checksum, base): + return checksum in cls.contains_many(spec, [checksum], base) + + @classmethod + def contains_many(cls, spec, checksums, base): + checksums = sorted(set(checksums)) if spec["kind"] == "local": - artifact = cls._local_root(spec, base) / "objects" / checksum - return artifact.is_file() and digest(artifact) == checksum + root = cls._local_root(spec, base) / "objects" + return { + checksum for checksum in checksums + if (root / checksum).is_file() and digest(root / checksum) == checksum + } host, root = RepositoryManager.ssh_location(spec["location"]) - result = execute([ - "ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", host, - f"if test -f {shlex.quote(root + '/objects/' + checksum)}; then printf yes; fi", - ]) - return result == "yes" + object_root = root + "/objects" + checks = "; ".join( + f"test -f {shlex.quote(object_root + '/' + checksum)} && printf '%s\\n' " + f"{shlex.quote(checksum)} || true" + for checksum in checksums + ) + return set(execute( + ["ssh", *cls.SSH_OPTIONS, host, checks], timeout=75, + ).splitlines()) @classmethod def publish(cls, spec, source, checksum, base): - source = Path(source) - if digest(source) != checksum: - raise PloadError("artifact changed before publication") + cls.publish_many(spec, [(source, checksum)], base) + + @classmethod + def publish_many(cls, spec, artifacts, base): + """Publish multiple objects with one SSH query and one transfer.""" + artifacts = [(Path(source), checksum) for source, checksum in artifacts] + if not artifacts: + return + for source, checksum in artifacts: + if digest(source) != checksum: + raise PloadError("artifact changed before publication") if spec["kind"] == "local": - target = cls._local_root(spec, base) / "objects" / checksum - target.parent.mkdir(parents=True, exist_ok=True) - if target.exists() and digest(target) != checksum: - raise PloadError("repository object checksum mismatch") - if not target.exists(): - temporary = target.with_name(target.name + "." + uuid.uuid4().hex) - shutil.copyfile(source, temporary) - temporary.replace(target) + for source, checksum in artifacts: + target = cls._local_root(spec, base) / "objects" / checksum + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and digest(target) != checksum: + raise PloadError("repository object checksum mismatch") + if not target.exists(): + temporary = target.with_name(target.name + "." + uuid.uuid4().hex) + shutil.copyfile(source, temporary) + temporary.replace(target) return host, root = RepositoryManager.ssh_location(spec["location"]) - obj = root + "/objects/" + checksum - execute(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", host, - f"mkdir -p {shlex.quote(root + '/objects')}"]) - if not cls.contains(spec, checksum, base): - pending = obj + "." + uuid.uuid4().hex - execute(["scp", "-q", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", - str(source), host + ":" + pending]) - execute(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", host, - f"mv -n {shlex.quote(pending)} {shlex.quote(obj)}"]) - remote_hash = execute(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", host, - f"sha256sum {shlex.quote(obj)}"]).split()[0] - if remote_hash != checksum: - raise PloadError("remote artifact checksum mismatch") + checksums = sorted({checksum for _, checksum in artifacts}) + object_root = root + "/objects" + checks = "; ".join( + f"test -f {shlex.quote(object_root + '/' + checksum)} && printf '%s\\n' " + f"{shlex.quote(checksum)} || true" + for checksum in checksums + ) + existing = set(execute( + ["ssh", *cls.SSH_OPTIONS, host, + f"mkdir -p {shlex.quote(object_root)}; {checks}"], + timeout=75, + ).splitlines()) + missing = [(source, checksum) for source, checksum in artifacts + if checksum not in existing] + if missing: + pending_name = ".upload-" + uuid.uuid4().hex + remote_pending = root + "/" + pending_name + with tempfile.TemporaryDirectory(prefix="pload-publish-") as temporary: + stage = Path(temporary) + for source, checksum in missing: + staged = stage / checksum + try: + os.link(source, staged) + except OSError: + shutil.copyfile(source, staged) + execute( + ["ssh", *cls.SSH_OPTIONS, host, + f"mkdir -p {shlex.quote(remote_pending)}"], + timeout=75, + ) + execute( + ["scp", "-q", *cls.SSH_OPTIONS, + *[str(stage / checksum) for _, checksum in missing], + host + ":" + remote_pending + "/"], + timeout=300, + ) + commands = [] + for _, checksum in missing: + pending = remote_pending + "/" + checksum + target = object_root + "/" + checksum + commands.append( + f"test \"$(sha256sum {shlex.quote(pending)} | cut -d' ' -f1)\" = " + f"{shlex.quote(checksum)}" + ) + commands.append(f"mv -n {shlex.quote(pending)} {shlex.quote(target)}") + commands.append(f"rm -f {shlex.quote(pending)}") + commands.append(f"rmdir {shlex.quote(remote_pending)}") + execute( + ["ssh", *cls.SSH_OPTIONS, host, " && ".join(commands)], + timeout=75, + ) + verify = " && ".join( + f"test \"$(sha256sum {shlex.quote(object_root + '/' + checksum)} " + f"| cut -d' ' -f1)\" = {shlex.quote(checksum)}" + for checksum in checksums + ) + execute(["ssh", *cls.SSH_OPTIONS, host, verify], timeout=75) @classmethod def fetch(cls, spec, checksum, destination, base): - destination = Path(destination) - destination.parent.mkdir(parents=True, exist_ok=True) - temporary = destination.with_name(destination.name + "." + uuid.uuid4().hex) - if spec["kind"] == "local": - shutil.copyfile(cls._local_root(spec, base) / "objects" / checksum, temporary) - else: - host, root = RepositoryManager.ssh_location(spec["location"]) - execute(["scp", "-q", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", - host + ":" + root + "/objects/" + checksum, str(temporary)]) - if digest(temporary) != checksum: - temporary.unlink(missing_ok=True) - raise PloadError("retrieved artifact checksum mismatch") - temporary.replace(destination) + cls.fetch_many(spec, [(checksum, destination)], base) + @classmethod + def fetch_many(cls, spec, artifacts, base): + """Fetch multiple content-addressed objects through one SFTP session.""" + artifacts = [(checksum, Path(destination)) for checksum, destination in artifacts] + if not artifacts: + return + for _, destination in artifacts: + destination.parent.mkdir(parents=True, exist_ok=True) + if spec["kind"] == "local": + root = cls._local_root(spec, base) / "objects" + for checksum, destination in artifacts: + temporary = destination.with_name( + destination.name + "." + uuid.uuid4().hex + ) + shutil.copyfile(root / checksum, temporary) + if digest(temporary) != checksum: + temporary.unlink(missing_ok=True) + raise PloadError("retrieved artifact checksum mismatch") + temporary.replace(destination) + return + host, root = RepositoryManager.ssh_location(spec["location"]) + with tempfile.TemporaryDirectory(prefix="pload-fetch-") as temporary: + stage = Path(temporary) + commands = [] + for checksum, _ in artifacts: + commands.append( + f"get {root}/objects/{checksum} {stage / checksum}" + ) + execute( + ["sftp", "-q", "-b", "-", *cls.SSH_OPTIONS, host], + timeout=300, input_text="\n".join(commands) + "\n", + ) + for checksum, destination in artifacts: + downloaded = stage / checksum + if not downloaded.is_file() or digest(downloaded) != checksum: + raise PloadError("retrieved artifact checksum mismatch") + temporary_destination = destination.with_name( + destination.name + "." + uuid.uuid4().hex + ) + shutil.copyfile(downloaded, temporary_destination) + temporary_destination.replace(destination) class DeclarativeEnvironmentManager: def __init__(self, config): self.config = config self.cache = config.home / "cache" / "wheels" - def describe(self, source, output, name=None, mode="exact", repository=None, sources=None): + def describe( + self, source, output, name=None, mode="exact", repository=None, sources=None, + progress=None, + ): venvs = VenvManager(self.config) candidate = Path(source).expanduser() if candidate.is_file(): @@ -365,9 +467,15 @@ def describe(self, source, output, name=None, mode="exact", repository=None, sou "package": [], } self.cache.mkdir(parents=True, exist_ok=True) + publication = [] with tempfile.TemporaryDirectory(prefix="pload-describe-") as temporary: wheel_dir = Path(temporary) - for installed in packages: + for index_number, installed in enumerate(packages, 1): + if progress: + progress( + f"[{index_number}/{len(packages)}] Locking " + f"{installed['name']}=={installed['version']}" + ) source_name, package_index = source_overrides.get( normalized_name(installed["name"]), ("default", index) ) @@ -387,15 +495,25 @@ def describe(self, source, output, name=None, mode="exact", repository=None, sou shutil.copyfile(wheel, cached) locations = [] if selected_repository: - ArtifactRepository.publish( - repositories[selected_repository], cached, checksum, output.parent, - ) - locations.append(selected_repository) - record["artifact"].append({ + publication.append((cached, checksum)) + artifact = { "filename": wheel.name, "sha256": checksum, "tags": _wheel_tags(wheel.name), "repositories": locations, - }) + } + record["artifact"].append(artifact) data["package"].append(record) + if selected_repository and publication: + if progress: + progress( + f"Publishing {len(publication)} locked artifacts to " + f"{selected_repository}" + ) + ArtifactRepository.publish_many( + repositories[selected_repository], publication, output.parent, + ) + for package in data["package"]: + for artifact in package.get("artifact", []): + artifact["repositories"].append(selected_repository) validate_manifest(data) output.parent.mkdir(parents=True, exist_ok=True) temporary = output.with_name(output.name + ".tmp") @@ -451,6 +569,22 @@ def plan(self, manifest_path, offline=False): network_allowed = policy.get("network", "allow") == "allow" and not offline repositories = data.get("repositories", {}) packages = self._locked_packages(data) + repository_checksums = {} + for package in packages: + for artifact in package.get("artifact", []): + for repo_name in artifact.get("repositories", []): + if repo_name in repositories: + repository_checksums.setdefault(repo_name, set()).add( + artifact["sha256"] + ) + repository_objects = {} + for repo_name, checksums in repository_checksums.items(): + try: + repository_objects[repo_name] = ArtifactRepository.contains_many( + repositories[repo_name], checksums, path.parent, + ) + except PloadError: + repository_objects[repo_name] = set() plans = [] for package in packages: candidates = [] @@ -465,17 +599,11 @@ def plan(self, manifest_path, offline=False): )) for repo_name in artifact.get("repositories", []): spec = repositories.get(repo_name) - if spec: - try: - available = ArtifactRepository.contains( - spec, artifact["sha256"], path.parent - ) - except PloadError: - available = False - if available: - candidates.append(self._candidate( - "repository", repo_name, "remote", (0, 0, 1, repo_name), artifact - )) + if (spec and artifact["sha256"] + in repository_objects.get(repo_name, set())): + candidates.append(self._candidate( + "repository", repo_name, "remote", (0, 0, 1, repo_name), artifact + )) if network_allowed: for source_name in package.get("sources", []): source = data.get("sources", {}).get(source_name) @@ -528,7 +656,7 @@ def _locked_packages(data): "sources": ["default"], "artifact": []}) return result - def apply(self, manifest_path, name=None, offline=False): + def apply(self, manifest_path, name=None, offline=False, progress=None): path, data = load_manifest(manifest_path) plan = self.plan(path, offline=offline) unavailable = [item for item in plan["packages"] if not item["selected"]] @@ -555,9 +683,34 @@ def apply(self, manifest_path, name=None, offline=False): artifacts = [] compatible = [] self.cache.mkdir(parents=True, exist_ok=True) + repository_fetches = {} + for package_plan in plan["packages"]: + selected = package_plan["selected"] + artifact = selected.get("artifact") + if artifact and selected["method"] == "repository": + cached = self.cache / artifact["filename"] + if not (cached.is_file() and digest(cached) == artifact["sha256"]): + repository_fetches.setdefault(selected["location"], []).append( + (artifact["sha256"], cached) + ) + for repo_name, fetches in repository_fetches.items(): + if progress: + progress(f"Fetching {len(fetches)} artifacts from {repo_name}") + try: + ArtifactRepository.fetch_many( + data["repositories"][repo_name], fetches, path.parent, + ) + except PloadError as exc: + if progress: + progress(f"Batch fetch failed; trying fallback routes: {exc}") with tempfile.TemporaryDirectory(prefix="pload-apply-") as temporary: stage = Path(temporary) - for package_plan in plan["packages"]: + for index_number, package_plan in enumerate(plan["packages"], 1): + if progress: + progress( + f"[{index_number}/{len(plan['packages'])}] Preparing " + f"{package_plan['name']}=={package_plan['version']}" + ) selected = package_plan["selected"] artifact = selected.get("artifact") if artifact: @@ -590,6 +743,8 @@ def apply(self, manifest_path, name=None, offline=False): else: compatible.append(package_plan) venvs = VenvManager(self.config) + if progress: + progress(f"Creating environment {target_name}") env = venvs.create_venv( version=str(interpreter), name=target_name, description=f"Applied from {path.name}", @@ -620,6 +775,8 @@ def apply(self, manifest_path, name=None, offline=False): execute(pip + ["install", "--no-index", "--no-deps", "-r", str(requirements)]) execute(pip + ["check"], capture=False) + if progress: + progress("Verified exact requirements and dependency consistency") (env / ".pload-manifest.sha256").write_text( expected_state + "\n", encoding="utf-8" ) diff --git a/src/pload/snapshots.py b/src/pload/snapshots.py index 55ecbb6..7e01e90 100644 --- a/src/pload/snapshots.py +++ b/src/pload/snapshots.py @@ -50,9 +50,14 @@ """ -def execute(command, capture=True, cwd=None): +def execute(command, capture=True, cwd=None, timeout=None, input_text=None): try: - result = subprocess.run(command, capture_output=capture, text=True, cwd=cwd, check=False) + result = subprocess.run( + command, capture_output=capture, text=True, cwd=cwd, check=False, + timeout=timeout, input=input_text, + ) + except subprocess.TimeoutExpired as exc: + raise PloadError(f"{command[0]} timed out after {timeout} seconds") from exc except OSError as exc: raise PloadError(f"cannot run {command[0]}: {exc}") from exc if result.returncode: diff --git a/tests/test_declarative.py b/tests/test_declarative.py index 3d25bd1..41a548f 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -121,9 +121,11 @@ def test_describe_plan_apply_through_content_repository(tmp_path): manager = DeclarativeEnvironmentManager(source_config) manager.cache.mkdir(parents=True) shutil.copyfile(wheel, manager.cache / wheel.name) + describe_progress = [] manifest = manager.describe( "source", tmp_path / "pload.toml", name="restored", sources=["pload-demo=https://download.example.invalid/cu121"], + progress=describe_progress.append, ) _, data = load_manifest(manifest) artifact = data["package"][0]["artifact"][0] @@ -132,6 +134,8 @@ def test_describe_plan_apply_through_content_repository(tmp_path): assert data["sources"]["package-pload-demo"]["url"].endswith("/cu121") assert artifact["repositories"] == ["lab"] assert (repository / "objects" / artifact["sha256"]).is_file() + assert any("Locking pload-demo==1.0" in item for item in describe_progress) + assert any("Publishing 1 locked artifact" in item for item in describe_progress) shutil.rmtree(source_config.home / "cache") target_config = ConfigManager(home=tmp_path / "target-home") @@ -139,12 +143,15 @@ def test_describe_plan_apply_through_content_repository(tmp_path): target = DeclarativeEnvironmentManager(target_config) plan = target.plan(manifest) assert plan["packages"][0]["selected"]["method"] == "repository" - restored = target.apply(manifest) + apply_progress = [] + restored = target.apply(manifest, progress=apply_progress.append) output = execute([target_config.get_pip_command(restored)[0], "-c", "import pload_demo; print(pload_demo.answer)"]) assert output == "42" assert target.apply(manifest) == restored assert digest(target.cache / artifact["filename"]) == artifact["sha256"] + assert any("Preparing pload-demo==1.0" in item for item in apply_progress) + assert any("Verified exact requirements" in item for item in apply_progress) def test_apply_rolls_back_a_new_environment_after_install_failure(tmp_path): From 1c955c5be88fce95deacd8a5ec6199236b24b6f8 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 13:41:51 +0800 Subject: [PATCH 11/41] Document and enforce declarative configuration schema --- CHANGELOG.md | 5 +- README.md | 7 +- docs/declarative-environments.md | 178 ++++++++++++++++++++++++++++++- pyproject.toml | 2 +- src/pload/__init__.py | 2 +- src/pload/declarative.py | 68 +++++++++++- tests/test_declarative.py | 47 ++++++++ 7 files changed, 297 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d33581..d4da2d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -Pre-release: `1.1.0a3` +Pre-release: `1.1.0a4` ### Added @@ -16,6 +16,9 @@ Pre-release: `1.1.0a3` while keeping credentials out of the portable configuration. - Reuse exact wheel archives by SHA-256, deduplicate local/SSH storage, record detected NVIDIA driver provenance and roll back failed materializations. +- Add a complete field-by-field `pload.toml` reference, enforce agreement between + desired dependency pins and package locks, and make policy repositories and + `publish_missing_artifacts` drive lookup and automatic cache publication. ### Removed diff --git a/README.md b/README.md index 155ca1a..7247ccd 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,9 @@ pload apply pload.toml ``` See the [declarative environment guide](docs/declarative-environments.md) for the -schema, exact versus compatible policies, resource-selection algorithm, CUDA -wheel caching, idempotency and current platform boundaries. +[complete TOML field reference](docs/declarative-environments.md#complete-ploadtoml-field-reference), +exact versus compatible policies, resource-selection algorithm, CUDA wheel +caching, idempotency and current platform boundaries. ## Upgrade @@ -112,7 +113,7 @@ pload-install --yes --package-spec "pload==1.0.0" To opt into the declarative-environment pre-release for testing: ```console -pload-install --yes --package-spec "pload==1.1.0a3" +pload-install --yes --package-spec "pload==1.1.0a4" ``` To always follow the newest published release instead of pinning a version, diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index a6393e7..88f3d2a 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -37,6 +37,9 @@ system = "Linux" machine = "x86_64" dependencies = ["numpy==2.1.3", "torch==2.5.0+cu121"] +[capabilities] +nvidia_driver = ["550.90.07"] + [policy] reproducibility = "exact" network = "allow" @@ -68,6 +71,177 @@ repositories = ["lab"] credentials or query-string tokens are rejected; authentication belongs to the machine, not a shareable environment description. +## Complete `pload.toml` field reference + +The file has seven logical parts. `describe` normally writes all lock-related +fields, but the same schema can be reviewed or authored by hand. Unknown TOML +fields are reserved for future schema versions and should not be relied upon. + +### Top-level identity + +| Field | Type | Required | Meaning | +| --- | --- | --- | --- | +| `schema` | integer | yes | File-format version. The current and only accepted value is `1`. It is not the pload package version. | +| `name` | string | yes | Default name used by `pload apply`. It must be 1–100 letters, numbers, dots, underscores or hyphens, starting with a letter or number. `apply --name` may override it without changing the file. | + +### `[environment]`: desired runtime state + +| Field | Type | Default | Meaning and enforcement | +| --- | --- | --- | --- | +| `python` | string | none; required | Exact Python version, currently including the patch component, such as `3.12.7`. The planner reuses a matching discovered interpreter or schedules an installation when network policy allows it. | +| `implementation` | string | `"cpython"` | `sys.implementation.name`, normally `cpython` or `pypy`. `apply` rejects a different implementation. | +| `system` | string | empty | Target `platform.system()`, such as `Linux`, `Darwin` or `Windows`. A non-empty value is a hard compatibility constraint. | +| `machine` | string | empty | Target `platform.machine()`, such as `x86_64`, `AMD64`, `arm64` or `aarch64`. A non-empty value is a hard compatibility constraint. | +| `dependencies` | array of strings | `[]` | Complete desired application package set. Every entry must be an exact `NAME==VERSION` pin. When `[[package]]` lock entries exist, the two sections must describe the same normalized names and versions. pip, setuptools and wheel bootstrap packages are intentionally omitted by `describe`. | + +The environment section is desired state, not a command sequence. Paths to a +source virtual environment are never stored here, so the configuration remains +portable. The current schema describes one target platform and one exact Python +version; use separate files for different targets until multi-target locks exist. + +### `[capabilities]`: observed non-Python context + +Each capability is an array of strings. `describe` currently recognizes +`nvidia_driver` and records the driver versions reported by `nvidia-smi`: + +```toml +[capabilities] +nvidia_driver = ["550.90.07"] +``` + +Capabilities are provenance and planning information, not installable Python +artifacts. The current executor does not install, downgrade or require an exact +NVIDIA driver match. CUDA-enabled wheels are locked separately under +`[[package.artifact]]`; drivers and system libraries remain host responsibilities. +Unknown capability keys are preserved by the parser as arrays of strings but do +not affect planning in schema 1. + +### `[policy]`: permitted ways to reach the desired state + +| Field | Allowed values / type | Default | Meaning | +| --- | --- | --- | --- | +| `reproducibility` | `"exact"` or `"compatible"` | `"exact"` | `exact` requires a declared wheel identity and SHA-256 for every package. `compatible` keeps exact versions but may obtain a new artifact build from an index. | +| `network` | `"allow"` or `"offline"` | `"allow"` | Controls internet index access and automatic Python installation. `offline` still permits configured local and SSH repositories. CLI `--offline` can only make a run stricter; it cannot override an offline file to allow internet access. | +| `source_build` | `"fallback"` or `"forbid"` | `"fallback"` | Applies to compatible re-resolution. `forbid` passes `--only-binary=:all:`; `fallback` allows pip to use an sdist when no wheel exists. Exact mode always installs the locked wheel and never rebuilds it. | +| `publish_missing_artifacts` | boolean | `true` | After `apply` obtains exact wheels, publish their SHA-addressed bytes to every repository named by `policy.repositories`. `false` makes those repositories read-only candidates for this configuration. | +| `repositories` | array of strings | `[]` | Logical repository names preferred for lookup and optional automatic publication. Every name must have a matching `[repositories.NAME]` table. This is a candidate set, not a user-visible upload sequence. | + +Command-line `--offline` is intentionally one-way: it prevents a configuration +from unexpectedly using the internet, while a file declaring `network = +"offline"` cannot be weakened accidentally by omitting the flag. + +### `[sources.NAME]`: Python package indexes + +`NAME` is a logical identifier used by package locks, for example `default` or +`pytorch-cu121`: + +```toml +[sources.pytorch-cu121] +kind = "index" +url = "https://download.pytorch.org/whl/cu121" +``` + +| Field | Type | Meaning | +| --- | --- | --- | +| `kind` | string | Must be `"index"` in schema 1. | +| `url` | string | HTTP(S) Simple API base URL. Embedded usernames, passwords and query parameters are rejected because this file is intended to be shared. Configure authentication on the machine instead. | + +Source names must follow the same safe-name rules as the environment name. A +package lists its allowed source names in `package.sources`. In compatible mode, +the first listed source is the primary resolver source. In exact mode, an index +is accepted only if it returns the declared wheel filename and its bytes match +the declared SHA-256. + +### `[repositories.NAME]`: reusable artifact stores + +Repositories store immutable bytes by content hash rather than by environment: + +```toml +[repositories.lab] +kind = "ssh" +location = "frpxiaoxin:/home/tibless/pload-cloud" + +[repositories.shared-disk] +kind = "local" +location = "./pload-artifacts" +``` + +| Field | Allowed values | Meaning | +| --- | --- | --- | +| `kind` | `"local"` or `"ssh"` | Selects filesystem copying or OpenSSH/SFTP transport. | +| `location` | path or `HOST:/absolute/path` | A local absolute path, a path relative to the TOML file's directory, or an SSH alias/host plus absolute remote directory. SSH paths cannot contain spaces or `..`; `/` itself is rejected. | + +The physical layout is `objects/`. SSH credentials, host keys, ports, +proxies and jump hosts remain in normal OpenSSH configuration. The remote host +must provide a POSIX shell and `sha256sum`. Repository names referenced by a +policy or artifact must exist in this file. Missing or unreachable providers are +removed from the plan and another valid route is tried. + +The TOML control file may itself be versioned in Git or hosted on GitHub. Large +wheel bytes should remain in these content-addressed local/SSH repositories rather +than being committed to Git. + +### `[[package]]`: one locked package + +There is one array entry for every application dependency: + +| Field | Type | Required | Meaning | +| --- | --- | --- | --- | +| `name` | string | yes | Distribution name from installed metadata. Comparison is normalized according to Python package-name rules, so `python_dateutil` and `python-dateutil` identify the same project. | +| `version` | string | yes | Exact installed version, including a local suffix such as `+cu121` when present. | +| `sources` | array of strings | no | Logical `[sources.NAME]` entries allowed to satisfy this package. Each referenced source must exist. `describe -s PACKAGE=URL` creates a package-specific source when installed metadata cannot reliably reveal it. | +| `artifact` | array of tables | no | Exact wheel identities accepted for this package. Exact policy needs at least one reachable artifact route; compatible policy may omit artifacts and re-resolve the pinned version. | + +If `[[package]]` is entirely omitted, pload synthesizes package records from +`environment.dependencies`. That form is useful only for compatible online +resolution because it contains no exact artifact identity. + +### `[[package.artifact]]`: exact wheel identity + +An artifact table belongs to the immediately preceding `[[package]]` entry: + +| Field | Type | Required | Meaning | +| --- | --- | --- | --- | +| `filename` | string | yes | Exact wheel filename, including Python, ABI and platform tags. Source archives are not accepted as exact artifacts. | +| `sha256` | string | yes | Lowercase 64-character SHA-256 digest of the complete wheel. Every cache, repository and index retrieval is checked against it before installation. | +| `tags` | array of strings | no | Wheel compatibility tags extracted from the filename, such as `cp312-cp312-linux_x86_64` or `py3-none-any`. They are recorded for inspection; pip remains the final wheel-compatibility validator. | +| `repositories` | array of strings | no | Repositories known to contain this digest. The planner also checks repositories listed globally in `policy.repositories`, allowing newly cached objects to be reused without rewriting the package entry. | + +The schema permits more than one artifact entry for future multi-platform locks, +but schema 1 `describe` emits one artifact for the described target and the +environment's OS/architecture remain hard constraints. A hash proves byte +identity and detects corruption; it does not establish that a repository owner or +package is trustworthy. + +### Minimal hand-written compatible configuration + +This is the smallest practical file that asks pload to resolve pinned packages +online without claiming byte-for-byte wheel reproducibility: + +```toml +schema = 1 +name = "analysis" + +[environment] +python = "3.12.7" +dependencies = ["numpy==2.1.3", "pandas==2.2.3"] + +[policy] +reproducibility = "compatible" +network = "allow" +source_build = "forbid" +publish_missing_artifacts = false + +[sources.default] +kind = "index" +url = "https://pypi.org/simple" +``` + +`implementation`, `system`, `machine`, `capabilities`, `repositories` and +`[[package]]` are filled or omitted according to the defaults described above. +For durable sharing, prefer `pload describe` so exact package and artifact lock +entries are generated and validated automatically. + ## Describe an existing environment ```console @@ -197,5 +371,5 @@ is simply removed from the candidate set, and apply tries the next valid route. The earlier `export → repo push/pull → restore` interface is not the public model for 1.1. Its useful internals—probing, hashing, caches and content-addressed SSH storage—remain providers behind `describe`, `plan` and `apply`. Before a stable -storage—remain providers behind `describe`, `plan` and `apply`. The experimental -format was never part of a stable release and is intentionally not carried forward. +release, the experimental format was abandoned and is intentionally not carried +forward. diff --git a/pyproject.toml b/pyproject.toml index 96e9c0c..80f7bc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pload" -version = "1.1.0a3" +version = "1.1.0a4" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" diff --git a/src/pload/__init__.py b/src/pload/__init__.py index 170ef9f..6b46d25 100644 --- a/src/pload/__init__.py +++ b/src/pload/__init__.py @@ -1 +1 @@ -__version__ = "1.1.0a3" +__version__ = "1.1.0a4" diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 51741c7..f23a876 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -120,6 +120,18 @@ def load_manifest(path): data.setdefault("repositories", {}) data.setdefault("capabilities", {}) data.setdefault("package", []) + environment = data.setdefault("environment", {}) + environment.setdefault("implementation", "cpython") + environment.setdefault("system", "") + environment.setdefault("machine", "") + environment.setdefault("dependencies", []) + policy = data.get("policy") + if isinstance(policy, dict): + policy.setdefault("reproducibility", "exact") + policy.setdefault("network", "allow") + policy.setdefault("source_build", "fallback") + policy.setdefault("publish_missing_artifacts", True) + policy.setdefault("repositories", []) for package in data["package"]: package.setdefault("sources", []) package.setdefault("artifact", []) @@ -143,6 +155,12 @@ def validate_manifest(data): raise PloadError("policy.network must be allow or offline") if policy.get("source_build", "fallback") not in {"fallback", "forbid"}: raise PloadError("policy.source_build must be fallback or forbid") + if not isinstance(policy.get("publish_missing_artifacts", True), bool): + raise PloadError("policy.publish_missing_artifacts must be true or false") + policy_repositories = policy.get("repositories", []) + if (not isinstance(policy_repositories, list) + or any(not isinstance(name, str) for name in policy_repositories)): + raise PloadError("policy.repositories must be an array of repository names") dependencies = environment.get("dependencies", []) if not isinstance(dependencies, list) or any(not PIN.fullmatch(item) for item in dependencies): raise PloadError("environment.dependencies must contain exact NAME==VERSION pins") @@ -187,6 +205,19 @@ def validate_manifest(data): f"artifact {artifact['filename']} references unknown repository " f"{repository_name}" ) + if data.get("package"): + declared_pins = { + (normalized_name(match.group(1)), match.group(2)) + for item in dependencies if (match := PIN.fullmatch(item)) + } + locked_pins = { + (normalized_name(package["name"]), package["version"]) + for package in data["package"] + } + if declared_pins != locked_pins: + raise PloadError( + "environment.dependencies and [[package]] locks must describe the same pins" + ) for name, repository in data.get("repositories", {}).items(): safe_name(name) kind = repository.get("kind") @@ -198,6 +229,14 @@ def validate_manifest(data): raise PloadError(f"repository {name} has no location") else: raise PloadError(f"repository {name} must be local or ssh") + unknown_policy_repositories = set(policy_repositories).difference( + data.get("repositories", {}) + ) + if unknown_policy_repositories: + raise PloadError( + "policy.repositories references unknown repositories: " + + ", ".join(sorted(unknown_policy_repositories)) + ) return data @@ -569,10 +608,14 @@ def plan(self, manifest_path, offline=False): network_allowed = policy.get("network", "allow") == "allow" and not offline repositories = data.get("repositories", {}) packages = self._locked_packages(data) + preferred_repositories = policy.get("repositories", []) repository_checksums = {} for package in packages: for artifact in package.get("artifact", []): - for repo_name in artifact.get("repositories", []): + repo_names = dict.fromkeys( + artifact.get("repositories", []) + preferred_repositories + ) + for repo_name in repo_names: if repo_name in repositories: repository_checksums.setdefault(repo_name, set()).add( artifact["sha256"] @@ -592,12 +635,17 @@ def plan(self, manifest_path, offline=False): cached = self.cache / artifact["filename"] adjacent = path.parent / "artifacts" / artifact["filename"] if cached.is_file() and digest(cached) == artifact["sha256"]: - candidates.append(self._candidate("cache", cached, "ready", (0, 0, 0, 0))) + candidates.append(self._candidate( + "cache", cached, "ready", (0, 0, 0, 0), artifact + )) if adjacent.is_file() and digest(adjacent) == artifact["sha256"]: candidates.append(self._candidate( - "configuration-artifact", adjacent, "ready", (0, 0, 0, 1) + "configuration-artifact", adjacent, "ready", (0, 0, 0, 1), artifact )) - for repo_name in artifact.get("repositories", []): + repo_names = dict.fromkeys( + artifact.get("repositories", []) + preferred_repositories + ) + for repo_name in repo_names: spec = repositories.get(repo_name) if (spec and artifact["sha256"] in repository_objects.get(repo_name, set())): @@ -681,6 +729,7 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): return existing raise PloadError(f"environment {target_name!r} exists with different state") artifacts = [] + locked_files = [] compatible = [] self.cache.mkdir(parents=True, exist_ok=True) repository_fetches = {} @@ -740,8 +789,19 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): ) cached, artifact = acquired artifacts.append(cached.resolve().as_uri() + "#sha256=" + artifact["sha256"]) + locked_files.append((cached, artifact["sha256"])) else: compatible.append(package_plan) + if (locked_files + and data["policy"].get("publish_missing_artifacts", True)): + for repo_name in data["policy"].get("repositories", []): + if progress: + progress( + f"Publishing {len(locked_files)} locked artifacts to {repo_name}" + ) + ArtifactRepository.publish_many( + data["repositories"][repo_name], locked_files, path.parent, + ) venvs = VenvManager(self.config) if progress: progress(f"Creating environment {target_name}") diff --git a/tests/test_declarative.py b/tests/test_declarative.py index 41a548f..89365ca 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -49,6 +49,7 @@ def simple_manifest(mode="compatible"): "policy": { "reproducibility": mode, "network": "allow", "source_build": "fallback", "publish_missing_artifacts": True, + "repositories": [], }, "sources": {"default": {"kind": "index", "url": "https://pypi.org/simple"}}, "repositories": {}, @@ -89,6 +90,20 @@ def test_manifest_rejects_dangling_provider_references(): validate_manifest(data) +def test_manifest_rejects_mismatched_dependency_and_package_locks(): + data = simple_manifest() + data["environment"]["dependencies"] = ["other==1.0"] + with pytest.raises(PloadError, match="same pins"): + validate_manifest(data) + + +def test_manifest_rejects_unknown_policy_repository(): + data = simple_manifest() + data["policy"]["repositories"] = ["missing"] + with pytest.raises(PloadError, match="unknown repositories"): + validate_manifest(data) + + def test_compatible_plan_uses_index_and_offline_reports_missing(tmp_path): path = tmp_path / "pload.toml" path.write_text(dump_manifest(simple_manifest()), encoding="utf-8") @@ -180,6 +195,38 @@ def test_apply_rolls_back_a_new_environment_after_install_failure(tmp_path): VenvManager(config).resolve_existing("demo") +def test_apply_publishes_acquired_artifacts_to_policy_repository(tmp_path): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "artifacts") + checksum = digest(wheel) + repository = tmp_path / "store" + data = simple_manifest("exact") + data["name"] = "published" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + data["policy"]["repositories"] = ["lab"] + data["repositories"] = { + "lab": {"kind": "local", "location": str(repository)}, + } + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": checksum, + "tags": ["py3-none-any"], "repositories": [], + }], + }] + manifest = tmp_path / "pload.toml" + manifest.write_text(dump_manifest(data), encoding="utf-8") + DeclarativeEnvironmentManager(config).apply(manifest) + assert (repository / "objects" / checksum).is_file() + + def test_declarative_commands_and_shell_integration(capsys): parser = build_parser() assert parser.parse_args(["describe", "v1", "-o", "env.toml"]).output == "env.toml" From 794a2a941fa010ea1d9586c079e7893a3095c5a3 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 14:05:32 +0800 Subject: [PATCH 12/41] Reject incompatible cached wheels during planning --- CHANGELOG.md | 5 +- README.md | 2 +- docs/declarative-environments.md | 14 +++++ pyproject.toml | 3 +- src/pload/__init__.py | 2 +- src/pload/cli.py | 12 +++- src/pload/declarative.py | 91 +++++++++++++++++++++++---- tests/test_declarative.py | 105 ++++++++++++++++++++++++++++++- 8 files changed, 212 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4da2d5..4c4811a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -Pre-release: `1.1.0a4` +Pre-release: `1.1.0a5` ### Added @@ -19,6 +19,9 @@ Pre-release: `1.1.0a4` - Add a complete field-by-field `pload.toml` reference, enforce agreement between desired dependency pins and package locks, and make policy repositories and `publish_missing_artifacts` drive lookup and automatic cache publication. +- Validate locked wheel Python, ABI and platform tags against the requested + runtime during planning; incompatible cache entries can no longer be reported + as ready, and compatible mode explicitly plans network re-resolution instead. ### Removed diff --git a/README.md b/README.md index 7247ccd..9d5b48b 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ pload-install --yes --package-spec "pload==1.0.0" To opt into the declarative-environment pre-release for testing: ```console -pload-install --yes --package-spec "pload==1.1.0a4" +pload-install --yes --package-spec "pload==1.1.0a5" ``` To always follow the newest published release instead of pinning a version, diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index 88f3d2a..4cf7b44 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -130,6 +130,15 @@ Command-line `--offline` is intentionally one-way: it prevents a configuration from unexpectedly using the internet, while a file declaring `network = "offline"` cannot be weakened accidentally by omitting the flag. +Changing `environment.python` does not make native wheels portable. During +planning, pload parses every locked wheel filename and compares its Python, ABI +and platform tags with the requested runtime. For example, a cached `cp39-cp39` +wheel is rejected for Python 3.8 even when its checksum is valid. Under `exact`, +that makes the plan unavailable; under `compatible`, the planner may select an +explicit `index-resolve / network` route for that package. Universal wheels such +as `py3-none-any` can still be reused. `apply` never silently treats an +incompatible cached wheel as ready. + ### `[sources.NAME]`: Python package indexes `NAME` is a logical identifier used by package locks, for example `default` or @@ -299,6 +308,11 @@ The planner inventories: 5. package indexes named in the file; 6. target OS, architecture and Python implementation. +The Python row and package rows describe separate resources. A plan may reuse +every package from cache while still showing `Python: install-python · ... · +network`; in that case the download is the Python runtime itself. `reuse-python · +... · ready` means no runtime download is planned. + It first rejects candidates that violate hard requirements. Remaining candidates are ordered lexicographically by reproducibility loss, compatibility risk, network acquisition, execution cost and a deterministic tie-breaker. This avoids diff --git a/pyproject.toml b/pyproject.toml index 80f7bc4..e95f322 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pload" -version = "1.1.0a4" +version = "1.1.0a5" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" @@ -21,6 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3 :: Only", ] dependencies = [ + "packaging>=23.2,<26", "rich>=13.7,<15", "tomli>=2; python_version < '3.11'", ] diff --git a/src/pload/__init__.py b/src/pload/__init__.py index 6b46d25..711f574 100644 --- a/src/pload/__init__.py +++ b/src/pload/__init__.py @@ -1 +1 @@ -__version__ = "1.1.0a4" +__version__ = "1.1.0a5" diff --git a/src/pload/cli.py b/src/pload/cli.py index bf382bc..7fb4166 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -447,7 +447,8 @@ def print_declarative_plan(plan): console = Console(highlight=False) python = plan["python"] console.print(Panel.fit( - f"[bold]{plan['name']}[/]\nPython: [cyan]{python['method']}[/] · {python['location']}", + f"[bold]{plan['name']}[/]\nPython: [cyan]{python['method']}[/] · " + f"{python['location']} · [bold]{python['status']}[/]", title="[bold cyan]Environment plan[/]", border_style="blue", )) @@ -463,8 +464,13 @@ def print_declarative_plan(plan): table.add_row(package["name"], package["version"], selected["method"], selected["status"], selected["location"]) else: - table.add_row(package["name"], package["version"], "—", "unavailable", - "No resource satisfies the configuration") + rejected = package.get("rejections", []) + if rejected: + table.add_row(package["name"], package["version"], "—", "incompatible", + rejected[0]["artifact"]) + else: + table.add_row(package["name"], package["version"], "—", "unavailable", + "No resource satisfies the configuration") console.print(table) diff --git a/src/pload/declarative.py b/src/pload/declarative.py index f23a876..c0b0cac 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -3,6 +3,7 @@ import hashlib import json import os +import platform import re import shlex import shutil @@ -11,6 +12,9 @@ from pathlib import Path from urllib.parse import urlsplit +from packaging.tags import compatible_tags, cpython_tags, platform_tags +from packaging.utils import InvalidWheelFilename, parse_wheel_filename + try: import tomllib except ModuleNotFoundError: # pragma: no cover - exercised by the Python 3.8 CI job @@ -31,6 +35,49 @@ def normalized_name(value): return re.sub(r"[-_.]+", "-", value).lower() +def _normalized_machine(value): + aliases = { + "amd64": "x86_64", "x64": "x86_64", + "aarch64": "arm64", "arm64": "arm64", + } + normalized = str(value).lower() + return aliases.get(normalized, normalized) + + +def _target_wheel_tags(environment): + """Return wheel tags installable on the requested runtime on this host.""" + if (environment.get("system") + and environment["system"].lower() != platform.system().lower()): + return set() + if (environment.get("machine") + and _normalized_machine(environment["machine"]) + != _normalized_machine(platform.machine())): + return set() + try: + version = tuple(int(part) for part in environment["python"].split(".")[:2]) + except (KeyError, TypeError, ValueError): + return set() + if len(version) != 2: + return set() + platforms = list(platform_tags()) + implementation = environment.get("implementation", "cpython").lower() + interpreter = f"cp{version[0]}{version[1]}" if implementation == "cpython" else None + result = set(compatible_tags( + python_version=version, interpreter=interpreter, platforms=platforms, + )) + if implementation == "cpython": + result.update(cpython_tags(python_version=version, platforms=platforms)) + return result + + +def _artifact_is_compatible(artifact, supported_tags): + try: + wheel_tags = parse_wheel_filename(artifact["filename"])[3] + except (InvalidWheelFilename, KeyError, TypeError): + return False + return bool(wheel_tags.intersection(supported_tags)) + + def manifest_digest(data): encoded = json.dumps(data, sort_keys=True, separators=(",", ":")).encode() return hashlib.sha256(encoded).hexdigest() @@ -609,9 +656,20 @@ def plan(self, manifest_path, offline=False): repositories = data.get("repositories", {}) packages = self._locked_packages(data) preferred_repositories = policy.get("repositories", []) + supported_tags = _target_wheel_tags(data["environment"]) + try: + python = str(self.config.get_python_path(data["environment"]["python"])) + python_action = {"method": "reuse-python", "location": python, "status": "ready"} + except PythonNotFoundError: + python_action = { + "method": "install-python", "location": data["environment"]["python"], + "status": "network" if network_allowed else "unavailable", + } repository_checksums = {} for package in packages: for artifact in package.get("artifact", []): + if not _artifact_is_compatible(artifact, supported_tags): + continue repo_names = dict.fromkeys( artifact.get("repositories", []) + preferred_repositories ) @@ -631,7 +689,19 @@ def plan(self, manifest_path, offline=False): plans = [] for package in packages: candidates = [] + rejections = [] for artifact in package.get("artifact", []): + if not _artifact_is_compatible(artifact, supported_tags): + rejections.append({ + "artifact": artifact["filename"], + "reason": ( + f"incompatible with {data['environment']['implementation']} " + f"{data['environment']['python']} on " + f"{data['environment'].get('system') or platform.system()} " + f"{data['environment'].get('machine') or platform.machine()}" + ), + }) + continue cached = self.cache / artifact["filename"] adjacent = path.parent / "artifacts" / artifact["filename"] if cached.is_file() and digest(cached) == artifact["sha256"]: @@ -660,8 +730,7 @@ def plan(self, manifest_path, offline=False): "index-exact", source_name, "network", (0, 0, 2, source_name), artifact, )) - if (not package.get("artifact") - and policy.get("reproducibility") == "compatible" and network_allowed): + if policy.get("reproducibility") == "compatible" and network_allowed: source_name = (package.get("sources") or ["default"])[0] candidates.append(self._candidate( "index-resolve", source_name, "network", (1, 1, 2, source_name) @@ -671,15 +740,8 @@ def plan(self, manifest_path, offline=False): "name": package["name"], "version": package["version"], "selected": candidates[0] if candidates else None, "alternatives": candidates[1:], + "rejections": rejections, }) - try: - python = str(self.config.get_python_path(data["environment"]["python"])) - python_action = {"method": "reuse-python", "location": python, "status": "ready"} - except PythonNotFoundError: - python_action = { - "method": "install-python", "location": data["environment"]["python"], - "status": "network" if network_allowed else "unavailable", - } ready = (python_action["status"] != "unavailable" and all(item["selected"] for item in plans)) return {"path": str(path), "name": data["name"], "python": python_action, @@ -709,8 +771,13 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): plan = self.plan(path, offline=offline) unavailable = [item for item in plan["packages"] if not item["selected"]] if unavailable: - names = ", ".join(f"{item['name']}=={item['version']}" for item in unavailable) - raise PloadError(f"no valid reproduction route for: {names}") + details = [] + for item in unavailable: + label = f"{item['name']}=={item['version']}" + if item.get("rejections"): + label += f" ({item['rejections'][0]['reason']})" + details.append(label) + raise PloadError("no valid reproduction route for: " + ", ".join(details)) if plan["python"]["method"] == "install-python": if plan["python"]["status"] == "unavailable": raise PloadError("required Python is unavailable while offline") diff --git a/tests/test_declarative.py b/tests/test_declarative.py index 89365ca..21e9df0 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -5,7 +5,9 @@ from pathlib import Path import pytest +from packaging.tags import platform_tags +import pload.declarative as declarative_module from pload.cli import build_parser, main, shell_script from pload.declarative import ( DeclarativeEnvironmentManager, @@ -20,9 +22,9 @@ from pload.snapshots import digest, execute -def tiny_wheel(directory): +def tiny_wheel(directory, tag="py3-none-any"): directory.mkdir(parents=True, exist_ok=True) - path = directory / "pload_demo-1.0-py3-none-any.whl" + path = directory / f"pload_demo-1.0-{tag}.whl" with zipfile.ZipFile(path, "w") as wheel: wheel.writestr("pload_demo.py", "answer = 42\n") wheel.writestr( @@ -31,7 +33,7 @@ def tiny_wheel(directory): ) wheel.writestr( "pload_demo-1.0.dist-info/WHEEL", - "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + f"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: {tag}\n", ) wheel.writestr("pload_demo-1.0.dist-info/RECORD", "") return path @@ -120,6 +122,103 @@ def test_exact_configuration_requires_an_artifact_route(tmp_path): assert manager.plan(path)["ready"] is False +def test_changed_python_rejects_incompatible_cached_wheel_before_apply(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + incompatible_tag = f"cp37-cp37m-{next(iter(platform_tags()))}" + wheel = tiny_wheel(tmp_path / "fixture", incompatible_tag) + data = simple_manifest("exact") + data["name"] = "changed-python" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": [incompatible_tag], "repositories": [], + }], + }] + manifest = tmp_path / "pload.toml" + manifest.write_text(dump_manifest(data), encoding="utf-8") + manager = DeclarativeEnvironmentManager(config) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + + plan = manager.plan(manifest) + package = plan["packages"][0] + assert package["selected"] is None + assert package["rejections"][0]["artifact"] == wheel.name + assert "incompatible with" in package["rejections"][0]["reason"] + assert plan["ready"] is False + + monkeypatch.setattr( + VenvManager, "create_venv", + lambda *args, **kwargs: pytest.fail("apply created an environment for an invalid plan"), + ) + monkeypatch.setattr( + manager, "_acquire", + lambda *args, **kwargs: pytest.fail("apply downloaded an incompatible artifact"), + ) + with pytest.raises(PloadError, match="incompatible with"): + manager.apply(manifest) + + data["policy"]["reproducibility"] = "compatible" + manifest.write_text(dump_manifest(data), encoding="utf-8") + compatible = manager.plan(manifest)["packages"][0] + assert compatible["selected"]["method"] == "index-resolve" + assert compatible["selected"]["status"] == "network" + + +def test_cache_plan_applies_without_package_network_access(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + data = simple_manifest("exact") + data["name"] = "cache-only" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + data["policy"]["publish_missing_artifacts"] = False + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + }], + }] + manifest = tmp_path / "pload.toml" + manifest.write_text(dump_manifest(data), encoding="utf-8") + manager = DeclarativeEnvironmentManager(config) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + assert manager.plan(manifest)["packages"][0]["selected"]["method"] == "cache" + + original_execute = declarative_module.execute + + def reject_package_network(command, *args, **kwargs): + if "download" in command: + pytest.fail("a cache plan attempted a package download") + if "install" in command and "--no-index" not in command: + pytest.fail("a cache plan allowed pip index access") + return original_execute(command, *args, **kwargs) + + monkeypatch.setattr(declarative_module, "execute", reject_package_network) + restored = manager.apply(manifest) + output = original_execute([ + config.get_pip_command(restored)[0], "-c", "import pload_demo; print(pload_demo.answer)", + ]) + assert output == "42" + + def test_describe_plan_apply_through_content_repository(tmp_path): source_config = ConfigManager(home=tmp_path / "source-home") repository = tmp_path / "artifact-store" From 92dcf8347370a7c4be36bb5399ffc84b92aed521 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 15:44:47 +0800 Subject: [PATCH 13/41] Resolve and lock unpinned environment dependencies --- CHANGELOG.md | 5 +- README.md | 2 +- docs/declarative-environments.md | 23 ++++- pyproject.toml | 2 +- src/pload/__init__.py | 2 +- src/pload/cli.py | 6 ++ src/pload/declarative.py | 166 ++++++++++++++++++++++++++++--- src/pload/help_content.py | 4 + tests/test_declarative.py | 58 ++++++++++- 9 files changed, 248 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c4811a..bfb6f70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -Pre-release: `1.1.0a5` +Pre-release: `1.1.0a6` ### Added @@ -22,6 +22,9 @@ Pre-release: `1.1.0a5` - Validate locked wheel Python, ABI and platform tags against the requested runtime during planning; incompatible cache entries can no longer be reported as ready, and compatible mode explicitly plans network re-resolution instead. +- Accept unpinned dependency intentions such as `numpy` and `pandas>=2`; the + first online plan resolves the complete wheel dependency closure, records exact + versions, hashes and tags in the TOML file, and subsequent plans reuse the lock. ### Removed diff --git a/README.md b/README.md index 9d5b48b..56adf3e 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ pload-install --yes --package-spec "pload==1.0.0" To opt into the declarative-environment pre-release for testing: ```console -pload-install --yes --package-spec "pload==1.1.0a5" +pload-install --yes --package-spec "pload==1.1.0a6" ``` To always follow the newest published release instead of pinning a version, diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index 4cf7b44..e2ef228 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -92,13 +92,27 @@ fields are reserved for future schema versions and should not be relied upon. | `implementation` | string | `"cpython"` | `sys.implementation.name`, normally `cpython` or `pypy`. `apply` rejects a different implementation. | | `system` | string | empty | Target `platform.system()`, such as `Linux`, `Darwin` or `Windows`. A non-empty value is a hard compatibility constraint. | | `machine` | string | empty | Target `platform.machine()`, such as `x86_64`, `AMD64`, `arm64` or `aarch64`. A non-empty value is a hard compatibility constraint. | -| `dependencies` | array of strings | `[]` | Complete desired application package set. Every entry must be an exact `NAME==VERSION` pin. When `[[package]]` lock entries exist, the two sections must describe the same normalized names and versions. pip, setuptools and wheel bootstrap packages are intentionally omitted by `describe`. | +| `dependencies` | array of strings | `[]` | Desired packages. Entries may be names (`numpy`), version constraints (`pandas>=2,<3`) or exact pins (`numpy==2.1.3`). On the first online plan, non-exact requirements are resolved for the requested Python, the complete dependency closure replaces this list as exact pins, and matching `[[package]]` artifact locks are written. Direct URLs and environment markers are intentionally rejected in schema 1. pip, setuptools and wheel bootstrap packages are omitted by `describe`. | The environment section is desired state, not a command sequence. Paths to a source virtual environment are never stored here, so the configuration remains portable. The current schema describes one target platform and one exact Python version; use separate files for different targets until multi-target locks exist. +For example, this is a valid initial request: + +```toml +dependencies = ["numpy", "pandas>=2,<3"] +``` + +`pload plan` asks pip under the requested interpreter to resolve these requirements +and their transitive dependencies as compatible wheels. It then atomically rewrites +the same file with exact `NAME==VERSION` dependencies plus wheel filenames, +SHA-256 hashes and tags. Resolution uses all declared indexes, with `default` as +the primary index. It requires network access once; `--offline` rejects an +unlocked file rather than guessing. A second plan is read-only and does not run +the resolver again. + ### `[capabilities]`: observed non-Python context Each capability is an array of strings. `describe` currently recognizes @@ -299,6 +313,13 @@ pload plan --offline pload plan --json ``` +If the dependency list contains unpinned requirements and no existing package +lock, the first online `plan` performs a lock step before resource selection. Its +output reports that `pload.toml` was updated. The generated lock contains the +complete wheel dependency closure, so `apply` does not ask pip to resolve +dependencies again. Schema 1 currently requires wheel availability during this +automatic lock step; it does not create a portable exact lock from an sdist. + The planner inventories: 1. installed and discoverable Python interpreters; diff --git a/pyproject.toml b/pyproject.toml index e95f322..f6018a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pload" -version = "1.1.0a5" +version = "1.1.0a6" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" diff --git a/src/pload/__init__.py b/src/pload/__init__.py index 711f574..5553f80 100644 --- a/src/pload/__init__.py +++ b/src/pload/__init__.py @@ -1 +1 @@ -__version__ = "1.1.0a5" +__version__ = "1.1.0a6" diff --git a/src/pload/cli.py b/src/pload/cli.py index 7fb4166..5802886 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -445,6 +445,12 @@ def _global_options_table(root_parser): def print_declarative_plan(plan): console = Console(highlight=False) + if plan.get("lock"): + count = len(plan["lock"]["resolved"]) + console.print( + f"[bold green]✓ Locked {count} packages[/] and updated " + f"[cyan]{plan['path']}[/]" + ) python = plan["python"] console.print(Panel.fit( f"[bold]{plan['name']}[/]\nPython: [cyan]{python['method']}[/] · " diff --git a/src/pload/declarative.py b/src/pload/declarative.py index c0b0cac..685ba00 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -12,6 +12,7 @@ from pathlib import Path from urllib.parse import urlsplit +from packaging.requirements import InvalidRequirement, Requirement from packaging.tags import compatible_tags, cpython_tags, platform_tags from packaging.utils import InvalidWheelFilename, parse_wheel_filename @@ -35,6 +36,20 @@ def normalized_name(value): return re.sub(r"[-_.]+", "-", value).lower() +def _parse_dependency(value): + if not isinstance(value, str) or not value.strip(): + raise PloadError("environment.dependencies must contain package requirements") + try: + requirement = Requirement(value) + except InvalidRequirement as exc: + raise PloadError(f"invalid dependency requirement: {value}") from exc + if requirement.url: + raise PloadError("direct-URL dependencies are not portable; declare an index source") + if requirement.marker: + raise PloadError("dependency environment markers are not supported in schema 1") + return requirement + + def _normalized_machine(value): aliases = { "amd64": "x86_64", "x64": "x86_64", @@ -209,8 +224,12 @@ def validate_manifest(data): or any(not isinstance(name, str) for name in policy_repositories)): raise PloadError("policy.repositories must be an array of repository names") dependencies = environment.get("dependencies", []) - if not isinstance(dependencies, list) or any(not PIN.fullmatch(item) for item in dependencies): - raise PloadError("environment.dependencies must contain exact NAME==VERSION pins") + if not isinstance(dependencies, list): + raise PloadError("environment.dependencies must be an array of package requirements") + requirements = [_parse_dependency(item) for item in dependencies] + dependency_names = [normalized_name(item.name) for item in requirements] + if len(dependency_names) != len(set(dependency_names)): + raise PloadError("environment.dependencies contains duplicate package requirements") seen = set() for name, source in data.get("sources", {}).items(): safe_name(name) @@ -253,18 +272,27 @@ def validate_manifest(data): f"{repository_name}" ) if data.get("package"): - declared_pins = { - (normalized_name(match.group(1)), match.group(2)) - for item in dependencies if (match := PIN.fullmatch(item)) - } - locked_pins = { - (normalized_name(package["name"]), package["version"]) + locked = { + normalized_name(package["name"]): package["version"] for package in data["package"] } - if declared_pins != locked_pins: - raise PloadError( - "environment.dependencies and [[package]] locks must describe the same pins" - ) + if all(PIN.fullmatch(item) for item in dependencies): + declared_pins = { + (normalized_name(match.group(1)), match.group(2)) + for item in dependencies if (match := PIN.fullmatch(item)) + } + if declared_pins != set(locked.items()): + raise PloadError( + "environment.dependencies and [[package]] locks must describe the same pins" + ) + else: + for requirement in requirements: + version = locked.get(normalized_name(requirement.name)) + if version is None or (requirement.specifier + and not requirement.specifier.contains(version)): + raise PloadError( + f"locked packages do not satisfy dependency requirement: {requirement}" + ) for name, repository in data.get("repositories", {}).items(): safe_name(name) kind = repository.get("kind") @@ -649,8 +677,118 @@ def _obtain_wheel(self, interpreter, installed, wheel_dir, index): raise PloadError(f"download did not produce one exact wheel for {installed['name']}") return matches[0] + def _lock_requested_dependencies(self, path, data, offline=False): + requested = list(data["environment"].get("dependencies", [])) + if not requested or all(PIN.fullmatch(item) for item in requested): + return None + + if data.get("package"): + resolved = [ + f"{package['name']}=={package['version']}" + for package in sorted( + data["package"], key=lambda item: normalized_name(item["name"]) + ) + ] + data["environment"]["dependencies"] = resolved + validate_manifest(data) + self._write_manifest(path, data) + return {"updated": True, "requested": requested, "resolved": resolved} + + network_allowed = ( + data["policy"].get("network", "allow") == "allow" and not offline + ) + if not network_allowed: + raise PloadError( + "unlocked dependencies require one online resolution before offline planning" + ) + try: + interpreter = self.config.get_python_path(data["environment"]["python"]) + except PythonNotFoundError: + PythonManager(self.config).install_python(data["environment"]["python"]) + interpreter = self.config.get_python_path(data["environment"]["python"]) + self._check_runtime(interpreter, data["environment"]) + + sources = data.get("sources", {}) + if not sources: + index = public_index( + self.config.settings.get("pip_index") or "https://pypi.org/simple" + ) + data["sources"] = {"default": {"kind": "index", "url": index}} + sources = data["sources"] + source_names = sorted(sources, key=lambda name: (name != "default", name)) + primary = sources[source_names[0]]["url"] + + self.cache.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="pload-lock-") as temporary: + destination = Path(temporary) + command = [ + str(interpreter), "-m", "pip", "download", + "--disable-pip-version-check", "--only-binary=:all:", + "--dest", str(destination), "--index-url", primary, + ] + for source_name in source_names[1:]: + command += ["--extra-index-url", sources[source_name]["url"]] + try: + execute(command + requested) + except PloadError as exc: + raise PloadError( + "cannot resolve unlocked dependencies to compatible wheels for " + f"Python {data['environment']['python']}" + ) from exc + + wheels = sorted(destination.glob("*.whl"), key=lambda item: item.name.lower()) + if not wheels: + raise PloadError("dependency resolution did not produce any wheels") + records = [] + for wheel in wheels: + try: + package_name, package_version, _, wheel_tags = parse_wheel_filename( + wheel.name + ) + except InvalidWheelFilename as exc: + raise PloadError(f"resolver produced an invalid wheel: {wheel.name}") from exc + checksum = digest(wheel) + cached = self.cache / wheel.name + if cached.is_file() and digest(cached) != checksum: + raise PloadError(f"conflicting cached artifact: {wheel.name}") + if not cached.exists(): + shutil.copyfile(wheel, cached) + records.append({ + "name": str(package_name), + "version": str(package_version), + "sources": source_names, + "artifact": [{ + "filename": wheel.name, + "sha256": checksum, + "tags": sorted(str(tag) for tag in wheel_tags), + "repositories": [], + }], + }) + + locked = {normalized_name(item["name"]): item["version"] for item in records} + for value in requested: + requirement = _parse_dependency(value) + version = locked.get(normalized_name(requirement.name)) + if version is None or (requirement.specifier + and not requirement.specifier.contains(version)): + raise PloadError(f"resolver did not satisfy dependency requirement: {value}") + records.sort(key=lambda item: normalized_name(item["name"])) + resolved = [f"{item['name']}=={item['version']}" for item in records] + data["package"] = records + data["environment"]["dependencies"] = resolved + validate_manifest(data) + self._write_manifest(path, data) + return {"updated": True, "requested": requested, "resolved": resolved} + + @staticmethod + def _write_manifest(path, data): + temporary = path.with_name(path.name + ".tmp") + temporary.write_text(dump_manifest(data), encoding="utf-8") + temporary.replace(path) + def plan(self, manifest_path, offline=False): path, data = load_manifest(manifest_path) + lock = self._lock_requested_dependencies(path, data, offline=offline) policy = data["policy"] network_allowed = policy.get("network", "allow") == "allow" and not offline repositories = data.get("repositories", {}) @@ -745,7 +883,7 @@ def plan(self, manifest_path, offline=False): ready = (python_action["status"] != "unavailable" and all(item["selected"] for item in plans)) return {"path": str(path), "name": data["name"], "python": python_action, - "packages": plans, "ready": ready} + "packages": plans, "ready": ready, "lock": lock} @staticmethod def _candidate(method, location, status, cost, artifact=None): @@ -769,6 +907,8 @@ def _locked_packages(data): def apply(self, manifest_path, name=None, offline=False, progress=None): path, data = load_manifest(manifest_path) plan = self.plan(path, offline=offline) + if plan.get("lock"): + path, data = load_manifest(path) unavailable = [item for item in plan["packages"] if not item["selected"]] if unavailable: details = [] diff --git a/src/pload/help_content.py b/src/pload/help_content.py index ffaa1f4..c58c5d6 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -379,6 +379,10 @@ caches, adjacent artifacts, content-addressed repositories and declared indexes. Incompatible resources are rejected first; valid routes are ordered by exactness, compatibility risk, transfer, execution cost and a deterministic tie-breaker. +Dependencies may initially be unpinned (`numpy`, `pandas>=2`). The first online +plan resolves their complete wheel dependency closure, writes exact versions and +artifact hashes back to the TOML file, then plans from that lock. Later plans do +not resolve again; `--offline` requires the lock to exist already. Planning is optional transparency: normal users can go directly to `pload apply`. """ GUIDES[("apply",)] = r""" diff --git a/tests/test_declarative.py b/tests/test_declarative.py index 21e9df0..8181ea2 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -71,10 +71,12 @@ def test_manifest_roundtrip(tmp_path): assert loaded == data -def test_manifest_requires_exact_pins(): +def test_manifest_accepts_unpinned_requirements_but_rejects_direct_urls(): data = simple_manifest() data["environment"]["dependencies"] = ["demo>=1"] - with pytest.raises(PloadError, match="exact"): + validate_manifest(data) + data["environment"]["dependencies"] = ["demo @ https://example.com/demo.whl"] + with pytest.raises(PloadError, match="direct-URL"): validate_manifest(data) @@ -219,6 +221,58 @@ def reject_package_network(command, *args, **kwargs): assert output == "42" +def test_plan_locks_unpinned_dependencies_once_then_applies_from_cache(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + data = simple_manifest("exact") + data["name"] = "auto-locked" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo>=1"], + }) + data["policy"]["publish_missing_artifacts"] = False + data["package"] = [] + manifest = tmp_path / "pload.toml" + manifest.write_text(dump_manifest(data), encoding="utf-8") + manager = DeclarativeEnvironmentManager(config) + original_execute = declarative_module.execute + download_calls = [] + + def fake_resolver(command, *args, **kwargs): + if "download" in command: + download_calls.append(command) + assert "--no-deps" not in command + destination = Path(command[command.index("--dest") + 1]) + shutil.copyfile(wheel, destination / wheel.name) + return "" + if "install" in command: + assert "--no-index" in command + return original_execute(command, *args, **kwargs) + + monkeypatch.setattr(declarative_module, "execute", fake_resolver) + first = manager.plan(manifest) + assert first["lock"]["requested"] == ["pload-demo>=1"] + assert first["lock"]["resolved"] == ["pload-demo==1.0"] + assert first["packages"][0]["selected"]["method"] == "cache" + _, locked = load_manifest(manifest) + assert locked["environment"]["dependencies"] == ["pload-demo==1.0"] + assert locked["package"][0]["artifact"][0]["filename"] == wheel.name + + second = manager.plan(manifest) + assert second["lock"] is None + assert len(download_calls) == 1 + restored = manager.apply(manifest) + assert len(download_calls) == 1 + output = original_execute([ + config.get_pip_command(restored)[0], "-c", "import pload_demo; print(pload_demo.answer)", + ]) + assert output == "42" + + def test_describe_plan_apply_through_content_repository(tmp_path): source_config = ConfigManager(home=tmp_path / "source-home") repository = tmp_path / "artifact-store" From 6ea1a517429fa3f3e004419d4905018a77cd1186 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 15:56:05 +0800 Subject: [PATCH 14/41] Reconcile stale locks after dependency edits --- CHANGELOG.md | 5 ++- README.md | 2 +- docs/declarative-environments.md | 9 +++++ pyproject.toml | 2 +- src/pload/__init__.py | 2 +- src/pload/declarative.py | 63 +++++++++++++++++++------------- tests/test_declarative.py | 60 +++++++++++++++++++++++++----- 7 files changed, 103 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfb6f70..f994d9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -Pre-release: `1.1.0a6` +Pre-release: `1.1.0a7` ### Added @@ -25,6 +25,9 @@ Pre-release: `1.1.0a6` - Accept unpinned dependency intentions such as `numpy` and `pandas>=2`; the first online plan resolves the complete wheel dependency closure, records exact versions, hashes and tags in the TOML file, and subsequent plans reuse the lock. +- Treat edits that add or change dependencies beside an existing `[[package]]` + lock as a stale lock to reconcile, not an invalid configuration; re-resolution + uses the complete requested set and prefers existing pload cache wheels. ### Removed diff --git a/README.md b/README.md index 56adf3e..07ceb07 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ pload-install --yes --package-spec "pload==1.0.0" To opt into the declarative-environment pre-release for testing: ```console -pload-install --yes --package-spec "pload==1.1.0a6" +pload-install --yes --package-spec "pload==1.1.0a7" ``` To always follow the newest published release instead of pinning a version, diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index e2ef228..693c554 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -113,6 +113,15 @@ the primary index. It requires network access once; `--offline` rejects an unlocked file rather than guessing. A second plan is read-only and does not run the resolver again. +Editing dependencies after a lock exists is also supported. For example, adding +`"torch"` to a configuration whose `[[package]]` entries describe an older +environment marks that lock as stale. The next online plan resolves the complete +requested set—including existing exact constraints—rather than rejecting the +file or silently dropping the new package. The resolver receives the pload wheel +cache as a local candidate source, so compatible cached wheels are preferred and +only missing artifacts need index access. The reconciled complete lock replaces +the old one atomically. + ### `[capabilities]`: observed non-Python context Each capability is an array of strings. `describe` currently recognizes diff --git a/pyproject.toml b/pyproject.toml index f6018a6..ac0d17a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pload" -version = "1.1.0a6" +version = "1.1.0a7" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" diff --git a/src/pload/__init__.py b/src/pload/__init__.py index 5553f80..3debd0e 100644 --- a/src/pload/__init__.py +++ b/src/pload/__init__.py @@ -1 +1 @@ -__version__ = "1.1.0a6" +__version__ = "1.1.0a7" diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 685ba00..4b620f3 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -271,28 +271,9 @@ def validate_manifest(data): f"artifact {artifact['filename']} references unknown repository " f"{repository_name}" ) - if data.get("package"): - locked = { - normalized_name(package["name"]): package["version"] - for package in data["package"] - } - if all(PIN.fullmatch(item) for item in dependencies): - declared_pins = { - (normalized_name(match.group(1)), match.group(2)) - for item in dependencies if (match := PIN.fullmatch(item)) - } - if declared_pins != set(locked.items()): - raise PloadError( - "environment.dependencies and [[package]] locks must describe the same pins" - ) - else: - for requirement in requirements: - version = locked.get(normalized_name(requirement.name)) - if version is None or (requirement.specifier - and not requirement.specifier.contains(version)): - raise PloadError( - f"locked packages do not satisfy dependency requirement: {requirement}" - ) + # A dependency edit can intentionally make the existing package lock stale. + # Planning reconciles and rewrites it; validation only checks each section's + # shape and internal safety. for name, repository in data.get("repositories", {}).items(): safe_name(name) kind = repository.get("kind") @@ -679,10 +660,29 @@ def _obtain_wheel(self, interpreter, installed, wheel_dir, index): def _lock_requested_dependencies(self, path, data, offline=False): requested = list(data["environment"].get("dependencies", [])) - if not requested or all(PIN.fullmatch(item) for item in requested): + if not requested: return None + requirements = [_parse_dependency(value) for value in requested] + all_exact = all(PIN.fullmatch(item) for item in requested) + locked = { + normalized_name(package["name"]): package["version"] + for package in data.get("package", []) + } + satisfies_requests = all( + normalized_name(requirement.name) in locked + and (not requirement.specifier or requirement.specifier.contains( + locked[normalized_name(requirement.name)] + )) + for requirement in requirements + ) + exact_lock_matches = not all_exact or { + (normalized_name(match.group(1)), match.group(2)) + for item in requested if (match := PIN.fullmatch(item)) + } == set(locked.items()) - if data.get("package"): + if data.get("package") and satisfies_requests and exact_lock_matches: + if all_exact: + return None resolved = [ f"{package['name']}=={package['version']}" for package in sorted( @@ -694,6 +694,9 @@ def _lock_requested_dependencies(self, path, data, offline=False): self._write_manifest(path, data) return {"updated": True, "requested": requested, "resolved": resolved} + if all_exact and not data.get("package"): + return None + network_allowed = ( data["policy"].get("network", "allow") == "allow" and not offline ) @@ -719,12 +722,18 @@ def _lock_requested_dependencies(self, path, data, offline=False): primary = sources[source_names[0]]["url"] self.cache.mkdir(parents=True, exist_ok=True) + known_repositories = { + (artifact["filename"], artifact["sha256"]): artifact.get("repositories", []) + for package in data.get("package", []) + for artifact in package.get("artifact", []) + } with tempfile.TemporaryDirectory(prefix="pload-lock-") as temporary: destination = Path(temporary) command = [ str(interpreter), "-m", "pip", "download", "--disable-pip-version-check", "--only-binary=:all:", - "--dest", str(destination), "--index-url", primary, + "--dest", str(destination), "--find-links", str(self.cache), + "--index-url", primary, ] for source_name in source_names[1:]: command += ["--extra-index-url", sources[source_name]["url"]] @@ -761,7 +770,9 @@ def _lock_requested_dependencies(self, path, data, offline=False): "filename": wheel.name, "sha256": checksum, "tags": sorted(str(tag) for tag in wheel_tags), - "repositories": [], + "repositories": known_repositories.get( + (wheel.name, checksum), [] + ), }], }) diff --git a/tests/test_declarative.py b/tests/test_declarative.py index 8181ea2..a28dff2 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -22,20 +22,22 @@ from pload.snapshots import digest, execute -def tiny_wheel(directory, tag="py3-none-any"): +def tiny_wheel(directory, tag="py3-none-any", distribution="pload_demo", version="1.0"): directory.mkdir(parents=True, exist_ok=True) - path = directory / f"pload_demo-1.0-{tag}.whl" + path = directory / f"{distribution}-{version}-{tag}.whl" + module = distribution.replace("-", "_") + project = distribution.replace("_", "-") with zipfile.ZipFile(path, "w") as wheel: - wheel.writestr("pload_demo.py", "answer = 42\n") + wheel.writestr(f"{module}.py", "answer = 42\n") wheel.writestr( - "pload_demo-1.0.dist-info/METADATA", - "Metadata-Version: 2.1\nName: pload-demo\nVersion: 1.0\n", + f"{distribution}-{version}.dist-info/METADATA", + f"Metadata-Version: 2.1\nName: {project}\nVersion: {version}\n", ) wheel.writestr( - "pload_demo-1.0.dist-info/WHEEL", + f"{distribution}-{version}.dist-info/WHEEL", f"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: {tag}\n", ) - wheel.writestr("pload_demo-1.0.dist-info/RECORD", "") + wheel.writestr(f"{distribution}-{version}.dist-info/RECORD", "") return path @@ -94,11 +96,10 @@ def test_manifest_rejects_dangling_provider_references(): validate_manifest(data) -def test_manifest_rejects_mismatched_dependency_and_package_locks(): +def test_manifest_allows_dependency_edits_to_make_the_package_lock_stale(): data = simple_manifest() data["environment"]["dependencies"] = ["other==1.0"] - with pytest.raises(PloadError, match="same pins"): - validate_manifest(data) + validate_manifest(data) def test_manifest_rejects_unknown_policy_repository(): @@ -273,6 +274,45 @@ def fake_resolver(command, *args, **kwargs): assert output == "42" +def test_plan_relocks_when_new_dependency_makes_existing_lock_stale(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + demo = tiny_wheel(tmp_path / "fixture") + extra = tiny_wheel(tmp_path / "fixture", distribution="extra_demo") + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0", "extra-demo"], + }) + manifest = tmp_path / "pload.toml" + manifest.write_text(dump_manifest(data), encoding="utf-8") + manager = DeclarativeEnvironmentManager(config) + calls = [] + + def fake_resolver(command, *args, **kwargs): + if "download" not in command: + return "" + calls.append(command) + assert "--find-links" in command + assert "pload-demo==1.0" in command + assert "extra-demo" in command + destination = Path(command[command.index("--dest") + 1]) + shutil.copyfile(demo, destination / demo.name) + shutil.copyfile(extra, destination / extra.name) + return "" + + monkeypatch.setattr(declarative_module, "execute", fake_resolver) + plan = manager.plan(manifest) + assert len(calls) == 1 + assert plan["lock"]["requested"] == ["pload-demo==1.0", "extra-demo"] + assert plan["lock"]["resolved"] == ["extra-demo==1.0", "pload-demo==1.0"] + _, locked = load_manifest(manifest) + assert {item["name"] for item in locked["package"]} == {"extra-demo", "pload-demo"} + + def test_describe_plan_apply_through_content_repository(tmp_path): source_config = ConfigManager(home=tmp_path / "source-home") repository = tmp_path / "artifact-store" From bd72253dfb939175c3312cb3252b56c195f71f47 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 16:08:44 +0800 Subject: [PATCH 15/41] Separate managed locks from user configuration --- CHANGELOG.md | 16 ++-- README.md | 7 +- docs/declarative-environments.md | 132 +++++++++++++++++++----------- pyproject.toml | 2 +- src/pload/__init__.py | 2 +- src/pload/cli.py | 5 +- src/pload/declarative.py | 133 ++++++++++++++++++++++++------- tests/test_declarative.py | 86 ++++++++++++++++---- 8 files changed, 281 insertions(+), 102 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f994d9e..2fa6c7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,12 @@ ## Unreleased -Pre-release: `1.1.0a7` +Pre-release: `1.1.0a8` ### Added -- Make one portable `pload.toml` the source of truth for desired Python, - exact dependencies, policies, repositories and locked artifact hashes. +- Keep user intent in a clean portable `pload.toml` and store pload-managed + exact dependency, artifact and hash decisions in sibling `.pload_lock.toml`. - Add `pload describe`, `pload plan` and idempotent `pload apply`; resource copying, downloading and publication are internal planner/executor steps. - Inventory the local wheel cache, adjacent artifacts, local/SSH content stores, @@ -16,7 +16,7 @@ Pre-release: `1.1.0a7` while keeping credentials out of the portable configuration. - Reuse exact wheel archives by SHA-256, deduplicate local/SSH storage, record detected NVIDIA driver provenance and roll back failed materializations. -- Add a complete field-by-field `pload.toml` reference, enforce agreement between +- Add complete field-by-field configuration and lock references, enforce agreement between desired dependency pins and package locks, and make policy repositories and `publish_missing_artifacts` drive lookup and automatic cache publication. - Validate locked wheel Python, ABI and platform tags against the requested @@ -24,10 +24,12 @@ Pre-release: `1.1.0a7` as ready, and compatible mode explicitly plans network re-resolution instead. - Accept unpinned dependency intentions such as `numpy` and `pandas>=2`; the first online plan resolves the complete wheel dependency closure, records exact - versions, hashes and tags in the TOML file, and subsequent plans reuse the lock. -- Treat edits that add or change dependencies beside an existing `[[package]]` - lock as a stale lock to reconcile, not an invalid configuration; re-resolution + versions, hashes and tags in the managed lock, and subsequent plans reuse it. +- Treat edits that add or change dependencies beside an existing lock as a stale + lock to reconcile, not an invalid configuration; re-resolution uses the complete requested set and prefers existing pload cache wheels. +- Migrate lock tables embedded by earlier 1.1 previews into `.pload_lock.toml` + without another dependency resolution or artifact download. ### Removed diff --git a/README.md b/README.md index 07ceb07..5a0c817 100644 --- a/README.md +++ b/README.md @@ -78,8 +78,9 @@ profile when `--shell` is supplied or already configured. ## Reproduce and share environments -The `1.1` pre-release uses one `pload.toml` as the portable source of truth. -The user declares the desired environment; pload inventories local caches, +The `1.1` pre-release uses a clean, user-authored `pload.toml` as the portable +source of truth and writes exact resolver output to the sibling +`.pload_lock.toml`. The user declares the desired environment; pload inventories local caches, configured local/SSH content stores, package indexes and Python installations, then performs transfers and installation internally: @@ -113,7 +114,7 @@ pload-install --yes --package-spec "pload==1.0.0" To opt into the declarative-environment pre-release for testing: ```console -pload-install --yes --package-spec "pload==1.1.0a7" +pload-install --yes --package-spec "pload==1.1.0a8" ``` To always follow the newest published release instead of pinning a version, diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index 693c554..a5b01bc 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -1,30 +1,24 @@ # Declarative environments The `1.1` pre-release replaces the transport-oriented snapshot experiment with -one portable environment configuration. Users describe the desired state; pload -discovers resources, chooses a deterministic plan and materializes that state. +a small portable configuration plus a pload-managed lock. Users describe the +desired state; pload discovers resources, chooses a deterministic plan and +materializes that state. ```console pload describe v2 -o pload.toml pload apply pload.toml ``` -`describe` is the bridge for an existing environment. For a new project, the -same file may be written by hand with exact `NAME==VERSION` dependencies and a -`compatible` policy, then locked more strictly after it has been materialized. +`describe` is the bridge for an existing environment. For a new project, +`pload.toml` may be written by hand with names, ranges or exact +`NAME==VERSION` requirements; `plan` produces the managed lock. -## One file, two responsibilities +## Two files, two responsibilities -`pload.toml` contains both: - -- **specification** — the Python, dependencies, target platform and policy the - user wants; -- **lock** — exact package versions, wheel identities, SHA-256 hashes, compatible - tags and logical resource locations discovered by pload. - -The file is the control plane. Python runtimes and wheel bytes remain in caches, -indexes or content-addressed repositories and are the data plane. Credentials are -never stored in the file; SSH uses the existing SSH agent/configuration. +`pload.toml` is the file a user reads and edits. It contains only the desired +Python, direct dependency requirements, target platform, policies and resource +providers. For example: ```toml schema = 1 @@ -35,7 +29,7 @@ python = "3.12.7" implementation = "cpython" system = "Linux" machine = "x86_64" -dependencies = ["numpy==2.1.3", "torch==2.5.0+cu121"] +dependencies = ["numpy>=2,<3", "torch"] [capabilities] nvidia_driver = ["550.90.07"] @@ -54,6 +48,17 @@ url = "https://pypi.org/simple" [repositories.lab] kind = "ssh" location = "frpxiaoxin:/home/tibless/pload-cloud" +``` + +`pload plan` writes its decisions to `.pload_lock.toml` in the same directory: + +```toml +schema = 1 +configuration = "pload.toml" +configuration_sha256 = "..." + +[environment] +dependencies = ["numpy==2.1.3", "torch==2.5.0+cu121", "..."] [[package]] name = "torch" @@ -67,15 +72,34 @@ tags = ["cp312-cp312-linux_x86_64"] repositories = ["lab"] ``` -`describe` writes the lock entries automatically. URLs containing embedded +The lock is generated data, not a second user configuration. It records the +complete transitive dependency closure, exact versions, wheel identities, +SHA-256 hashes, compatibility tags and known artifact locations. The +`configuration_sha256` binds it to the current `pload.toml`; editing the user +configuration makes the old lock stale and the next online plan replaces it +atomically. Commit both files when reproducibility matters, but normally edit +only `pload.toml`. + +The lock is necessary for **exact** reproduction because a requirement such as +`torch` or even `torch==2.8.0` does not uniquely identify all transitive versions +or a particular wheel build. Package indexes change over time. The filename and +SHA-256 in the lock let pload prove that a cached, cloud-hosted or downloaded +artifact is the same byte sequence chosen originally. Deleting the lock is safe: +the next online `plan` resolves a new one, but the result may differ. Offline +planning cannot regenerate a missing or stale lock. + +Python runtimes and wheel bytes remain in caches, indexes or content-addressed +repositories; the two TOML files contain only control information. Credentials +are never stored in either file; SSH uses the existing SSH agent/configuration. +`describe` writes both files automatically. URLs containing embedded credentials or query-string tokens are rejected; authentication belongs to the machine, not a shareable environment description. ## Complete `pload.toml` field reference -The file has seven logical parts. `describe` normally writes all lock-related -fields, but the same schema can be reviewed or authored by hand. Unknown TOML -fields are reserved for future schema versions and should not be relied upon. +`pload.toml` has five user-facing logical parts. Unknown TOML fields are reserved +for future schema versions and should not be relied upon. The generated lock +format is documented separately below so it can be audited, not hand-maintained. ### Top-level identity @@ -92,7 +116,7 @@ fields are reserved for future schema versions and should not be relied upon. | `implementation` | string | `"cpython"` | `sys.implementation.name`, normally `cpython` or `pypy`. `apply` rejects a different implementation. | | `system` | string | empty | Target `platform.system()`, such as `Linux`, `Darwin` or `Windows`. A non-empty value is a hard compatibility constraint. | | `machine` | string | empty | Target `platform.machine()`, such as `x86_64`, `AMD64`, `arm64` or `aarch64`. A non-empty value is a hard compatibility constraint. | -| `dependencies` | array of strings | `[]` | Desired packages. Entries may be names (`numpy`), version constraints (`pandas>=2,<3`) or exact pins (`numpy==2.1.3`). On the first online plan, non-exact requirements are resolved for the requested Python, the complete dependency closure replaces this list as exact pins, and matching `[[package]]` artifact locks are written. Direct URLs and environment markers are intentionally rejected in schema 1. pip, setuptools and wheel bootstrap packages are omitted by `describe`. | +| `dependencies` | array of strings | `[]` | Direct package requirements owned by the user. Entries may be names (`numpy`), version constraints (`pandas>=2,<3`) or exact pins (`numpy==2.1.3`). `plan` never replaces this list with transitive dependencies; resolved versions belong to `.pload_lock.toml`. Direct URLs and environment markers are intentionally rejected in schema 1. pip, setuptools and wheel bootstrap packages are omitted by `describe`. | The environment section is desired state, not a command sequence. Paths to a source virtual environment are never stored here, so the configuration remains @@ -106,22 +130,30 @@ dependencies = ["numpy", "pandas>=2,<3"] ``` `pload plan` asks pip under the requested interpreter to resolve these requirements -and their transitive dependencies as compatible wheels. It then atomically rewrites -the same file with exact `NAME==VERSION` dependencies plus wheel filenames, -SHA-256 hashes and tags. Resolution uses all declared indexes, with `default` as -the primary index. It requires network access once; `--offline` rejects an -unlocked file rather than guessing. A second plan is read-only and does not run -the resolver again. +and their transitive dependencies as compatible wheels. It then atomically writes +exact `NAME==VERSION` results, wheel filenames, SHA-256 hashes and tags to +`.pload_lock.toml`; `pload.toml` remains unchanged. Resolution uses all declared +indexes, with `default` as the primary index. It requires network access once; +`--offline` rejects an unlocked file rather than guessing. A second plan is +read-only and does not run the resolver again. Editing dependencies after a lock exists is also supported. For example, adding -`"torch"` to a configuration whose `[[package]]` entries describe an older -environment marks that lock as stale. The next online plan resolves the complete +`"torch"` to a configuration with an existing `.pload_lock.toml` marks that lock +as stale. The next online plan resolves the complete requested set—including existing exact constraints—rather than rejecting the file or silently dropping the new package. The resolver receives the pload wheel cache as a local candidate source, so compatible cached wheels are preferred and only missing artifacts need index access. The reconciled complete lock replaces the old one atomically. +Configurations produced by pre-`1.1.0a8` previews may still contain embedded +`[[package]]` tables. The first `plan` migrates those tables to +`.pload_lock.toml` without downloading again and rewrites `pload.toml` in the +clean declaration-only format. Because older previews replaced the original +direct requirements with the complete closure, they cannot always recover which +packages the user originally typed; those exact requirements remain valid and +can be simplified manually afterward. + ### `[capabilities]`: observed non-Python context Each capability is an array of strings. `describe` currently recognizes @@ -213,7 +245,15 @@ The TOML control file may itself be versioned in Git or hosted on GitHub. Large wheel bytes should remain in these content-addressed local/SSH repositories rather than being committed to Git. -### `[[package]]`: one locked package +## Complete `.pload_lock.toml` field reference + +The lock's top-level `schema` is currently `1`. `configuration` names the sibling +user configuration and `configuration_sha256` identifies its canonical content. +`[environment].dependencies` contains the complete, sorted exact dependency +closure selected by the resolver. These fields let pload distinguish a reusable +lock from one made stale by editing or renaming the configuration. + +### `[[package]]`: one resolved package There is one array entry for every application dependency: @@ -224,9 +264,8 @@ There is one array entry for every application dependency: | `sources` | array of strings | no | Logical `[sources.NAME]` entries allowed to satisfy this package. Each referenced source must exist. `describe -s PACKAGE=URL` creates a package-specific source when installed metadata cannot reliably reveal it. | | `artifact` | array of tables | no | Exact wheel identities accepted for this package. Exact policy needs at least one reachable artifact route; compatible policy may omit artifacts and re-resolve the pinned version. | -If `[[package]]` is entirely omitted, pload synthesizes package records from -`environment.dependencies`. That form is useful only for compatible online -resolution because it contains no exact artifact identity. +Users should not add or remove these records manually. Delete `.pload_lock.toml` +and run `pload plan` when a complete re-resolution is wanted. ### `[[package.artifact]]`: exact wheel identity @@ -269,10 +308,11 @@ kind = "index" url = "https://pypi.org/simple" ``` -`implementation`, `system`, `machine`, `capabilities`, `repositories` and -`[[package]]` are filled or omitted according to the defaults described above. -For durable sharing, prefer `pload describe` so exact package and artifact lock -entries are generated and validated automatically. +`implementation`, `system`, `machine`, `capabilities` and `repositories` are +filled or omitted according to the defaults described above. `.pload_lock.toml` +is generated separately. For durable sharing, commit both files and prefer +`pload describe` so exact package and artifact identity is captured rather than +inferred. ## Describe an existing environment @@ -285,7 +325,8 @@ pload describe /project/.venv/bin/python -o project.pload.toml The default `exact` mode obtains one wheel for every application package, computes its SHA-256, places it in the shared cache and automatically publishes it to the first configured local/SSH artifact repository. The repository is -embedded in the resulting configuration as a logical resource provider. +declared in `pload.toml`; the artifact-to-repository association is recorded in +`.pload_lock.toml`. Use a specific configured repository when necessary: @@ -322,9 +363,9 @@ pload plan --offline pload plan --json ``` -If the dependency list contains unpinned requirements and no existing package -lock, the first online `plan` performs a lock step before resource selection. Its -output reports that `pload.toml` was updated. The generated lock contains the +If no current `.pload_lock.toml` exists, the first online `plan` performs a lock +step before resource selection—even when every direct requirement is pinned. Its +output reports that `.pload_lock.toml` was written. The generated lock contains the complete wheel dependency closure, so `apply` does not ask pip to resolve dependencies again. Schema 1 currently requires wheel availability during this automatic lock step; it does not create a portable exact lock from an sdist. @@ -390,9 +431,10 @@ objects/ ``` Two configurations referencing the same large CUDA wheel therefore store and -transfer one object. The TOML contains the checksum and logical repository name; -the user does not run upload/download commands. A missing or unreachable provider -is simply removed from the candidate set, and apply tries the next valid route. +transfer one object. `.pload_lock.toml` contains the checksum and logical +repository name; the user does not run upload/download commands. A missing or +unreachable provider is simply removed from the candidate set, and apply tries +the next valid route. ## Compatibility and boundaries diff --git a/pyproject.toml b/pyproject.toml index ac0d17a..28dfca2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pload" -version = "1.1.0a7" +version = "1.1.0a8" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" diff --git a/src/pload/__init__.py b/src/pload/__init__.py index 3debd0e..0c8b8d1 100644 --- a/src/pload/__init__.py +++ b/src/pload/__init__.py @@ -1 +1 @@ -__version__ = "1.1.0a7" +__version__ = "1.1.0a8" diff --git a/src/pload/cli.py b/src/pload/cli.py index 5802886..74c7a93 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -447,9 +447,10 @@ def print_declarative_plan(plan): console = Console(highlight=False) if plan.get("lock"): count = len(plan["lock"]["resolved"]) + action = "Migrated" if plan["lock"].get("migrated") else "Locked" console.print( - f"[bold green]✓ Locked {count} packages[/] and updated " - f"[cyan]{plan['path']}[/]" + f"[bold green]✓ {action} {count} packages[/] in " + f"[cyan]{plan['lock']['path']}[/]" ) python = plan["python"] console.print(Panel.fit( diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 4b620f3..1c9e348 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -107,7 +107,7 @@ def _array(values): def dump_manifest(data): - """Serialize the deliberately small schema without a TOML writer dependency.""" + """Serialize only the user-authored environment declaration.""" environment = data["environment"] policy = data["policy"] lines = [ @@ -150,6 +150,28 @@ def dump_manifest(data): f"kind = {_quoted(repository['kind'])}", f"location = {_quoted(repository['location'])}", ] + return "\n".join(lines) + "\n" + + +def lock_path(manifest_path): + """Return the pload-managed lock beside a user configuration.""" + return Path(manifest_path).parent / ".pload_lock.toml" + + +def configuration_digest(data): + return hashlib.sha256(dump_manifest(data).encode("utf-8")).hexdigest() + + +def dump_lock(manifest_path, data): + """Serialize resolver output separately from the user configuration.""" + lines = [ + "schema = 1", + f"configuration = {_quoted(Path(manifest_path).name)}", + f"configuration_sha256 = {_quoted(configuration_digest(data))}", + "", + "[environment]", + f"dependencies = {_array(data.get('resolved_dependencies', []))}", + ] for package in data.get("package", []): lines += [ "", @@ -171,6 +193,35 @@ def dump_manifest(data): return "\n".join(lines) + "\n" +def load_lock(manifest_path, configuration): + """Load a current sidecar lock; stale locks are ignored and regenerated.""" + path = lock_path(manifest_path) + if not path.is_file(): + return path, None, False + try: + with path.open("rb") as stream: + lock = tomllib.load(stream) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise PloadError(f"cannot read environment lock {path}: {exc}") from exc + if lock.get("schema") != 1: + raise PloadError(f"unsupported environment lock schema in {path}") + packages = lock.setdefault("package", []) + environment = lock.setdefault("environment", {}) + environment.setdefault("dependencies", []) + for package in packages: + package.setdefault("sources", []) + package.setdefault("artifact", []) + current = ( + lock.get("configuration") == Path(manifest_path).name + and lock.get("configuration_sha256") == configuration_digest(configuration) + ) + if current: + combined = dict(configuration) + combined["package"] = packages + validate_manifest(combined) + return path, lock, current + + def load_manifest(path): path = Path(path).expanduser().resolve() try: @@ -611,9 +662,9 @@ def describe( artifact["repositories"].append(selected_repository) validate_manifest(data) output.parent.mkdir(parents=True, exist_ok=True) - temporary = output.with_name(output.name + ".tmp") - temporary.write_text(dump_manifest(data), encoding="utf-8") - temporary.replace(output) + self._write_manifest(output, data) + data["resolved_dependencies"] = pins + self._write_lock(output, data) return output def _obtain_wheel(self, interpreter, installed, wheel_dir, index): @@ -658,12 +709,32 @@ def _obtain_wheel(self, interpreter, installed, wheel_dir, index): raise PloadError(f"download did not produce one exact wheel for {installed['name']}") return matches[0] - def _lock_requested_dependencies(self, path, data, offline=False): + def _load_state(self, manifest_path): + path, data = load_manifest(manifest_path) + legacy_packages = list(data.get("package", [])) + _, lock, current = load_lock(path, data) + if current: + data["package"] = lock.get("package", []) + data["resolved_dependencies"] = lock["environment"].get("dependencies", []) + elif legacy_packages: + # Versions before 1.1.0a8 embedded resolver output in pload.toml. + # Keep it long enough to migrate it into the sidecar without a download. + data["package"] = legacy_packages + data["resolved_dependencies"] = [ + f"{item['name']}=={item['version']}" for item in legacy_packages + ] + else: + data["package"] = [] + data["resolved_dependencies"] = [] + return path, data, current, bool(legacy_packages) + + def _lock_requested_dependencies( + self, path, data, lock_current=False, legacy=False, offline=False, + ): requested = list(data["environment"].get("dependencies", [])) if not requested: return None requirements = [_parse_dependency(value) for value in requested] - all_exact = all(PIN.fullmatch(item) for item in requested) locked = { normalized_name(package["name"]): package["version"] for package in data.get("package", []) @@ -675,26 +746,22 @@ def _lock_requested_dependencies(self, path, data, offline=False): )) for requirement in requirements ) - exact_lock_matches = not all_exact or { - (normalized_name(match.group(1)), match.group(2)) - for item in requested if (match := PIN.fullmatch(item)) - } == set(locked.items()) - - if data.get("package") and satisfies_requests and exact_lock_matches: - if all_exact: - return None + if data.get("package") and satisfies_requests and (lock_current or legacy): resolved = [ f"{package['name']}=={package['version']}" for package in sorted( data["package"], key=lambda item: normalized_name(item["name"]) ) ] - data["environment"]["dependencies"] = resolved - validate_manifest(data) - self._write_manifest(path, data) - return {"updated": True, "requested": requested, "resolved": resolved} - - if all_exact and not data.get("package"): + data["resolved_dependencies"] = resolved + if legacy: + self._write_manifest(path, data) + self._write_lock(path, data) + return { + "updated": True, "migrated": True, + "path": str(lock_path(path)), + "requested": requested, "resolved": resolved, + } return None network_allowed = ( @@ -786,10 +853,13 @@ def _lock_requested_dependencies(self, path, data, offline=False): records.sort(key=lambda item: normalized_name(item["name"])) resolved = [f"{item['name']}=={item['version']}" for item in records] data["package"] = records - data["environment"]["dependencies"] = resolved + data["resolved_dependencies"] = resolved validate_manifest(data) - self._write_manifest(path, data) - return {"updated": True, "requested": requested, "resolved": resolved} + self._write_lock(path, data) + return { + "updated": True, "path": str(lock_path(path)), + "requested": requested, "resolved": resolved, + } @staticmethod def _write_manifest(path, data): @@ -797,9 +867,18 @@ def _write_manifest(path, data): temporary.write_text(dump_manifest(data), encoding="utf-8") temporary.replace(path) + @staticmethod + def _write_lock(path, data): + destination = lock_path(path) + temporary = destination.with_name(destination.name + ".tmp") + temporary.write_text(dump_lock(path, data), encoding="utf-8") + temporary.replace(destination) + def plan(self, manifest_path, offline=False): - path, data = load_manifest(manifest_path) - lock = self._lock_requested_dependencies(path, data, offline=offline) + path, data, current, legacy = self._load_state(manifest_path) + lock = self._lock_requested_dependencies( + path, data, lock_current=current, legacy=legacy, offline=offline, + ) policy = data["policy"] network_allowed = policy.get("network", "allow") == "allow" and not offline repositories = data.get("repositories", {}) @@ -916,10 +995,10 @@ def _locked_packages(data): return result def apply(self, manifest_path, name=None, offline=False, progress=None): - path, data = load_manifest(manifest_path) + path, data, _, _ = self._load_state(manifest_path) plan = self.plan(path, offline=offline) if plan.get("lock"): - path, data = load_manifest(path) + path, data, _, _ = self._load_state(path) unavailable = [item for item in plan["packages"] if not item["selected"]] if unavailable: details = [] diff --git a/tests/test_declarative.py b/tests/test_declarative.py index a28dff2..49fb409 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -11,8 +11,11 @@ from pload.cli import build_parser, main, shell_script from pload.declarative import ( DeclarativeEnvironmentManager, + dump_lock, dump_manifest, + load_lock, load_manifest, + lock_path, validate_manifest, ) from pload.errors import PloadError @@ -64,13 +67,52 @@ def simple_manifest(mode="compatible"): } +def write_configuration(path, data, with_lock=True): + path.write_text(dump_manifest(data), encoding="utf-8") + if with_lock and data.get("package"): + locked = dict(data) + locked["resolved_dependencies"] = [ + f"{item['name']}=={item['version']}" for item in data["package"] + ] + lock_path(path).write_text(dump_lock(path, locked), encoding="utf-8") + + def test_manifest_roundtrip(tmp_path): data = simple_manifest() path = tmp_path / "pload.toml" - path.write_text(dump_manifest(data), encoding="utf-8") + write_configuration(path, data) loaded_path, loaded = load_manifest(path) assert loaded_path == path - assert loaded == data + assert loaded["environment"] == data["environment"] + assert loaded["package"] == [] + _, locked, current = load_lock(path, loaded) + assert current is True + assert locked["package"] == data["package"] + + +def test_plan_migrates_legacy_embedded_lock_without_resolving(tmp_path, monkeypatch): + data = simple_manifest("exact") + path = tmp_path / "pload.toml" + legacy = dump_manifest(data) + """ +[[package]] +name = "demo" +version = "1.0" +sources = ["default"] +""" + path.write_text(legacy, encoding="utf-8") + monkeypatch.setattr( + declarative_module, "execute", + lambda *args, **kwargs: pytest.fail("legacy migration invoked the resolver"), + ) + + plan = DeclarativeEnvironmentManager(ConfigManager(home=tmp_path / "home")).plan(path) + + assert plan["lock"]["migrated"] is True + assert "[[package]]" not in path.read_text(encoding="utf-8") + _, configuration = load_manifest(path) + _, locked, current = load_lock(path, configuration) + assert current is True + assert locked["package"][0]["name"] == "demo" def test_manifest_accepts_unpinned_requirements_but_rejects_direct_urls(): @@ -111,7 +153,7 @@ def test_manifest_rejects_unknown_policy_repository(): def test_compatible_plan_uses_index_and_offline_reports_missing(tmp_path): path = tmp_path / "pload.toml" - path.write_text(dump_manifest(simple_manifest()), encoding="utf-8") + write_configuration(path, simple_manifest()) manager = DeclarativeEnvironmentManager(ConfigManager(home=tmp_path / "home")) assert manager.plan(path)["packages"][0]["selected"]["method"] == "index-resolve" assert manager.plan(path, offline=True)["packages"][0]["selected"] is None @@ -120,7 +162,7 @@ def test_compatible_plan_uses_index_and_offline_reports_missing(tmp_path): def test_exact_configuration_requires_an_artifact_route(tmp_path): data = simple_manifest("exact") path = tmp_path / "pload.toml" - path.write_text(dump_manifest(data), encoding="utf-8") + write_configuration(path, data) manager = DeclarativeEnvironmentManager(ConfigManager(home=tmp_path / "home")) assert manager.plan(path)["ready"] is False @@ -147,7 +189,7 @@ def test_changed_python_rejects_incompatible_cached_wheel_before_apply(tmp_path, }], }] manifest = tmp_path / "pload.toml" - manifest.write_text(dump_manifest(data), encoding="utf-8") + write_configuration(manifest, data) manager = DeclarativeEnvironmentManager(config) manager.cache.mkdir(parents=True) shutil.copyfile(wheel, manager.cache / wheel.name) @@ -171,7 +213,7 @@ def test_changed_python_rejects_incompatible_cached_wheel_before_apply(tmp_path, manager.apply(manifest) data["policy"]["reproducibility"] = "compatible" - manifest.write_text(dump_manifest(data), encoding="utf-8") + write_configuration(manifest, data) compatible = manager.plan(manifest)["packages"][0] assert compatible["selected"]["method"] == "index-resolve" assert compatible["selected"]["status"] == "network" @@ -199,7 +241,7 @@ def test_cache_plan_applies_without_package_network_access(tmp_path, monkeypatch }], }] manifest = tmp_path / "pload.toml" - manifest.write_text(dump_manifest(data), encoding="utf-8") + write_configuration(manifest, data) manager = DeclarativeEnvironmentManager(config) manager.cache.mkdir(parents=True) shutil.copyfile(wheel, manager.cache / wheel.name) @@ -238,7 +280,7 @@ def test_plan_locks_unpinned_dependencies_once_then_applies_from_cache(tmp_path, data["policy"]["publish_missing_artifacts"] = False data["package"] = [] manifest = tmp_path / "pload.toml" - manifest.write_text(dump_manifest(data), encoding="utf-8") + write_configuration(manifest, data, with_lock=False) manager = DeclarativeEnvironmentManager(config) original_execute = declarative_module.execute download_calls = [] @@ -259,7 +301,10 @@ def fake_resolver(command, *args, **kwargs): assert first["lock"]["requested"] == ["pload-demo>=1"] assert first["lock"]["resolved"] == ["pload-demo==1.0"] assert first["packages"][0]["selected"]["method"] == "cache" - _, locked = load_manifest(manifest) + _, configuration = load_manifest(manifest) + assert configuration["environment"]["dependencies"] == ["pload-demo>=1"] + _, locked, current = load_lock(manifest, configuration) + assert current is True assert locked["environment"]["dependencies"] == ["pload-demo==1.0"] assert locked["package"][0]["artifact"][0]["filename"] == wheel.name @@ -285,10 +330,12 @@ def test_plan_relocks_when_new_dependency_makes_existing_lock_stale(tmp_path, mo "implementation": sys.implementation.name, "system": platform.system(), "machine": platform.machine(), - "dependencies": ["pload-demo==1.0", "extra-demo"], + "dependencies": ["pload-demo==1.0"], }) manifest = tmp_path / "pload.toml" - manifest.write_text(dump_manifest(data), encoding="utf-8") + write_configuration(manifest, data) + data["environment"]["dependencies"].append("extra-demo") + write_configuration(manifest, data, with_lock=False) manager = DeclarativeEnvironmentManager(config) calls = [] @@ -309,7 +356,12 @@ def fake_resolver(command, *args, **kwargs): assert len(calls) == 1 assert plan["lock"]["requested"] == ["pload-demo==1.0", "extra-demo"] assert plan["lock"]["resolved"] == ["extra-demo==1.0", "pload-demo==1.0"] - _, locked = load_manifest(manifest) + _, configuration = load_manifest(manifest) + assert configuration["environment"]["dependencies"] == [ + "pload-demo==1.0", "extra-demo", + ] + _, locked, current = load_lock(manifest, configuration) + assert current is True assert {item["name"] for item in locked["package"]} == {"extra-demo", "pload-demo"} @@ -336,9 +388,11 @@ def test_describe_plan_apply_through_content_repository(tmp_path): progress=describe_progress.append, ) _, data = load_manifest(manifest) - artifact = data["package"][0]["artifact"][0] + _, locked, current = load_lock(manifest, data) + assert current is True + artifact = locked["package"][0]["artifact"][0] assert data["sources"]["default"]["url"] == "https://example.invalid/simple" - assert data["package"][0]["sources"] == ["package-pload-demo"] + assert locked["package"][0]["sources"] == ["package-pload-demo"] assert data["sources"]["package-pload-demo"]["url"].endswith("/cu121") assert artifact["repositories"] == ["lab"] assert (repository / "objects" / artifact["sha256"]).is_file() @@ -381,7 +435,7 @@ def test_apply_rolls_back_a_new_environment_after_install_failure(tmp_path): "tags": ["py3-none-any"], "repositories": [], }] manifest = tmp_path / "pload.toml" - manifest.write_text(dump_manifest(data), encoding="utf-8") + write_configuration(manifest, data) with pytest.raises(PloadError): DeclarativeEnvironmentManager(config).apply(manifest) with pytest.raises(PloadError): @@ -415,7 +469,7 @@ def test_apply_publishes_acquired_artifacts_to_policy_repository(tmp_path): }], }] manifest = tmp_path / "pload.toml" - manifest.write_text(dump_manifest(data), encoding="utf-8") + write_configuration(manifest, data) DeclarativeEnvironmentManager(config).apply(manifest) assert (repository / "objects" / checksum).is_file() From fe9c99cf145a483367c3b04646806ded62e885a8 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 16:20:29 +0800 Subject: [PATCH 16/41] Show live progress while planning --- CHANGELOG.md | 5 ++++- README.md | 2 +- pyproject.toml | 2 +- src/pload/__init__.py | 2 +- src/pload/cli.py | 13 ++++++++++++- src/pload/declarative.py | 11 ++++++++++- tests/test_declarative.py | 6 +++++- 7 files changed, 34 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fa6c7d..d88b7bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -Pre-release: `1.1.0a8` +Pre-release: `1.1.0a9` ### Added @@ -30,6 +30,9 @@ Pre-release: `1.1.0a8` uses the complete requested set and prefers existing pload cache wheels. - Migrate lock tables embedded by earlier 1.1 previews into `.pload_lock.toml` without another dependency resolution or artifact download. +- Show an animated planning status immediately, including dependency-resolution + and artifact-repository phases, instead of appearing frozen while pip or SSH + work is captured in the background. ### Removed diff --git a/README.md b/README.md index 5a0c817..4d89403 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ pload-install --yes --package-spec "pload==1.0.0" To opt into the declarative-environment pre-release for testing: ```console -pload-install --yes --package-spec "pload==1.1.0a8" +pload-install --yes --package-spec "pload==1.1.0a9" ``` To always follow the newest published release instead of pinning a version, diff --git a/pyproject.toml b/pyproject.toml index 28dfca2..e692d9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pload" -version = "1.1.0a8" +version = "1.1.0a9" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" diff --git a/src/pload/__init__.py b/src/pload/__init__.py index 0c8b8d1..16542b2 100644 --- a/src/pload/__init__.py +++ b/src/pload/__init__.py @@ -1 +1 @@ -__version__ = "1.1.0a8" +__version__ = "1.1.0a9" diff --git a/src/pload/cli.py b/src/pload/cli.py index 74c7a93..8a25519 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -766,10 +766,21 @@ def run(argv=None): ), )) elif command == "plan": - plan = manager.plan(args.file, args.offline) if args.json: + plan = manager.plan(args.file, args.offline) print(json.dumps(plan, ensure_ascii=False, indent=2)) else: + progress_console = Console(highlight=False, stderr=True) + with progress_console.status( + "[cyan]Reading configuration and available resources…[/]", + spinner="dots", + ) as status: + plan = manager.plan( + args.file, args.offline, + progress=lambda message: status.update( + f"[cyan]{message}…[/]" + ), + ) print_declarative_plan(plan) else: progress_console = Console(highlight=False) diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 1c9e348..e3fb151 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -730,6 +730,7 @@ def _load_state(self, manifest_path): def _lock_requested_dependencies( self, path, data, lock_current=False, legacy=False, offline=False, + progress=None, ): requested = list(data["environment"].get("dependencies", [])) if not requested: @@ -805,6 +806,11 @@ def _lock_requested_dependencies( for source_name in source_names[1:]: command += ["--extra-index-url", sources[source_name]["url"]] try: + if progress: + progress( + "Configuration changed; resolving dependencies for Python " + f"{data['environment']['python']}" + ) execute(command + requested) except PloadError as exc: raise PloadError( @@ -874,10 +880,11 @@ def _write_lock(path, data): temporary.write_text(dump_lock(path, data), encoding="utf-8") temporary.replace(destination) - def plan(self, manifest_path, offline=False): + def plan(self, manifest_path, offline=False, progress=None): path, data, current, legacy = self._load_state(manifest_path) lock = self._lock_requested_dependencies( path, data, lock_current=current, legacy=legacy, offline=offline, + progress=progress, ) policy = data["policy"] network_allowed = policy.get("network", "allow") == "allow" and not offline @@ -908,6 +915,8 @@ def plan(self, manifest_path, offline=False): ) repository_objects = {} for repo_name, checksums in repository_checksums.items(): + if progress: + progress(f"Checking artifact repository {repo_name}") try: repository_objects[repo_name] = ArtifactRepository.contains_many( repositories[repo_name], checksums, path.parent, diff --git a/tests/test_declarative.py b/tests/test_declarative.py index 49fb409..89e5857 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -352,8 +352,12 @@ def fake_resolver(command, *args, **kwargs): return "" monkeypatch.setattr(declarative_module, "execute", fake_resolver) - plan = manager.plan(manifest) + progress = [] + plan = manager.plan(manifest, progress=progress.append) assert len(calls) == 1 + assert progress == [ + f"Configuration changed; resolving dependencies for Python {platform.python_version()}" + ] assert plan["lock"]["requested"] == ["pload-demo==1.0", "extra-demo"] assert plan["lock"]["resolved"] == ["extra-demo==1.0", "pload-demo==1.0"] _, configuration = load_manifest(manifest) From 6002f26595817aa56e379de819b6fc0c3b83c6cb Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 16:34:14 +0800 Subject: [PATCH 17/41] Make planning strictly read-only --- CHANGELOG.md | 17 ++++---- README.md | 7 ++- docs/declarative-environments.md | 74 ++++++++++++++++++++------------ pyproject.toml | 2 +- src/pload/__init__.py | 2 +- src/pload/cli.py | 43 +++++++++++++++---- src/pload/declarative.py | 58 ++++++++++++++++++++++--- src/pload/help_content.py | 41 ++++++++++++------ tests/test_declarative.py | 45 ++++++++++++++----- 9 files changed, 211 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d88b7bd..2f7802d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,14 +2,14 @@ ## Unreleased -Pre-release: `1.1.0a9` +Pre-release: `1.1.0a10` ### Added - Keep user intent in a clean portable `pload.toml` and store pload-managed exact dependency, artifact and hash decisions in sibling `.pload_lock.toml`. -- Add `pload describe`, `pload plan` and idempotent `pload apply`; resource - copying, downloading and publication are internal planner/executor steps. +- Add `pload describe`, explicit `pload lock`, read-only `pload plan` and + idempotent `pload apply`; downloading cannot be triggered by planning. - Inventory the local wheel cache, adjacent artifacts, local/SSH content stores, configured indexes and Python installations before choosing a route. - Allow package-specific indexes in `describe`, including CUDA wheel indexes, @@ -23,16 +23,17 @@ Pre-release: `1.1.0a9` runtime during planning; incompatible cache entries can no longer be reported as ready, and compatible mode explicitly plans network re-resolution instead. - Accept unpinned dependency intentions such as `numpy` and `pandas>=2`; the - first online plan resolves the complete wheel dependency closure, records exact - versions, hashes and tags in the managed lock, and subsequent plans reuse it. + explicit lock command resolves the complete wheel dependency closure, records + exact versions, hashes and tags, and subsequent plans inspect it read-only. - Treat edits that add or change dependencies beside an existing lock as a stale lock to reconcile, not an invalid configuration; re-resolution uses the complete requested set and prefers existing pload cache wheels. - Migrate lock tables embedded by earlier 1.1 previews into `.pload_lock.toml` without another dependency resolution or artifact download. -- Show an animated planning status immediately, including dependency-resolution - and artifact-repository phases, instead of appearing frozen while pip or SSH - work is captured in the background. +- Show animated status for explicit locking and artifact-repository inspection + instead of appearing frozen while pip or SSH work is captured in the background. +- Refuse `apply` when the lock is absent or stale instead of resolving and + downloading implicitly. ### Removed diff --git a/README.md b/README.md index 4d89403..10e3ae5 100644 --- a/README.md +++ b/README.md @@ -86,10 +86,15 @@ then performs transfers and installation internally: ```console pload describe v2 -o pload.toml +pload lock pload.toml pload plan pload.toml pload apply pload.toml ``` +`plan` is strictly read-only: it never invokes pip, downloads packages, or +changes either TOML file. Run `lock` explicitly after editing dependencies; +`apply` refuses a missing or stale lock instead of resolving behind your back. + See the [declarative environment guide](docs/declarative-environments.md) for the [complete TOML field reference](docs/declarative-environments.md#complete-ploadtoml-field-reference), exact versus compatible policies, resource-selection algorithm, CUDA wheel @@ -114,7 +119,7 @@ pload-install --yes --package-spec "pload==1.0.0" To opt into the declarative-environment pre-release for testing: ```console -pload-install --yes --package-spec "pload==1.1.0a9" +pload-install --yes --package-spec "pload==1.1.0a10" ``` To always follow the newest published release instead of pinning a version, diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index a5b01bc..c95e603 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -7,12 +7,14 @@ materializes that state. ```console pload describe v2 -o pload.toml +pload lock pload.toml +pload plan pload.toml pload apply pload.toml ``` `describe` is the bridge for an existing environment. For a new project, `pload.toml` may be written by hand with names, ranges or exact -`NAME==VERSION` requirements; `plan` produces the managed lock. +`NAME==VERSION` requirements; `lock` produces the managed lock. ## Two files, two responsibilities @@ -50,7 +52,7 @@ kind = "ssh" location = "frpxiaoxin:/home/tibless/pload-cloud" ``` -`pload plan` writes its decisions to `.pload_lock.toml` in the same directory: +`pload lock` writes its decisions to `.pload_lock.toml` in the same directory: ```toml schema = 1 @@ -76,8 +78,8 @@ The lock is generated data, not a second user configuration. It records the complete transitive dependency closure, exact versions, wheel identities, SHA-256 hashes, compatibility tags and known artifact locations. The `configuration_sha256` binds it to the current `pload.toml`; editing the user -configuration makes the old lock stale and the next online plan replaces it -atomically. Commit both files when reproducibility matters, but normally edit +configuration makes the old lock stale and the next explicit `pload lock` +replaces it atomically. Commit both files when reproducibility matters, but normally edit only `pload.toml`. The lock is necessary for **exact** reproduction because a requirement such as @@ -85,8 +87,8 @@ The lock is necessary for **exact** reproduction because a requirement such as or a particular wheel build. Package indexes change over time. The filename and SHA-256 in the lock let pload prove that a cached, cloud-hosted or downloaded artifact is the same byte sequence chosen originally. Deleting the lock is safe: -the next online `plan` resolves a new one, but the result may differ. Offline -planning cannot regenerate a missing or stale lock. +the next online `pload lock` resolves a new one, but the result may differ. +`plan` never regenerates it. Python runtimes and wheel bytes remain in caches, indexes or content-addressed repositories; the two TOML files contain only control information. Credentials @@ -129,17 +131,17 @@ For example, this is a valid initial request: dependencies = ["numpy", "pandas>=2,<3"] ``` -`pload plan` asks pip under the requested interpreter to resolve these requirements +`pload lock` asks pip under the requested interpreter to resolve these requirements and their transitive dependencies as compatible wheels. It then atomically writes exact `NAME==VERSION` results, wheel filenames, SHA-256 hashes and tags to `.pload_lock.toml`; `pload.toml` remains unchanged. Resolution uses all declared indexes, with `default` as the primary index. It requires network access once; -`--offline` rejects an unlocked file rather than guessing. A second plan is -read-only and does not run the resolver again. +`lock --offline` rejects unavailable resolver inputs rather than guessing. +`plan` is always read-only and never runs the resolver. Editing dependencies after a lock exists is also supported. For example, adding `"torch"` to a configuration with an existing `.pload_lock.toml` marks that lock -as stale. The next online plan resolves the complete +as stale. The next online `pload lock` resolves the complete requested set—including existing exact constraints—rather than rejecting the file or silently dropping the new package. The resolver receives the pload wheel cache as a local candidate source, so compatible cached wheels are preferred and @@ -147,7 +149,7 @@ only missing artifacts need index access. The reconciled complete lock replaces the old one atomically. Configurations produced by pre-`1.1.0a8` previews may still contain embedded -`[[package]]` tables. The first `plan` migrates those tables to +`[[package]]` tables. The first `pload lock` migrates those tables to `.pload_lock.toml` without downloading again and rewrites `pload.toml` in the clean declaration-only format. Because older previews replaced the original direct requirements with the complete closure, they cannot always recover which @@ -265,7 +267,7 @@ There is one array entry for every application dependency: | `artifact` | array of tables | no | Exact wheel identities accepted for this package. Exact policy needs at least one reachable artifact route; compatible policy may omit artifacts and re-resolve the pinned version. | Users should not add or remove these records manually. Delete `.pload_lock.toml` -and run `pload plan` when a complete re-resolution is wanted. +and run `pload lock` when a complete re-resolution is wanted. ### `[[package.artifact]]`: exact wheel identity @@ -354,6 +356,21 @@ pload describe v2 --mode compatible Compatible mode is useful for exploration, but it is weaker than an artifact lock. +## Lock the declared dependencies + +```console +pload lock +pload lock project.pload.toml +pload lock project.pload.toml --offline +``` + +`lock` is the explicit boundary at which pload may invoke pip, resolve the full +dependency closure, obtain wheel bytes, populate the local cache and atomically +write `.pload_lock.toml`. It does not modify `pload.toml`. Run it after changing +Python, dependencies, indexes or any other resolution input. Keeping this as a +separate command prevents a read-only preview from changing the resources it is +supposed to inspect. + ## Plan against available resources ```console @@ -363,14 +380,14 @@ pload plan --offline pload plan --json ``` -If no current `.pload_lock.toml` exists, the first online `plan` performs a lock -step before resource selection—even when every direct requirement is pinned. Its -output reports that `.pload_lock.toml` was written. The generated lock contains the -complete wheel dependency closure, so `apply` does not ask pip to resolve -dependencies again. Schema 1 currently requires wheel availability during this -automatic lock step; it does not create a portable exact lock from an sdist. +`plan` is strictly read-only. It never invokes pip, downloads a package, fills +the cache, writes a lock or changes the user configuration. If +`.pload_lock.toml` is absent or its configuration digest is stale, the plan is +not ready: direct requirements are shown as `lock-required / pending` and the +user is told to run `pload lock`. This is intentionally different from a route +that only happens to become ready because planning downloaded it. -The planner inventories: +With a current lock, the planner inventories: 1. installed and discoverable Python interpreters; 2. the pload wheel cache; @@ -406,15 +423,16 @@ pload apply project.pload.toml --offline `apply` performs the internal operations that were previously exposed as export, push, pull and restore: -1. resolve or install the declared Python; -2. verify implementation, exact Python version, OS and architecture; -3. reuse exact local artifacts; -4. retrieve missing SHA-addressed artifacts from configured repositories; -5. fall back to an index only when policy permits it; -6. verify every artifact before installation; -7. create and register the environment; -8. install exact packages and run `pip check`; -9. record the applied configuration digest inside the environment. +1. require a current `.pload_lock.toml` and refuse implicit re-resolution; +2. resolve or install the declared Python; +3. verify implementation, exact Python version, OS and architecture; +4. reuse exact local artifacts; +5. retrieve missing SHA-addressed artifacts from configured repositories; +6. fall back to an index only when policy permits it; +7. verify every artifact before installation; +8. create and register the environment; +9. install exact packages and run `pip check`; +10. record the applied configuration digest inside the environment. Running `apply` again is idempotent when the environment already carries the same configuration digest. If the name exists with different state, pload refuses to diff --git a/pyproject.toml b/pyproject.toml index e692d9d..17ac31a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pload" -version = "1.1.0a9" +version = "1.1.0a10" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" diff --git a/src/pload/__init__.py b/src/pload/__init__.py index 16542b2..f7f9421 100644 --- a/src/pload/__init__.py +++ b/src/pload/__init__.py @@ -1 +1 @@ -__version__ = "1.1.0a9" +__version__ = "1.1.0a10" diff --git a/src/pload/cli.py b/src/pload/cli.py index 8a25519..cde92f8 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -29,6 +29,7 @@ "python": ("py",), "config": ("cfg",), "describe": (), + "lock": (), "plan": (), "apply": (), "repo": (), @@ -297,6 +298,14 @@ def build_parser(): metavar="PACKAGE=URL", help="package-specific index; repeat for packages such as CUDA-enabled torch", ) + lock = subparsers.add_parser( + "lock", description="Resolve dependencies and write the managed environment lock.", + help="resolve and write .pload_lock.toml", + ) + lock.add_argument("file", nargs="?", default="pload.toml", + help="environment configuration (default: pload.toml)") + lock.add_argument("--offline", "-o", action="store_true", + help="use only already available resolver resources") plan = subparsers.add_parser( "plan", description="Compare a declarative environment with all available resources.", help="explain how pload would satisfy a configuration", @@ -445,12 +454,10 @@ def _global_options_table(root_parser): def print_declarative_plan(plan): console = Console(highlight=False) - if plan.get("lock"): - count = len(plan["lock"]["resolved"]) - action = "Migrated" if plan["lock"].get("migrated") else "Locked" + if plan["lock"]["required"]: console.print( - f"[bold green]✓ {action} {count} packages[/] in " - f"[cyan]{plan['lock']['path']}[/]" + f"[bold yellow]Lock: {plan['lock']['status']}[/] · plan is read-only; " + "run [bold green]pload lock[/] to resolve dependencies" ) python = plan["python"] console.print(Panel.fit( @@ -552,6 +559,7 @@ def _brief_help(parser, command_path, root_parser=None): "config": "Show effective configuration", "shell-init": "Print shell activation integration", "describe": "Write one portable configuration for an existing environment", + "lock": "Resolve dependencies into .pload_lock.toml", "plan": "Compare a configuration with available resources", "apply": "Materialize the environment declared by pload.toml", "repo": "Manage local and SSH artifact providers", @@ -634,6 +642,8 @@ def render_landing(): table.add_row("pload cfg", "Show where pload stores its data") table.add_row("pload python list", "Find every usable Python interpreter") table.add_row("pload describe v1", "Describe an environment in pload.toml") + table.add_row("pload lock", "Resolve and record exact package artifacts") + table.add_row("pload plan", "Preview without downloading or changing files") table.add_row("pload apply", "Create exactly what pload.toml declares") table.add_row("pload new", "Open guided environment creation") table.add_row("pload new -n data -v 3.12", "Create a named virtual environment") @@ -648,7 +658,7 @@ def shell_script(shell): if shell in {"bash", "zsh"}: return r'''pload() { case "${1:-}" in - ""|new|init|i|rm|remove|del|delete|list|ls|path|p|shell-init|shell|python|py|config|cfg|describe|plan|apply|repo|-*) + ""|new|init|i|rm|remove|del|delete|list|ls|path|p|shell-init|shell|python|py|config|cfg|describe|lock|plan|apply|repo|-*) command pload "$@" ;; *) @@ -661,7 +671,7 @@ def shell_script(shell): if shell == "fish": return r'''function pload switch "$argv[1]" - case '' new init i rm remove del delete list ls path p shell-init shell python py config cfg describe plan apply repo '-*' + case '' new init i rm remove del delete list ls path p shell-init shell python py config cfg describe lock plan apply repo '-*' command pload $argv case '*' set -l name . @@ -675,7 +685,7 @@ def shell_script(shell): end''' return r'''function pload { param([Parameter(ValueFromRemainingArguments = $true)][string[]]$PloadArgs) - $commands = @('new', 'init', 'i', 'rm', 'remove', 'del', 'delete', 'list', 'ls', 'path', 'p', 'shell-init', 'shell', 'python', 'py', 'config', 'cfg', 'describe', 'plan', 'apply', 'repo', '-h', '--help', '--version') + $commands = @('new', 'init', 'i', 'rm', 'remove', 'del', 'delete', 'list', 'ls', 'path', 'p', 'shell-init', 'shell', 'python', 'py', 'config', 'cfg', 'describe', 'lock', 'plan', 'apply', 'repo', '-h', '--help', '--version') $backend = Get-Command -Name @('pload.exe', 'pload.cmd') -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1 if (-not $backend) { throw 'pload executable not found on PATH' } if ($PloadArgs.Count -eq 0) { @@ -741,7 +751,7 @@ def run(argv=None): venvs = VenvManager(config) dependencies = DependencyManager(config) - if command in {"describe", "plan", "apply", "repo"}: + if command in {"describe", "lock", "plan", "apply", "repo"}: if command == "repo": from pload.snapshots import RepositoryManager @@ -765,6 +775,21 @@ def run(argv=None): Text("• " + message, style="cyan") ), )) + elif command == "lock": + progress_console = Console(highlight=False, stderr=True) + with progress_console.status( + "[cyan]Reading configuration…[/]", spinner="dots", + ) as status: + result = manager.lock( + args.file, args.offline, + progress=lambda message: status.update(f"[cyan]{message}…[/]"), + ) + count = len(result["resolved"]) + action = "Wrote" if result["updated"] else "Reused" + Console(highlight=False).print( + f"[bold green]✓ {action} {count} locked packages[/] in " + f"[cyan]{result['path']}[/]" + ) elif command == "plan": if args.json: plan = manager.plan(args.file, args.offline) diff --git a/src/pload/declarative.py b/src/pload/declarative.py index e3fb151..e29d2c3 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -880,12 +880,35 @@ def _write_lock(path, data): temporary.write_text(dump_lock(path, data), encoding="utf-8") temporary.replace(destination) - def plan(self, manifest_path, offline=False, progress=None): + def lock(self, manifest_path, offline=False, progress=None): + """Resolve and persist an exact sidecar lock as an explicit mutation.""" path, data, current, legacy = self._load_state(manifest_path) - lock = self._lock_requested_dependencies( + result = self._lock_requested_dependencies( path, data, lock_current=current, legacy=legacy, offline=offline, progress=progress, ) + if result: + return result + return { + "updated": False, + "path": str(lock_path(path)), + "requested": list(data["environment"].get("dependencies", [])), + "resolved": list(data.get("resolved_dependencies", [])), + } + + def plan(self, manifest_path, offline=False, progress=None): + path, data, current, legacy = self._load_state(manifest_path) + dependencies = data["environment"].get("dependencies", []) + lock_required = bool(dependencies) and not current + sidecar = lock_path(path) + lock = { + "status": ( + "current" if current else "legacy" if legacy + else "stale" if sidecar.exists() else "missing" + ), + "path": str(sidecar), + "required": lock_required, + } policy = data["policy"] network_allowed = policy.get("network", "allow") == "allow" and not offline repositories = data.get("repositories", {}) @@ -900,6 +923,24 @@ def plan(self, manifest_path, offline=False, progress=None): "method": "install-python", "location": data["environment"]["python"], "status": "network" if network_allowed else "unavailable", } + if lock_required: + pending = [] + for value in dependencies: + requirement = _parse_dependency(value) + exact = PIN.fullmatch(value) + pending.append({ + "name": requirement.name, + "version": exact.group(2) if exact else str(requirement.specifier) or "unresolved", + "selected": self._candidate( + "lock-required", sidecar, "pending", (0, 0, 0, 0) + ), + "alternatives": [], + "rejections": [], + }) + return { + "path": str(path), "name": data["name"], "python": python_action, + "packages": pending, "ready": False, "lock": lock, + } repository_checksums = {} for package in packages: for artifact in package.get("artifact", []): @@ -979,7 +1020,7 @@ def plan(self, manifest_path, offline=False, progress=None): "alternatives": candidates[1:], "rejections": rejections, }) - ready = (python_action["status"] != "unavailable" + ready = (not lock_required and python_action["status"] != "unavailable" and all(item["selected"] for item in plans)) return {"path": str(path), "name": data["name"], "python": python_action, "packages": plans, "ready": ready, "lock": lock} @@ -999,15 +1040,20 @@ def _locked_packages(data): result = [] for pin in data["environment"].get("dependencies", []): match = PIN.fullmatch(pin) + if not match: + continue result.append({"name": match.group(1), "version": match.group(2), "sources": ["default"], "artifact": []}) return result def apply(self, manifest_path, name=None, offline=False, progress=None): path, data, _, _ = self._load_state(manifest_path) - plan = self.plan(path, offline=offline) - if plan.get("lock"): - path, data, _, _ = self._load_state(path) + plan = self.plan(path, offline=offline, progress=progress) + if plan["lock"]["required"]: + raise PloadError( + "environment lock is missing or stale; review the configuration, then run " + f"'pload lock {path.name}'" + ) unavailable = [item for item in plan["packages"] if not item["selected"]] if unavailable: details = [] diff --git a/src/pload/help_content.py b/src/pload/help_content.py index c58c5d6..10e5c60 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -55,14 +55,15 @@ ```console $ pload describe v1 -o pload.toml +$ pload lock pload.toml $ pload plan pload.toml $ pload apply pload.toml ``` -The TOML file describes the desired Python, exact packages, platform, policies, -artifact hashes, indexes and logical artifact providers. `apply` inventories the -current machine and automatically reuses, copies or downloads the best valid -resources; users do not perform separate upload, download or migration steps. +`pload.toml` describes user intent and `.pload_lock.toml` records exact package +artifacts. `plan` is strictly read-only. `lock` is the explicit operation that +resolves dependencies and may download artifacts; `apply` only accepts a current +lock and materializes its selected routes. """, ("new",): r""" # Create a managed environment @@ -376,14 +377,27 @@ ``` Planning reads the configuration, discovers Python interpreters and checks local -caches, adjacent artifacts, content-addressed repositories and declared indexes. +caches, adjacent artifacts and content-addressed repositories. Incompatible resources are rejected first; valid routes are ordered by exactness, compatibility risk, transfer, execution cost and a deterministic tie-breaker. -Dependencies may initially be unpinned (`numpy`, `pandas>=2`). The first online -plan resolves their complete wheel dependency closure, writes exact versions and -artifact hashes back to the TOML file, then plans from that lock. Later plans do -not resolve again; `--offline` requires the lock to exist already. -Planning is optional transparency: normal users can go directly to `pload apply`. +It never invokes pip, downloads packages, writes a lock or changes either TOML +file. A missing or stale lock is reported as `lock-required / pending`; run +`pload lock` explicitly before planning exact resource routes. +""" +GUIDES[("lock",)] = r""" +# Resolve the user declaration into an exact managed lock + +```console +pload lock +pload lock project.pload.toml +pload lock --offline +``` + +Locking is an explicit mutating operation. It resolves direct and transitive +dependencies for the configured Python, obtains exact wheels when necessary, +computes SHA-256 identities and atomically writes `.pload_lock.toml` beside the +configuration. `pload.toml` is never rewritten. Run it after changing Python, +dependencies, sources or another resolution input. """ GUIDES[("apply",)] = r""" # Materialize the configuration @@ -394,10 +408,11 @@ pload apply --offline ``` -Apply resolves or installs Python, obtains exact artifacts through the selected +Apply requires a current `.pload_lock.toml`, then installs Python when needed, +obtains exact artifacts through the selected resource routes, verifies hashes, creates the environment, installs packages and -runs `pip check`. Upload, download and copying are internal plan steps rather than -separate user commands. Reapplying the same configuration is idempotent; an +runs `pip check`. It never silently resolves or updates a stale lock. Reapplying +the same configuration is idempotent; an existing environment with different state is never silently overwritten. """ GUIDES[("repo",)] = r""" diff --git a/tests/test_declarative.py b/tests/test_declarative.py index 89e5857..cab5a2b 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -90,7 +90,7 @@ def test_manifest_roundtrip(tmp_path): assert locked["package"] == data["package"] -def test_plan_migrates_legacy_embedded_lock_without_resolving(tmp_path, monkeypatch): +def test_lock_migrates_legacy_embedded_lock_without_resolving(tmp_path, monkeypatch): data = simple_manifest("exact") path = tmp_path / "pload.toml" legacy = dump_manifest(data) + """ @@ -105,9 +105,9 @@ def test_plan_migrates_legacy_embedded_lock_without_resolving(tmp_path, monkeypa lambda *args, **kwargs: pytest.fail("legacy migration invoked the resolver"), ) - plan = DeclarativeEnvironmentManager(ConfigManager(home=tmp_path / "home")).plan(path) + result = DeclarativeEnvironmentManager(ConfigManager(home=tmp_path / "home")).lock(path) - assert plan["lock"]["migrated"] is True + assert result["migrated"] is True assert "[[package]]" not in path.read_text(encoding="utf-8") _, configuration = load_manifest(path) _, locked, current = load_lock(path, configuration) @@ -264,7 +264,7 @@ def reject_package_network(command, *args, **kwargs): assert output == "42" -def test_plan_locks_unpinned_dependencies_once_then_applies_from_cache(tmp_path, monkeypatch): +def test_lock_resolves_once_and_plan_remains_read_only(tmp_path, monkeypatch): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) wheel = tiny_wheel(tmp_path / "fixture") @@ -297,9 +297,20 @@ def fake_resolver(command, *args, **kwargs): return original_execute(command, *args, **kwargs) monkeypatch.setattr(declarative_module, "execute", fake_resolver) + before = manifest.read_bytes() + pending = manager.plan(manifest) + assert pending["lock"]["status"] == "missing" + assert pending["lock"]["required"] is True + assert pending["packages"][0]["selected"]["method"] == "lock-required" + assert download_calls == [] + assert manifest.read_bytes() == before + assert not lock_path(manifest).exists() + + result = manager.lock(manifest) + assert result["requested"] == ["pload-demo>=1"] + assert result["resolved"] == ["pload-demo==1.0"] first = manager.plan(manifest) - assert first["lock"]["requested"] == ["pload-demo>=1"] - assert first["lock"]["resolved"] == ["pload-demo==1.0"] + assert first["lock"]["status"] == "current" assert first["packages"][0]["selected"]["method"] == "cache" _, configuration = load_manifest(manifest) assert configuration["environment"]["dependencies"] == ["pload-demo>=1"] @@ -309,7 +320,7 @@ def fake_resolver(command, *args, **kwargs): assert locked["package"][0]["artifact"][0]["filename"] == wheel.name second = manager.plan(manifest) - assert second["lock"] is None + assert second["lock"]["status"] == "current" assert len(download_calls) == 1 restored = manager.apply(manifest) assert len(download_calls) == 1 @@ -352,14 +363,25 @@ def fake_resolver(command, *args, **kwargs): return "" monkeypatch.setattr(declarative_module, "execute", fake_resolver) + before_manifest = manifest.read_bytes() + before_lock = lock_path(manifest).read_bytes() + plan = manager.plan(manifest) + assert plan["lock"]["status"] == "stale" + assert plan["lock"]["required"] is True + assert calls == [] + assert manifest.read_bytes() == before_manifest + assert lock_path(manifest).read_bytes() == before_lock + with pytest.raises(PloadError, match="pload lock"): + manager.apply(manifest) + progress = [] - plan = manager.plan(manifest, progress=progress.append) + result = manager.lock(manifest, progress=progress.append) assert len(calls) == 1 assert progress == [ f"Configuration changed; resolving dependencies for Python {platform.python_version()}" ] - assert plan["lock"]["requested"] == ["pload-demo==1.0", "extra-demo"] - assert plan["lock"]["resolved"] == ["extra-demo==1.0", "pload-demo==1.0"] + assert result["requested"] == ["pload-demo==1.0", "extra-demo"] + assert result["resolved"] == ["extra-demo==1.0", "pload-demo==1.0"] _, configuration = load_manifest(manifest) assert configuration["environment"]["dependencies"] == [ "pload-demo==1.0", "extra-demo", @@ -485,8 +507,9 @@ def test_declarative_commands_and_shell_integration(capsys): "describe", "v1", "-s", "torch=https://download.pytorch.org/whl/cu121", ]) assert parsed.package_sources == ["torch=https://download.pytorch.org/whl/cu121"] + assert parser.parse_args(["lock"]).file == "pload.toml" assert parser.parse_args(["apply"]).file == "pload.toml" - for command in ("describe", "plan", "apply"): + for command in ("describe", "lock", "plan", "apply"): assert main([command, "-h"]) == 0 for shell in ("bash", "zsh", "fish", "powershell"): assert command in shell_script(shell) From fc7fe5f0e2c37aba8c0c16749535ec5daae68d26 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 17:07:18 +0800 Subject: [PATCH 18/41] Refactor reproduction into strict resource plans --- src/pload/cli.py | 4 + src/pload/declarative.py | 303 +++++++++++++++++++++++++++++++------ src/pload/managers/venv.py | 15 +- src/pload/resource_plan.py | 104 +++++++++++++ src/pload/snapshots.py | 14 +- tests/test_declarative.py | 170 ++++++++++++++++++++- 6 files changed, 555 insertions(+), 55 deletions(-) create mode 100644 src/pload/resource_plan.py diff --git a/src/pload/cli.py b/src/pload/cli.py index cde92f8..8fa7994 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -486,6 +486,10 @@ def print_declarative_plan(plan): table.add_row(package["name"], package["version"], "—", "unavailable", "No resource satisfies the configuration") console.print(table) + console.print( + "[dim]Saved acquisition choices:[/] " + f"[cyan]{plan['plan_path']}[/]" + ) def _print_spaced_section(console, renderable): diff --git a/src/pload/declarative.py b/src/pload/declarative.py index e29d2c3..af0644d 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -7,8 +7,10 @@ import re import shlex import shutil +import sys import tempfile import uuid +import zipfile from pathlib import Path from urllib.parse import urlsplit @@ -25,6 +27,7 @@ from pload.managers.platform import PythonNotFoundError from pload.managers.pyversion import PythonManager from pload.managers.venv import VenvManager +from pload.resource_plan import file_digest, load_plan, write_plan from pload.snapshots import RepositoryManager, digest, execute, probe, public_index, safe_name PIN = re.compile(r"([A-Za-z0-9][A-Za-z0-9_.-]*)==([A-Za-z0-9][A-Za-z0-9_.+!-]*)") @@ -532,6 +535,67 @@ def __init__(self, config): self.config = config self.cache = config.home / "cache" / "wheels" + def _external_cache_roots(self): + """Return package cache roots without creating or changing them.""" + configured = self.config.settings.get("resource_cache_dirs") + if configured is not None: + values = configured + elif os.environ.get("PLOAD_RESOURCE_CACHE_DIRS"): + values = os.environ["PLOAD_RESOURCE_CACHE_DIRS"].split(os.pathsep) + elif sys.platform == "win32": + local = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData/Local")) + values = [local / "pip/Cache", local / "uv/cache"] + elif sys.platform == "darwin": + values = [Path.home() / "Library/Caches/pip", Path.home() / "Library/Caches/uv"] + else: + cache_home = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + values = [cache_home / "pip", cache_home / "uv"] + roots = [] + for value in values: + root = Path(value).expanduser() + if root.is_dir() and root.resolve() != self.cache.resolve(): + roots.append(root.resolve()) + return list(dict.fromkeys(roots)) + + @staticmethod + def _find_external_artifact(roots, artifact): + """Find an exact wheel in known caches; matching a name alone is insufficient.""" + for root in roots: + try: + matches = root.rglob(artifact["filename"]) + for candidate in matches: + if candidate.is_file() and digest(candidate) == artifact["sha256"]: + return candidate + except OSError: + continue + return None + + def _environment_resources(self, expected, progress=None): + """Index exact, safely copyable distributions in compatible pload environments.""" + resources = {} + for environment in VenvManager(self.config).environments(register_discovered=False): + root = Path(environment["path"]) + interpreter = root / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + try: + runtime = probe(interpreter) + except PloadError: + continue + if any(runtime.get(key) != expected.get(key) for key in + ("python", "implementation", "system", "machine")): + continue + if progress: + progress(f"Inspecting packages in environment {environment['id']}") + for package in runtime.get("packages", []): + if not package.get("copyable") or not package.get("record_sha256"): + continue + key = (normalized_name(package["name"]), package["version"]) + resources.setdefault(key, []).append({ + "path": str(root), + "fingerprint": package["record_sha256"], + "id": environment["id"], + }) + return resources + def describe( self, source, output, name=None, mode="exact", repository=None, sources=None, progress=None, @@ -937,10 +1001,15 @@ def plan(self, manifest_path, offline=False, progress=None): "alternatives": [], "rejections": [], }) - return { + result = { "path": str(path), "name": data["name"], "python": python_action, "packages": pending, "ready": False, "lock": lock, } + result["plan_path"] = str(write_plan( + path, configuration_digest(data), + file_digest(sidecar) if sidecar.is_file() else "", result, + )) + return result repository_checksums = {} for package in packages: for artifact in package.get("artifact", []): @@ -964,6 +1033,10 @@ def plan(self, manifest_path, offline=False, progress=None): ) except PloadError: repository_objects[repo_name] = set() + external_cache_roots = self._external_cache_roots() + environment_resources = self._environment_resources( + data["environment"], progress=progress, + ) plans = [] for package in packages: candidates = [] @@ -990,6 +1063,11 @@ def plan(self, manifest_path, offline=False, progress=None): candidates.append(self._candidate( "configuration-artifact", adjacent, "ready", (0, 0, 0, 1), artifact )) + external = self._find_external_artifact(external_cache_roots, artifact) + if external: + candidates.append(self._candidate( + "external-cache", external, "ready", (0, 0, 0, 2), artifact + )) repo_names = dict.fromkeys( artifact.get("repositories", []) + preferred_repositories ) @@ -1008,6 +1086,14 @@ def plan(self, manifest_path, offline=False, progress=None): "index-exact", source_name, "network", (0, 0, 2, source_name), artifact, )) + environment_key = (normalized_name(package["name"]), package["version"]) + for resource in environment_resources.get(environment_key, []): + candidate = self._candidate( + "environment-copy", resource["path"], "ready", + (0, 0, 0, 3, resource["id"]), + ) + candidate["fingerprint"] = resource["fingerprint"] + candidates.append(candidate) if policy.get("reproducibility") == "compatible" and network_allowed: source_name = (package.get("sources") or ["default"])[0] candidates.append(self._candidate( @@ -1022,13 +1108,27 @@ def plan(self, manifest_path, offline=False, progress=None): }) ready = (not lock_required and python_action["status"] != "unavailable" and all(item["selected"] for item in plans)) - return {"path": str(path), "name": data["name"], "python": python_action, - "packages": plans, "ready": ready, "lock": lock} + result = {"path": str(path), "name": data["name"], "python": python_action, + "packages": plans, "ready": ready, "lock": lock} + result["plan_path"] = str(write_plan( + path, configuration_digest(data), file_digest(sidecar), result, + )) + return result @staticmethod def _candidate(method, location, status, cost, artifact=None): result = {"method": method, "location": str(location), "status": status, "cost": list(cost)} + result["estimated_seconds"] = { + "cache": 0.05, + "configuration-artifact": 0.1, + "external-cache": 0.2, + "environment-copy": 0.5, + "repository": 5.0, + "index-exact": 30.0, + "index-resolve": 45.0, + "lock-required": 0.0, + }.get(method, 0.0) if artifact: result["artifact"] = artifact return result @@ -1047,13 +1147,39 @@ def _locked_packages(data): return result def apply(self, manifest_path, name=None, offline=False, progress=None): - path, data, _, _ = self._load_state(manifest_path) - plan = self.plan(path, offline=offline, progress=progress) - if plan["lock"]["required"]: + path, data, current, _ = self._load_state(manifest_path) + if data["environment"].get("dependencies") and not current: raise PloadError( - "environment lock is missing or stale; review the configuration, then run " - f"'pload lock {path.name}'" + "environment lock is missing or stale; run pload plan after resolving metadata" ) + lock_file = lock_path(path) + _, saved = load_plan( + path, configuration_digest(data), + file_digest(lock_file) if lock_file.is_file() else "", + ) + self._validate_saved_plan(saved, data) + plan = { + "path": str(path), "name": saved["name"], "ready": saved["ready"], + "python": saved["python"], "packages": [], + } + for package in saved.get("package", []): + selected = None + if package["method"] != "unavailable": + selected = { + "method": package["method"], "location": package["location"], + "status": package["status"], + "estimated_seconds": package.get("estimated_seconds", 0.0), + } + if package.get("fingerprint"): + selected["fingerprint"] = package["fingerprint"] + if package.get("artifact"): + selected["artifact"] = package["artifact"] + plan["packages"].append({ + "name": package["name"], "version": package["version"], + "selected": selected, "alternatives": [], + "rejections": ([{"reason": package["reason"]}] + if package.get("reason") else []), + }) unavailable = [item for item in plan["packages"] if not item["selected"]] if unavailable: details = [] @@ -1081,8 +1207,8 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): return existing raise PloadError(f"environment {target_name!r} exists with different state") artifacts = [] - locked_files = [] compatible = [] + environment_copies = [] self.cache.mkdir(parents=True, exist_ok=True) repository_fetches = {} for package_plan in plan["packages"]: @@ -1103,7 +1229,10 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): ) except PloadError as exc: if progress: - progress(f"Batch fetch failed; trying fallback routes: {exc}") + progress( + "Batch fetch failed; retrying the same selected repository " + f"route individually: {exc}" + ) with tempfile.TemporaryDirectory(prefix="pload-apply-") as temporary: stage = Path(temporary) for index_number, package_plan in enumerate(plan["packages"], 1): @@ -1115,45 +1244,24 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): selected = package_plan["selected"] artifact = selected.get("artifact") if artifact: - routes = [selected] + package_plan["alternatives"] - failures = [] - acquired = None - for route in routes: - route_artifact = route.get("artifact") - if not route_artifact: - continue - cached = self.cache / route_artifact["filename"] + cached = self.cache / artifact["filename"] + if not (cached.is_file() and digest(cached) == artifact["sha256"]): try: - if not (cached.is_file() - and digest(cached) == route_artifact["sha256"]): - self._acquire(path, data, package_plan, route, cached, stage) - if digest(cached) != route_artifact["sha256"]: - raise PloadError("checksum mismatch") + self._acquire(path, data, package_plan, selected, cached, stage) except (OSError, PloadError) as exc: - failures.append(f"{route['method']}: {exc}") - continue - acquired = (cached, route_artifact) - break - if not acquired: - detail = failures[-1] if failures else "no artifact route" - raise PloadError( - f"cannot acquire {package_plan['name']}=={package_plan['version']}: {detail}" - ) - cached, artifact = acquired + raise PloadError( + f"planned route failed for {package_plan['name']}==" + f"{package_plan['version']} ({selected['method']}): {exc}; " + "run pload plan to choose another route" + ) from exc + if digest(cached) != artifact["sha256"]: + raise PloadError("planned artifact checksum mismatch") artifacts.append(cached.resolve().as_uri() + "#sha256=" + artifact["sha256"]) - locked_files.append((cached, artifact["sha256"])) else: - compatible.append(package_plan) - if (locked_files - and data["policy"].get("publish_missing_artifacts", True)): - for repo_name in data["policy"].get("repositories", []): - if progress: - progress( - f"Publishing {len(locked_files)} locked artifacts to {repo_name}" - ) - ArtifactRepository.publish_many( - data["repositories"][repo_name], locked_files, path.parent, - ) + if selected["method"] == "environment-copy": + environment_copies.append(package_plan) + else: + compatible.append(package_plan) venvs = VenvManager(self.config) if progress: progress(f"Creating environment {target_name}") @@ -1166,6 +1274,8 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): if artifacts: execute(pip + ["install", "--no-index", "--no-deps"] + artifacts, capture=False) + for package_plan in environment_copies: + self._copy_distribution(package_plan, env, stage) for package_plan in compatible: source_name = package_plan["selected"]["location"] source = data.get("sources", {}).get(source_name, {}) @@ -1197,10 +1307,113 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): venvs.remove_venv(target_name) raise + @staticmethod + def _validate_saved_plan(saved, data): + """Bind every executable route to one exact package in the current lock.""" + locked = { + (normalized_name(package["name"]), package["version"]): package + for package in DeclarativeEnvironmentManager._locked_packages(data) + } + planned = saved.get("package", []) + planned_keys = [ + (normalized_name(package.get("name", "")), package.get("version", "")) + for package in planned + ] + if len(planned_keys) != len(set(planned_keys)) or set(planned_keys) != set(locked): + raise PloadError("acquisition plan packages do not match the current lock") + artifact_methods = { + "cache", "configuration-artifact", "external-cache", + "repository", "index-exact", + } + allowed_methods = artifact_methods | { + "environment-copy", "index-resolve", "unavailable", + } + for package, key in zip(planned, planned_keys): + method = package.get("method") + if method not in allowed_methods: + raise PloadError(f"unsupported method in acquisition plan: {method}") + artifact = package.get("artifact") + if method in artifact_methods: + allowed = { + (item["filename"], item["sha256"]) + for item in locked[key].get("artifact", []) + } + identity = ( + artifact.get("filename"), artifact.get("sha256") + ) if isinstance(artifact, dict) else None + if identity not in allowed: + raise PloadError( + f"planned artifact is not present in the lock for " + f"{package['name']}=={package['version']}" + ) + elif artifact: + raise PloadError(f"method {method} must not contain a wheel artifact") + if method == "environment-copy" and not package.get("fingerprint"): + raise PloadError("environment-copy route has no package fingerprint") + if (method == "index-resolve" + and data["policy"].get("reproducibility") != "compatible"): + raise PloadError("index-resolve is forbidden by exact reproducibility policy") + + def _copy_distribution(self, package, target_environment, stage): + selected = package["selected"] + source_environment = Path(selected["location"]) + source_python = source_environment / ( + "Scripts/python.exe" if os.name == "nt" else "bin/python" + ) + archive = stage / (normalized_name(package["name"]) + ".zip") + script = r''' +import hashlib, json, os, sys, sysconfig, zipfile +from importlib import metadata +name, version, fingerprint, output = sys.argv[1:] +dist = metadata.distribution(name) +record = dist.read_text('RECORD') or '' +if dist.version != version or hashlib.sha256(record.encode()).hexdigest() != fingerprint: + raise SystemExit('source distribution changed after planning') +roots = [os.path.realpath(sysconfig.get_path(key)) for key in ('purelib', 'platlib') + if sysconfig.get_path(key)] +files = [] +for item in dist.files or []: + source = os.path.realpath(str(dist.locate_file(item))) + match = next((root for root in roots + if source == root or source.startswith(root + os.sep)), None) + if not match: + raise SystemExit('distribution contains files outside site-packages') + relative = os.path.relpath(source, match) + if relative.startswith('..' + os.sep) or os.path.isabs(relative): + raise SystemExit('unsafe distribution path') + if os.path.isfile(source): + files.append((source, relative)) +with zipfile.ZipFile(output, 'w', compression=zipfile.ZIP_DEFLATED) as bundle: + for source, relative in files: + bundle.write(source, relative) +''' + execute([ + str(source_python), "-c", script, package["name"], package["version"], + selected["fingerprint"], str(archive), + ]) + target_python = Path(target_environment) / ( + "Scripts/python.exe" if os.name == "nt" else "bin/python" + ) + site_packages = Path(execute([ + str(target_python), "-c", + "import sysconfig; print(sysconfig.get_path('purelib'))", + ])) + with zipfile.ZipFile(archive) as bundle: + for member in bundle.infolist(): + destination = (site_packages / member.filename).resolve() + if site_packages.resolve() not in destination.parents: + raise PloadError("environment package contains an unsafe path") + bundle.extractall(site_packages) + def _acquire(self, path, data, package, selected, destination, stage): artifact = selected["artifact"] if selected["method"] == "configuration-artifact": shutil.copyfile(selected["location"], destination) + elif selected["method"] == "external-cache": + source = Path(selected["location"]) + if not source.is_file() or digest(source) != artifact["sha256"]: + raise PloadError("selected external cache artifact changed or disappeared") + shutil.copyfile(source, destination) elif selected["method"] == "repository": ArtifactRepository.fetch( data["repositories"][selected["location"]], artifact["sha256"], diff --git a/src/pload/managers/venv.py b/src/pload/managers/venv.py index ee345ee..b139e84 100644 --- a/src/pload/managers/venv.py +++ b/src/pload/managers/venv.py @@ -163,8 +163,8 @@ def register_environment(self, path, name=None, description=None): self._write_registry(data) return entry - def environments(self): - """Return registered environments and import legacy managed environments.""" + def environments(self, register_discovered=True): + """Return known environments, optionally without importing discovered ones.""" data = self._read_registry() registered_paths = { str(Path(entry.get("path", "")).expanduser().resolve()) @@ -172,16 +172,23 @@ def environments(self): if entry.get("path") } root = self.config.venv_path + transient = [] if root.is_dir(): for child in sorted(root.iterdir(), key=lambda item: item.name.lower()): resolved = str(child.resolve()) if self._valid_environment(child) and resolved not in registered_paths: - self.register_environment(child, name=child.name) + if register_discovered: + self.register_environment(child, name=child.name) + else: + transient.append({ + "id": f"path:{child.name}", "name": child.name, + "path": resolved, "description": "", "created_at": "", + }) registered_paths.add(resolved) data = self._read_registry() result = [] - for entry in data["environments"]: + for entry in data["environments"] + transient: path = Path(entry.get("path", "")).expanduser() if self._valid_environment(path): item = dict(entry) diff --git a/src/pload/resource_plan.py b/src/pload/resource_plan.py new file mode 100644 index 0000000..8418355 --- /dev/null +++ b/src/pload/resource_plan.py @@ -0,0 +1,104 @@ +"""Persistent, auditable acquisition plans produced without acquiring packages.""" + +import hashlib +import json +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - Python 3.8 + import tomli as tomllib + +from pload.errors import PloadError + + +def plan_path(configuration_path): + return Path(configuration_path).parent / ".pload_plan.toml" + + +def file_digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def _quoted(value): + return json.dumps(str(value), ensure_ascii=False) + + +def dump_plan(configuration_path, configuration_sha256, lock_sha256, plan): + python = plan["python"] + lines = [ + "schema = 1", + f"configuration = {_quoted(Path(configuration_path).name)}", + f"configuration_sha256 = {_quoted(configuration_sha256)}", + f"lock_sha256 = {_quoted(lock_sha256)}", + f"name = {_quoted(plan['name'])}", + f"ready = {'true' if plan['ready'] else 'false'}", + "", + "[python]", + f"method = {_quoted(python['method'])}", + f"location = {_quoted(python['location'])}", + f"status = {_quoted(python['status'])}", + ] + for package in plan.get("packages", []): + selected = package.get("selected") or {} + rejection = (package.get("rejections") or [{}])[0] + reason = rejection.get("reason", "no resource satisfies the configuration") + lines += [ + "", + "[[package]]", + f"name = {_quoted(package['name'])}", + f"version = {_quoted(package['version'])}", + f"method = {_quoted(selected.get('method', 'unavailable'))}", + f"location = {_quoted(selected.get('location', ''))}", + f"status = {_quoted(selected.get('status', 'unavailable'))}", + f"estimated_seconds = {float(selected.get('estimated_seconds', 0.0))}", + ] + if not selected: + lines.append(f"reason = {_quoted(reason)}") + if selected.get("fingerprint"): + lines.append(f"fingerprint = {_quoted(selected['fingerprint'])}") + artifact = selected.get("artifact") + if artifact: + lines += [ + "", + "[package.artifact]", + f"filename = {_quoted(artifact['filename'])}", + f"sha256 = {_quoted(artifact['sha256'])}", + ] + return "\n".join(lines) + "\n" + + +def write_plan(configuration_path, configuration_sha256, lock_sha256, plan): + destination = plan_path(configuration_path) + temporary = destination.with_name(destination.name + ".tmp") + temporary.write_text( + dump_plan(configuration_path, configuration_sha256, lock_sha256, plan), + encoding="utf-8", + ) + temporary.replace(destination) + return destination + + +def load_plan(configuration_path, configuration_sha256, lock_sha256): + path = plan_path(configuration_path) + try: + with path.open("rb") as stream: + data = tomllib.load(stream) + except FileNotFoundError as exc: + raise PloadError( + f"acquisition plan is missing; run 'pload plan {Path(configuration_path).name}'" + ) from exc + except (OSError, tomllib.TOMLDecodeError) as exc: + raise PloadError(f"cannot read acquisition plan {path}: {exc}") from exc + if data.get("schema") != 1: + raise PloadError("unsupported acquisition plan schema") + if ( + data.get("configuration") != Path(configuration_path).name + or data.get("configuration_sha256") != configuration_sha256 + or data.get("lock_sha256") != lock_sha256 + ): + raise PloadError("acquisition plan is stale; run pload plan again") + packages = data.get("package", []) + if not isinstance(packages, list): + raise PloadError("invalid acquisition plan packages") + return path, data diff --git a/src/pload/snapshots.py b/src/pload/snapshots.py index 7e01e90..1ebca15 100644 --- a/src/pload/snapshots.py +++ b/src/pload/snapshots.py @@ -21,14 +21,24 @@ from pload.settings import save_settings PROBE = """ -import json, platform, sys, os, shutil, subprocess +import hashlib, json, platform, sys, os, shutil, subprocess, sysconfig from importlib import metadata packages = [] +library_roots = [os.path.realpath(sysconfig.get_path(key)) + for key in ('purelib', 'platlib') if sysconfig.get_path(key)] for dist in metadata.distributions(): name = dist.metadata.get('Name') if name and name.lower() not in ('pip', 'setuptools', 'wheel'): + record = dist.read_text('RECORD') or '' + locations = [os.path.realpath(str(dist.locate_file(item))) + for item in (dist.files or [])] + copyable = bool(locations) and all(any( + location == root or location.startswith(root + os.sep) + for root in library_roots) for location in locations) packages.append({'name': name, 'version': dist.version, - 'direct_url': dist.read_text('direct_url.json')}) + 'direct_url': dist.read_text('direct_url.json'), + 'record_sha256': hashlib.sha256(record.encode()).hexdigest(), + 'copyable': copyable}) accelerators = [] nvidia_smi = shutil.which('nvidia-smi') if nvidia_smi: diff --git a/tests/test_declarative.py b/tests/test_declarative.py index cab5a2b..aa66dee 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -21,6 +21,7 @@ from pload.errors import PloadError from pload.managers.platform import ConfigManager from pload.managers.venv import VenvManager +from pload.resource_plan import file_digest, write_plan from pload.settings import save_settings from pload.snapshots import digest, execute @@ -264,6 +265,99 @@ def reject_package_network(command, *args, **kwargs): assert output == "42" +def test_plan_discovers_and_apply_uses_exact_external_cache(tmp_path, monkeypatch): + external_cache = tmp_path / "pip-cache" + wheel = tiny_wheel(external_cache) + config_home = tmp_path / "home" + save_settings(config_home, {"resource_cache_dirs": [str(external_cache)]}) + config = ConfigManager(home=config_home) + config.get_python_path = lambda version=None: Path(sys.executable) + data = simple_manifest("exact") + data["name"] = "external-cache" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + }], + }] + manifest = tmp_path / "project" / "pload.toml" + manifest.parent.mkdir() + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + + selected = manager.plan(manifest)["packages"][0]["selected"] + assert selected["method"] == "external-cache" + assert selected["location"] == str(wheel.resolve()) + + original_execute = declarative_module.execute + + def reject_download(command, *args, **kwargs): + if "download" in command: + pytest.fail("external-cache route attempted a download") + return original_execute(command, *args, **kwargs) + + monkeypatch.setattr(declarative_module, "execute", reject_download) + restored = manager.apply(manifest) + assert digest(manager.cache / wheel.name) == digest(wheel) + assert original_execute([ + config.get_pip_command(restored)[0], "-c", + "import pload_demo; print(pload_demo.answer)", + ]) == "42" + + +def test_plan_reuses_package_from_compatible_pload_environment(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + source = VenvManager(config).create_venv(name="package-source") + original_execute = declarative_module.execute + original_execute(config.get_pip_command(source) + [ + "install", "--no-index", "--find-links", str(wheel.parent), "pload-demo==1.0", + ]) + data = simple_manifest("exact") + data["name"] = "environment-copy" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [], + }] + manifest = tmp_path / "project" / "pload.toml" + manifest.parent.mkdir() + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + + selected = manager.plan(manifest)["packages"][0]["selected"] + assert selected["method"] == "environment-copy" + assert selected["location"] == str(source) + assert len(selected["fingerprint"]) == 64 + + def reject_package_network(command, *args, **kwargs): + if "download" in command or ("install" in command and "--no-index" not in command): + pytest.fail("environment-copy route attempted package network access") + return original_execute(command, *args, **kwargs) + + monkeypatch.setattr(declarative_module, "execute", reject_package_network) + restored = manager.apply(manifest) + assert original_execute([ + config.get_pip_command(restored)[0], "-c", + "import pload_demo; print(pload_demo.answer)", + ]) == "42" + + def test_lock_resolves_once_and_plan_remains_read_only(tmp_path, monkeypatch): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) @@ -371,7 +465,7 @@ def fake_resolver(command, *args, **kwargs): assert calls == [] assert manifest.read_bytes() == before_manifest assert lock_path(manifest).read_bytes() == before_lock - with pytest.raises(PloadError, match="pload lock"): + with pytest.raises(PloadError, match="missing or stale"): manager.apply(manifest) progress = [] @@ -468,7 +562,7 @@ def test_apply_rolls_back_a_new_environment_after_install_failure(tmp_path): VenvManager(config).resolve_existing("demo") -def test_apply_publishes_acquired_artifacts_to_policy_repository(tmp_path): +def test_apply_requires_a_saved_plan_and_never_auto_publishes(tmp_path): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) wheel = tiny_wheel(tmp_path / "artifacts") @@ -496,8 +590,76 @@ def test_apply_publishes_acquired_artifacts_to_policy_repository(tmp_path): }] manifest = tmp_path / "pload.toml" write_configuration(manifest, data) - DeclarativeEnvironmentManager(config).apply(manifest) - assert (repository / "objects" / checksum).is_file() + manager = DeclarativeEnvironmentManager(config) + with pytest.raises(PloadError, match="plan is missing"): + manager.apply(manifest) + manager.plan(manifest) + manager.apply(manifest) + assert not (repository / "objects" / checksum).exists() + + +def test_apply_obeys_selected_route_without_fallback(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "artifacts") + data = simple_manifest("exact") + data["name"] = "strict-route" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + }], + }] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + manager.plan(manifest) + (manager.cache / wheel.name).unlink() + + calls = [] + + def fail_selected(*args): + calls.append(args[3]["method"]) + raise PloadError("selected cache vanished") + + monkeypatch.setattr(manager, "_acquire", fail_selected) + with pytest.raises(PloadError, match="planned route failed.*cache"): + manager.apply(manifest) + assert calls == ["cache"] + + +def test_apply_rejects_plan_that_omits_a_locked_package(tmp_path): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + }) + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + generated = manager.plan(manifest) + generated["packages"] = [] + generated["ready"] = True + write_plan( + manifest, declarative_module.configuration_digest(data), + file_digest(lock_path(manifest)), generated, + ) + with pytest.raises(PloadError, match="do not match the current lock"): + manager.apply(manifest) def test_declarative_commands_and_shell_integration(capsys): From 5dd00ef4255f186c20f4e2bea156677dab3ff38c Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 17:13:53 +0800 Subject: [PATCH 19/41] Resolve dependency locks from metadata only --- pyproject.toml | 3 +- src/pload/cli.py | 4 +- src/pload/declarative.py | 127 +++++++++++---- src/pload/metadata_resolver.py | 280 ++++++++++++++++++++++++++++++++ src/pload/resource_plan.py | 8 + tests/test_declarative.py | 144 +++++++++------- tests/test_metadata_resolver.py | 86 ++++++++++ 7 files changed, 555 insertions(+), 97 deletions(-) create mode 100644 src/pload/metadata_resolver.py create mode 100644 tests/test_metadata_resolver.py diff --git a/pyproject.toml b/pyproject.toml index 17ac31a..74d753f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,8 +21,9 @@ classifiers = [ "Programming Language :: Python :: 3 :: Only", ] dependencies = [ - "packaging>=23.2,<26", + "packaging>=23.2,<27", "rich>=13.7,<15", + "resolvelib>=1.1,<1.2", "tomli>=2; python_version < '3.11'", ] diff --git a/src/pload/cli.py b/src/pload/cli.py index 8fa7994..74b4fcf 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -456,8 +456,8 @@ def print_declarative_plan(plan): console = Console(highlight=False) if plan["lock"]["required"]: console.print( - f"[bold yellow]Lock: {plan['lock']['status']}[/] · plan is read-only; " - "run [bold green]pload lock[/] to resolve dependencies" + f"[bold yellow]Lock: {plan['lock']['status']}[/] · metadata was unavailable " + "offline; run [bold green]pload plan[/] with network access" ) python = plan["python"] console.print(Panel.fit( diff --git a/src/pload/declarative.py b/src/pload/declarative.py index af0644d..01906ce 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -13,6 +13,7 @@ import zipfile from pathlib import Path from urllib.parse import urlsplit +from urllib.request import Request, urlopen from packaging.requirements import InvalidRequirement, Requirement from packaging.tags import compatible_tags, cpython_tags, platform_tags @@ -27,6 +28,7 @@ from pload.managers.platform import PythonNotFoundError from pload.managers.pyversion import PythonManager from pload.managers.venv import VenvManager +from pload.metadata_resolver import resolve_metadata from pload.resource_plan import file_digest, load_plan, write_plan from pload.snapshots import RepositoryManager, digest, execute, probe, public_index, safe_name @@ -193,6 +195,14 @@ def dump_lock(manifest_path, data): f"tags = {_array(artifact.get('tags', []))}", f"repositories = {_array(artifact.get('repositories', []))}", ] + if artifact.get("url"): + lines.append(f"url = {_quoted(artifact['url'])}") + if artifact.get("size") is not None: + lines.append(f"size = {int(artifact['size'])}") + if artifact.get("metadata_sha256"): + lines.append( + f"metadata_sha256 = {_quoted(artifact['metadata_sha256'])}" + ) return "\n".join(lines) + "\n" @@ -325,6 +335,13 @@ def validate_manifest(data): f"artifact {artifact['filename']} references unknown repository " f"{repository_name}" ) + if artifact.get("url"): + parsed_artifact_url = urlsplit(artifact["url"]) + if (parsed_artifact_url.scheme not in {"http", "https"} + or not parsed_artifact_url.netloc + or parsed_artifact_url.username + or parsed_artifact_url.query): + raise PloadError(f"invalid public artifact URL for {package['name']}") # A dependency edit can intentionally make the existing package lock stale. # Planning reconciles and rewrites it; validation only checks each section's # shape and internal safety. @@ -988,28 +1005,52 @@ def plan(self, manifest_path, offline=False, progress=None): "status": "network" if network_allowed else "unavailable", } if lock_required: - pending = [] - for value in dependencies: - requirement = _parse_dependency(value) - exact = PIN.fullmatch(value) - pending.append({ - "name": requirement.name, - "version": exact.group(2) if exact else str(requirement.specifier) or "unresolved", - "selected": self._candidate( - "lock-required", sidecar, "pending", (0, 0, 0, 0) - ), - "alternatives": [], - "rejections": [], - }) - result = { - "path": str(path), "name": data["name"], "python": python_action, - "packages": pending, "ready": False, "lock": lock, - } - result["plan_path"] = str(write_plan( - path, configuration_digest(data), - file_digest(sidecar) if sidecar.is_file() else "", result, - )) - return result + if network_allowed: + sources = [ + (name, source["url"]) for name, source in data.get("sources", {}).items() + if source.get("kind") == "index" + ] + if not sources: + raise PloadError("metadata resolution requires at least one index source") + if progress: + progress( + f"Resolving dependency metadata for Python " + f"{data['environment']['python']}" + ) + packages = resolve_metadata( + dependencies, sources, data["environment"], supported_tags, + ) + data["package"] = packages + data["resolved_dependencies"] = [ + f"{package['name']}=={package['version']}" for package in packages + ] + self._write_lock(path, data) + current = True + lock_required = False + lock.update({"status": "generated", "required": False}) + else: + pending = [] + for value in dependencies: + requirement = _parse_dependency(value) + exact = PIN.fullmatch(value) + pending.append({ + "name": requirement.name, + "version": exact.group(2) if exact else str(requirement.specifier) or "unresolved", + "selected": self._candidate( + "lock-required", sidecar, "pending", (0, 0, 0, 0) + ), + "alternatives": [], + "rejections": [], + }) + result = { + "path": str(path), "name": data["name"], "python": python_action, + "packages": pending, "ready": False, "lock": lock, + } + result["plan_path"] = str(write_plan( + path, configuration_digest(data), + file_digest(sidecar) if sidecar.is_file() else "", result, + )) + return result repository_checksums = {} for package in packages: for artifact in package.get("artifact", []): @@ -1420,19 +1461,35 @@ def _acquire(self, path, data, package, selected, destination, stage): destination, path.parent, ) elif selected["method"] == "index-exact": - source = data["sources"][selected["location"]] - download = stage / normalized_name(package["name"]) - download.mkdir(exist_ok=True) - execute([ - str(self.config.get_python_path(data["environment"]["python"])), - "-m", "pip", "download", "--only-binary=:all:", "--no-deps", - "--dest", str(download), "--index-url", source["url"], - f"{package['name']}=={package['version']}", - ]) - candidate = download / artifact["filename"] - if not candidate.is_file(): - raise PloadError(f"index did not provide locked artifact {artifact['filename']}") - shutil.copyfile(candidate, destination) + if artifact.get("url"): + temporary = stage / artifact["filename"] + try: + request = Request( + artifact["url"], headers={"User-Agent": "pload-apply/1"}, + ) + with urlopen(request, timeout=60) as response, temporary.open("wb") as output: + shutil.copyfileobj(response, output) + except OSError as exc: + raise PloadError(f"cannot download planned artifact: {exc}") from exc + if digest(temporary) != artifact["sha256"]: + raise PloadError("downloaded artifact does not match the lock checksum") + shutil.copyfile(temporary, destination) + else: + source = data["sources"][selected["location"]] + download = stage / normalized_name(package["name"]) + download.mkdir(exist_ok=True) + execute([ + str(self.config.get_python_path(data["environment"]["python"])), + "-m", "pip", "download", "--only-binary=:all:", "--no-deps", + "--dest", str(download), "--index-url", source["url"], + f"{package['name']}=={package['version']}", + ]) + candidate = download / artifact["filename"] + if not candidate.is_file(): + raise PloadError( + f"index did not provide locked artifact {artifact['filename']}" + ) + shutil.copyfile(candidate, destination) else: raise PloadError(f"unsupported acquisition method: {selected['method']}") diff --git a/src/pload/metadata_resolver.py b/src/pload/metadata_resolver.py new file mode 100644 index 0000000..4af06f1 --- /dev/null +++ b/src/pload/metadata_resolver.py @@ -0,0 +1,280 @@ +"""Resolve exact wheel graphs using index metadata, never distribution bodies.""" + +import hashlib +import json +import platform +import sys +from dataclasses import dataclass +from email.parser import BytesParser +from email.policy import compat32 +from html.parser import HTMLParser +from urllib.error import HTTPError, URLError +from urllib.parse import unquote, urljoin, urlsplit +from urllib.request import Request, urlopen + +from packaging.markers import default_environment +from packaging.requirements import InvalidRequirement, Requirement +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.utils import InvalidWheelFilename, canonicalize_name, parse_wheel_filename +from packaging.version import InvalidVersion, Version +from resolvelib import BaseReporter, Resolver +from resolvelib.resolvers import ResolutionImpossible + +from pload.errors import PloadError + +MAX_INDEX_BYTES = 16 * 1024 * 1024 +MAX_METADATA_BYTES = 8 * 1024 * 1024 + + +def _read_url(url, accept, maximum): + request = Request(url, headers={"Accept": accept, "User-Agent": "pload-metadata/1"}) + try: + with urlopen(request, timeout=30) as response: + length = response.headers.get("Content-Length") + if length and int(length) > maximum: + raise PloadError(f"metadata response is too large: {url}") + content = response.read(maximum + 1) + if len(content) > maximum: + raise PloadError(f"metadata response is too large: {url}") + return content, response.headers.get_content_type() + except (HTTPError, URLError, OSError, ValueError) as exc: + raise PloadError(f"cannot read package metadata from {url}: {exc}") from exc + + +class _SimpleHTML(HTMLParser): + def __init__(self): + super().__init__() + self.files = [] + + def handle_starttag(self, tag, attrs): + if tag.lower() != "a": + return + values = dict(attrs) + href = values.get("href") + if not href: + return + metadata = values.get("data-core-metadata") + if metadata is None: + metadata = values.get("data-dist-info-metadata") + self.files.append({ + "url": href, + "filename": unquote(urlsplit(href).path.rsplit("/", 1)[-1]), + "hashes": _fragment_hashes(urlsplit(href).fragment), + "requires-python": values.get("data-requires-python"), + "core-metadata": _metadata_hashes(metadata), + "yanked": "data-yanked" in values, + "size": None, + }) + + +def _fragment_hashes(fragment): + if "=" not in fragment: + return {} + algorithm, value = fragment.split("=", 1) + return {algorithm.lower(): value} + + +def _metadata_hashes(value): + if value is None or value is False: + return None + if value is True or str(value).lower() == "true": + return {} + if isinstance(value, dict): + return value + return _fragment_hashes(str(value)) + + +@dataclass(frozen=True) +class MetadataCandidate: + name: str + version: Version + source: str + artifact_url: str + filename: str + sha256: str + size: int + metadata_sha256: str + + +class MetadataIndex: + def __init__(self, sources, supported_tags, python_version, fetch=None): + self.sources = sources + self.supported_tags = set(supported_tags) + self.python_version = Version(python_version) + self.fetch = fetch or _read_url + self.project_cache = {} + self.dependency_cache = {} + + def candidates(self, name): + key = canonicalize_name(name) + if key in self.project_cache: + return self.project_cache[key] + by_version = {} + metadata_missing = [] + source_errors = [] + for source_name, base_url in self.sources: + project_url = base_url.rstrip("/") + "/" + key + "/" + try: + body, content_type = self.fetch( + project_url, + "application/vnd.pypi.simple.v1+json, text/html;q=0.2", + MAX_INDEX_BYTES, + ) + except PloadError as exc: + source_errors.append(f"{source_name}: {exc}") + continue + if content_type == "application/vnd.pypi.simple.v1+json" or body.lstrip().startswith(b"{"): + try: + files = json.loads(body.decode("utf-8"))["files"] + except (KeyError, TypeError, ValueError, UnicodeDecodeError) as exc: + raise PloadError(f"invalid Simple API response for {name}") from exc + else: + parser = _SimpleHTML() + try: + parser.feed(body.decode("utf-8")) + except UnicodeDecodeError as exc: + raise PloadError(f"invalid Simple API HTML response for {name}") from exc + files = parser.files + for record in files: + filename = record.get("filename", "") + try: + project, version, _, tags = parse_wheel_filename(filename) + except (InvalidWheelFilename, InvalidVersion): + continue + if canonicalize_name(project) != key or not tags.intersection(self.supported_tags): + continue + if record.get("yanked"): + continue + requires_python = record.get("requires-python") + if requires_python: + try: + if not SpecifierSet(requires_python).contains(self.python_version): + continue + except InvalidSpecifier: + continue + sha256 = (record.get("hashes") or {}).get("sha256") + raw_metadata = record.get("core-metadata") + if raw_metadata is None: + raw_metadata = record.get("dist-info-metadata") + metadata_hashes = _metadata_hashes(raw_metadata) + if not sha256: + continue + if metadata_hashes is None: + metadata_missing.append(filename) + continue + url = urljoin(project_url, record["url"]) + candidate = MetadataCandidate( + name=key, version=version, source=source_name, + artifact_url=url.split("#", 1)[0], filename=filename, + sha256=sha256, size=int(record.get("size") or 0), + metadata_sha256=metadata_hashes.get("sha256", ""), + ) + current = by_version.get(version) + if current is None or (candidate.size or sys.maxsize) < (current.size or sys.maxsize): + by_version[version] = candidate + if not by_version and metadata_missing: + raise PloadError( + f"index has compatible files for {name}, but does not expose independent " + "core metadata; refusing to download a wheel during plan. Add a " + "metadata-capable source such as https://pypi.org/simple" + ) + if not by_version and source_errors: + raise PloadError( + f"no configured index could provide metadata for {name}: " + + "; ".join(source_errors) + ) + result = sorted(by_version.values(), key=lambda item: item.version, reverse=True) + self.project_cache[key] = result + return result + + def dependencies(self, candidate, marker_environment): + if candidate in self.dependency_cache: + return self.dependency_cache[candidate] + url = candidate.artifact_url + ".metadata" + body, _ = self.fetch(url, "application/octet-stream", MAX_METADATA_BYTES) + if candidate.metadata_sha256: + actual = hashlib.sha256(body).hexdigest() + if actual != candidate.metadata_sha256: + raise PloadError(f"core metadata checksum mismatch for {candidate.filename}") + message = BytesParser(policy=compat32).parsebytes(body) + if canonicalize_name(message.get("Name", "")) != candidate.name: + raise PloadError(f"core metadata name mismatch for {candidate.filename}") + if message.get("Version") != str(candidate.version): + raise PloadError(f"core metadata version mismatch for {candidate.filename}") + requirements = [] + for value in message.get_all("Requires-Dist", []): + try: + requirement = Requirement(value) + except InvalidRequirement as exc: + raise PloadError(f"invalid Requires-Dist in {candidate.filename}: {value}") from exc + if requirement.url: + raise PloadError( + f"{candidate.filename} uses a direct-URL dependency that cannot be planned" + ) + if requirement.marker and not requirement.marker.evaluate(marker_environment): + continue + requirements.append(requirement) + self.dependency_cache[candidate] = requirements + return requirements + + +class _Provider: + def __init__(self, index, marker_environment): + self.index = index + self.marker_environment = marker_environment + + def identify(self, requirement_or_candidate): + return canonicalize_name(requirement_or_candidate.name) + + def get_preference(self, identifier, resolutions, candidates, information, backtrack_causes): + return (identifier not in resolutions, identifier) + + def find_matches(self, identifier, requirements, incompatibilities): + constraints = list(requirements[identifier]) + rejected = set(incompatibilities[identifier]) + return [ + candidate for candidate in self.index.candidates(identifier) + if candidate not in rejected + and all(requirement.specifier.contains(candidate.version) for requirement in constraints) + ] + + @staticmethod + def is_satisfied_by(requirement, candidate): + return requirement.specifier.contains(candidate.version) + + def get_dependencies(self, candidate): + return self.index.dependencies(candidate, self.marker_environment) + + +def resolve_metadata(requirements, sources, environment, supported_tags, fetch=None): + """Return exact packages and wheel metadata without fetching any wheel body.""" + parsed = [Requirement(value) for value in requirements] + marker_environment = default_environment() + python = environment["python"] + marker_environment.update({ + "python_full_version": python, + "python_version": ".".join(python.split(".")[:2]), + "implementation_name": environment.get("implementation", "cpython"), + "platform_system": environment.get("system") or platform.system(), + "platform_machine": environment.get("machine") or platform.machine(), + "extra": "", + }) + index = MetadataIndex(sources, supported_tags, python, fetch=fetch) + try: + result = Resolver(_Provider(index, marker_environment), BaseReporter()).resolve(parsed) + except ResolutionImpossible as exc: + raise PloadError("dependency constraints cannot be resolved from index metadata") from exc + packages = [] + for name, candidate in sorted(result.mapping.items()): + packages.append({ + "name": name, "version": str(candidate.version), + "sources": [candidate.source], + "artifact": [{ + "filename": candidate.filename, "sha256": candidate.sha256, + "tags": [str(tag) for tag in parse_wheel_filename(candidate.filename)[3]], + "repositories": [], "url": candidate.artifact_url, + "size": candidate.size, + "metadata_sha256": candidate.metadata_sha256, + }], + }) + return packages diff --git a/src/pload/resource_plan.py b/src/pload/resource_plan.py index 8418355..6193b31 100644 --- a/src/pload/resource_plan.py +++ b/src/pload/resource_plan.py @@ -65,6 +65,14 @@ def dump_plan(configuration_path, configuration_sha256, lock_sha256, plan): f"filename = {_quoted(artifact['filename'])}", f"sha256 = {_quoted(artifact['sha256'])}", ] + if artifact.get("url"): + lines.append(f"url = {_quoted(artifact['url'])}") + if artifact.get("size") is not None: + lines.append(f"size = {int(artifact['size'])}") + if artifact.get("metadata_sha256"): + lines.append( + f"metadata_sha256 = {_quoted(artifact['metadata_sha256'])}" + ) return "\n".join(lines) + "\n" diff --git a/tests/test_declarative.py b/tests/test_declarative.py index aa66dee..e3ce952 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -1,3 +1,4 @@ +import io import platform import shutil import sys @@ -358,7 +359,7 @@ def reject_package_network(command, *args, **kwargs): ]) == "42" -def test_lock_resolves_once_and_plan_remains_read_only(tmp_path, monkeypatch): +def test_plan_resolves_metadata_once_without_downloading_wheels(tmp_path, monkeypatch): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) wheel = tiny_wheel(tmp_path / "fixture") @@ -377,35 +378,29 @@ def test_lock_resolves_once_and_plan_remains_read_only(tmp_path, monkeypatch): write_configuration(manifest, data, with_lock=False) manager = DeclarativeEnvironmentManager(config) original_execute = declarative_module.execute - download_calls = [] - - def fake_resolver(command, *args, **kwargs): - if "download" in command: - download_calls.append(command) - assert "--no-deps" not in command - destination = Path(command[command.index("--dest") + 1]) - shutil.copyfile(wheel, destination / wheel.name) - return "" - if "install" in command: - assert "--no-index" in command - return original_execute(command, *args, **kwargs) - - monkeypatch.setattr(declarative_module, "execute", fake_resolver) + resolver_calls = [] + + def fake_metadata(requirements, sources, environment, supported_tags): + resolver_calls.append(list(requirements)) + return [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + "url": "https://example.invalid/" + wheel.name, + "size": wheel.stat().st_size, "metadata_sha256": "", + }], + }] + + monkeypatch.setattr(declarative_module, "resolve_metadata", fake_metadata) before = manifest.read_bytes() - pending = manager.plan(manifest) - assert pending["lock"]["status"] == "missing" - assert pending["lock"]["required"] is True - assert pending["packages"][0]["selected"]["method"] == "lock-required" - assert download_calls == [] - assert manifest.read_bytes() == before - assert not lock_path(manifest).exists() - - result = manager.lock(manifest) - assert result["requested"] == ["pload-demo>=1"] - assert result["resolved"] == ["pload-demo==1.0"] first = manager.plan(manifest) - assert first["lock"]["status"] == "current" - assert first["packages"][0]["selected"]["method"] == "cache" + assert first["lock"]["status"] == "generated" + assert first["lock"]["required"] is False + assert first["packages"][0]["selected"]["method"] == "index-exact" + assert resolver_calls == [["pload-demo>=1"]] + assert manifest.read_bytes() == before + assert lock_path(manifest).is_file() _, configuration = load_manifest(manifest) assert configuration["environment"]["dependencies"] == ["pload-demo>=1"] _, locked, current = load_lock(manifest, configuration) @@ -415,16 +410,19 @@ def fake_resolver(command, *args, **kwargs): second = manager.plan(manifest) assert second["lock"]["status"] == "current" - assert len(download_calls) == 1 + assert len(resolver_calls) == 1 + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + manager.plan(manifest) restored = manager.apply(manifest) - assert len(download_calls) == 1 + assert len(resolver_calls) == 1 output = original_execute([ config.get_pip_command(restored)[0], "-c", "import pload_demo; print(pload_demo.answer)", ]) assert output == "42" -def test_plan_relocks_when_new_dependency_makes_existing_lock_stale(tmp_path, monkeypatch): +def test_plan_refreshes_stale_lock_using_only_metadata(tmp_path, monkeypatch): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) demo = tiny_wheel(tmp_path / "fixture") @@ -444,38 +442,24 @@ def test_plan_relocks_when_new_dependency_makes_existing_lock_stale(tmp_path, mo manager = DeclarativeEnvironmentManager(config) calls = [] - def fake_resolver(command, *args, **kwargs): - if "download" not in command: - return "" - calls.append(command) - assert "--find-links" in command - assert "pload-demo==1.0" in command - assert "extra-demo" in command - destination = Path(command[command.index("--dest") + 1]) - shutil.copyfile(demo, destination / demo.name) - shutil.copyfile(extra, destination / extra.name) - return "" - - monkeypatch.setattr(declarative_module, "execute", fake_resolver) + def fake_metadata(requirements, sources, environment, supported_tags): + calls.append(list(requirements)) + return [ + {"name": name, "version": "1.0", "sources": ["default"], + "artifact": [{"filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": []}]} + for name, wheel in (("pload-demo", demo), ("extra-demo", extra)) + ] + + monkeypatch.setattr(declarative_module, "resolve_metadata", fake_metadata) before_manifest = manifest.read_bytes() before_lock = lock_path(manifest).read_bytes() plan = manager.plan(manifest) - assert plan["lock"]["status"] == "stale" - assert plan["lock"]["required"] is True - assert calls == [] + assert plan["lock"]["status"] == "generated" + assert plan["lock"]["required"] is False + assert calls == [["pload-demo==1.0", "extra-demo"]] assert manifest.read_bytes() == before_manifest - assert lock_path(manifest).read_bytes() == before_lock - with pytest.raises(PloadError, match="missing or stale"): - manager.apply(manifest) - - progress = [] - result = manager.lock(manifest, progress=progress.append) - assert len(calls) == 1 - assert progress == [ - f"Configuration changed; resolving dependencies for Python {platform.python_version()}" - ] - assert result["requested"] == ["pload-demo==1.0", "extra-demo"] - assert result["resolved"] == ["extra-demo==1.0", "pload-demo==1.0"] + assert lock_path(manifest).read_bytes() != before_lock _, configuration = load_manifest(manifest) assert configuration["environment"]["dependencies"] == [ "pload-demo==1.0", "extra-demo", @@ -638,6 +622,48 @@ def fail_selected(*args): assert calls == ["cache"] +def test_apply_downloads_the_exact_url_saved_by_plan(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + data = simple_manifest("exact") + data["name"] = "exact-url" + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["pload-demo==1.0"], + }) + artifact_url = "https://files.example.invalid/" + wheel.name + data["package"] = [{ + "name": "pload-demo", "version": "1.0", "sources": ["default"], + "artifact": [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + "url": artifact_url, "size": wheel.stat().st_size, + }], + }] + manifest = tmp_path / "project" / "pload.toml" + manifest.parent.mkdir() + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + assert manager.plan(manifest)["packages"][0]["selected"]["method"] == "index-exact" + requested = [] + + def fake_urlopen(request, timeout): + requested.append(request.full_url) + return io.BytesIO(wheel.read_bytes()) + + monkeypatch.setattr(declarative_module, "urlopen", fake_urlopen) + restored = manager.apply(manifest) + assert requested == [artifact_url] + assert execute([ + config.get_pip_command(restored)[0], "-c", + "import pload_demo; print(pload_demo.answer)", + ]) == "42" + + def test_apply_rejects_plan_that_omits_a_locked_package(tmp_path): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) diff --git a/tests/test_metadata_resolver.py b/tests/test_metadata_resolver.py new file mode 100644 index 0000000..f51465a --- /dev/null +++ b/tests/test_metadata_resolver.py @@ -0,0 +1,86 @@ +import hashlib +import json + +import pytest +from packaging.tags import Tag + +from pload.errors import PloadError +from pload.metadata_resolver import resolve_metadata + + +def _project(filename, checksum, metadata_checksum, size=100): + return json.dumps({ + "meta": {"api-version": "1.1"}, + "name": filename.split("-", 1)[0], + "files": [{ + "filename": filename, + "url": "https://files.invalid/" + filename, + "hashes": {"sha256": checksum}, + "requires-python": ">=3.8", + "core-metadata": {"sha256": metadata_checksum}, + "yanked": False, + "size": size, + }], + "versions": [filename.split("-")[1]], + }).encode() + + +def test_resolver_reads_only_project_pages_and_core_metadata(): + root_metadata = ( + b"Metadata-Version: 2.1\nName: root-demo\nVersion: 1.0\n" + b"Requires-Dist: child-demo>=2\n\n" + ) + child_metadata = b"Metadata-Version: 2.1\nName: child-demo\nVersion: 2.1\n\n" + root_wheel = "root_demo-1.0-py3-none-any.whl" + child_wheel = "child_demo-2.1-py3-none-any.whl" + responses = { + "https://index.invalid/simple/root-demo/": _project( + root_wheel, "a" * 64, hashlib.sha256(root_metadata).hexdigest(), + ), + "https://index.invalid/simple/child-demo/": _project( + child_wheel, "b" * 64, hashlib.sha256(child_metadata).hexdigest(), + ), + "https://files.invalid/" + root_wheel + ".metadata": root_metadata, + "https://files.invalid/" + child_wheel + ".metadata": child_metadata, + } + requested = [] + + def fetch(url, accept, maximum): + requested.append(url) + return responses[url], ( + "application/vnd.pypi.simple.v1+json" + if url.endswith("/") else "application/octet-stream" + ) + + packages = resolve_metadata( + ["root-demo>=1"], [("default", "https://index.invalid/simple")], + {"python": "3.12.1", "implementation": "cpython", + "system": "Linux", "machine": "x86_64"}, + {Tag("py3", "none", "any")}, fetch=fetch, + ) + + assert [(item["name"], item["version"]) for item in packages] == [ + ("child-demo", "2.1"), ("root-demo", "1.0"), + ] + assert all(url.endswith(("/", ".metadata")) for url in requested) + assert not any(url.endswith(".whl") for url in requested) + + +def test_resolver_refuses_to_download_wheel_when_metadata_is_unavailable(): + wheel = "root_demo-1.0-py3-none-any.whl" + project = json.loads(_project(wheel, "a" * 64, "b" * 64)) + project["files"][0].pop("core-metadata") + requested = [] + + def fetch(url, accept, maximum): + requested.append(url) + return json.dumps(project).encode(), "application/vnd.pypi.simple.v1+json" + + with pytest.raises(PloadError, match="refusing to download a wheel during plan"): + resolve_metadata( + ["root-demo"], [("default", "https://index.invalid/simple")], + {"python": "3.12.1", "implementation": "cpython", + "system": "Linux", "machine": "x86_64"}, + {Tag("py3", "none", "any")}, fetch=fetch, + ) + assert requested == ["https://index.invalid/simple/root-demo/"] From e204bee6d323f17915bb7cd5695f7e97cece4754 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 17:17:38 +0800 Subject: [PATCH 20/41] Add interactive route selection and explicit remote backup --- src/pload/cli.py | 101 +++++++++++++++++++++++++++++++++++- src/pload/declarative.py | 98 ++++++++++++++++++++++++++++++++++ src/pload/help_content.py | 16 ++++++ src/pload/resource_plan.py | 40 ++++++++++++++ tests/test_declarative.py | 31 +++++++++++ tests/test_resource_plan.py | 34 ++++++++++++ 6 files changed, 318 insertions(+), 2 deletions(-) create mode 100644 tests/test_resource_plan.py diff --git a/src/pload/cli.py b/src/pload/cli.py index 74b4fcf..0e1014a 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -33,6 +33,7 @@ "plan": (), "apply": (), "repo": (), + "remote": (), } PYTHON_ALIASES = { "install": (), @@ -315,6 +316,10 @@ def build_parser(): plan.add_argument("--offline", "-o", action="store_true", help="plan using only resources available without the internet") plan.add_argument("--json", "-j", action="store_true", help="emit machine-readable JSON") + plan.add_argument( + "--no-ui", "-N", action="store_true", + help="select the fastest routes without opening the interactive route chooser", + ) apply = subparsers.add_parser( "apply", description="Materialize the desired environment from available resources.", help="create or verify an environment from pload.toml", @@ -335,6 +340,20 @@ def build_parser(): actions.add_parser("list", aliases=["ls"], description="Show configured repositories.") remove = actions.add_parser("remove", description="Remove configuration, keeping all remote files.") remove.add_argument("name", help="repository nickname") + remote = subparsers.add_parser( + "remote", help="explicitly back up installed package artifacts", + description="Search or store package artifacts in configured local/SSH repositories.", + ) + remote_actions = remote.add_subparsers(dest="remote_command", required=True) + remote_add = remote_actions.add_parser( + "add", description="Back up one installed package wheel; never runs during apply.", + ) + remote_add.add_argument("package", help="installed package name or exact NAME==VERSION") + remote_add.add_argument("--from", "-f", dest="environment", default=".", + help="source environment ID, name, or path (default: .)") + remote_add.add_argument("--remote", "-r", dest="repository", + help="configured repository name (default: first configured)") + remote_add.add_argument("--index", "-i", help="index used only if no original wheel is cached") return parser @@ -492,6 +511,66 @@ def print_declarative_plan(plan): ) +def choose_declarative_routes(plan): + """Navigate every package route with backtracking and undo in a terminal.""" + from pload.resource_plan import PlanSelection + + console = Console(highlight=False) + selection = PlanSelection(plan) + if not plan["packages"]: + return "save" + cursor = 0 + while True: + packages = plan["packages"] + package = packages[cursor] + routes = selection.routes(cursor) + console.clear() + console.print(Panel.fit( + f"[bold green]{package['name']}[/]==[yellow]{package['version']}[/]\n" + f"Package [bold]{cursor + 1}[/] of [bold]{len(packages)}[/]", + title="[bold cyan]Choose acquisition route[/]", border_style="blue", + )) + table = Table(box=box.ROUNDED, header_style="bold cyan", border_style="blue") + table.add_column("#", justify="right", style="bold yellow") + table.add_column("METHOD", style="green") + table.add_column("TIME", justify="right") + table.add_column("RESOURCE") + for index, route in enumerate(routes, 1): + marker = "✓" if index == 1 else str(index) + seconds = route.get("estimated_seconds", 0) + table.add_row(marker, route["method"], f"~{seconds:g}s", route["location"]) + if routes: + console.print(table) + else: + console.print("[bold red]No valid route is currently available.[/]") + console.print( + "[dim]number choose · Enter/n next · p previous · u undo · " + "r reset fastest · s save · a save & apply · q quit[/]" + ) + command = console.input("[bold cyan]route> [/]").strip().lower() + if command.isdigit(): + try: + selection.select(cursor, int(command) - 1) + except IndexError: + continue + if cursor < len(packages) - 1: + cursor += 1 + elif command in {"", "n", "next"}: + cursor = min(cursor + 1, len(packages) - 1) + elif command in {"p", "previous", "back"}: + cursor = max(cursor - 1, 0) + elif command in {"u", "undo"}: + selection.undo() + elif command in {"r", "reset"}: + selection.reset() + elif command in {"s", "save"}: + return "save" + elif command in {"a", "apply"}: + return "apply" + elif command in {"q", "quit"}: + return "quit" + + def _print_spaced_section(console, renderable): """Give a standalone section consistent visual breathing room.""" console.print() @@ -567,6 +646,7 @@ def _brief_help(parser, command_path, root_parser=None): "plan": "Compare a configuration with available resources", "apply": "Materialize the environment declared by pload.toml", "repo": "Manage local and SSH artifact providers", + "remote": "Explicitly back up installed package artifacts", } for command, summary in summaries.items(): aliases = COMMAND_ALIASES[command] @@ -755,7 +835,7 @@ def run(argv=None): venvs = VenvManager(config) dependencies = DependencyManager(config) - if command in {"describe", "lock", "plan", "apply", "repo"}: + if command in {"describe", "lock", "plan", "apply", "repo", "remote"}: if command == "repo": from pload.snapshots import RepositoryManager @@ -770,7 +850,16 @@ def run(argv=None): from pload.declarative import DeclarativeEnvironmentManager manager = DeclarativeEnvironmentManager(config) - if command == "describe": + if command == "remote": + result = manager.publish_package( + args.package, args.environment, args.repository, args.index, + ) + Console(highlight=False).print( + f"[bold green]✓ Backed up {result['package']}=={result['version']}[/] " + f"to [cyan]{result['repository']}[/]\n" + f"[dim]{result['filename']} · sha256:{result['sha256']}[/]" + ) + elif command == "describe": progress_console = Console(highlight=False) print(manager.describe( args.source, args.output, args.name, args.mode, args.repository, @@ -810,7 +899,15 @@ def run(argv=None): f"[cyan]{message}…[/]" ), ) + action = "save" + chooser_console = Console(highlight=False) + if chooser_console.is_terminal and not args.no_ui: + action = choose_declarative_routes(plan) + if action != "quit": + plan["plan_path"] = str(manager.save_plan(args.file, plan)) print_declarative_plan(plan) + if action == "apply": + chooser_console.print(manager.apply(args.file, offline=args.offline)) else: progress_console = Console(highlight=False) print(manager.apply( diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 01906ce..ce4e6d7 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -501,6 +501,42 @@ def publish_many(cls, spec, artifacts, base): ) execute(["ssh", *cls.SSH_OPTIONS, host, verify], timeout=75) + @classmethod + def publish_package_record(cls, spec, package, version, artifact, base): + """Publish a small searchable record separately from the immutable object.""" + record = { + "schema": 1, "name": normalized_name(package), "version": version, + "filename": artifact["filename"], "sha256": artifact["sha256"], + } + relative = ( + f"packages/{record['name']}/{version}/{artifact['filename']}.json" + ) + payload = json.dumps(record, sort_keys=True, indent=2) + "\n" + if spec["kind"] == "local": + target = cls._local_root(spec, base) / relative + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + "." + uuid.uuid4().hex) + temporary.write_text(payload, encoding="utf-8") + temporary.replace(target) + return + host, root = RepositoryManager.ssh_location(spec["location"]) + remote = root + "/" + relative + with tempfile.TemporaryDirectory(prefix="pload-record-") as temporary: + local = Path(temporary) / "record.json" + local.write_text(payload, encoding="utf-8") + pending = remote + "." + uuid.uuid4().hex + execute([ + "ssh", *cls.SSH_OPTIONS, host, + f"mkdir -p {shlex.quote(remote.rsplit('/', 1)[0])}", + ], timeout=75) + execute([ + "scp", "-q", *cls.SSH_OPTIONS, str(local), host + ":" + pending, + ], timeout=75) + execute([ + "ssh", *cls.SSH_OPTIONS, host, + f"mv {shlex.quote(pending)} {shlex.quote(remote)}", + ], timeout=75) + @classmethod def fetch(cls, spec, checksum, destination, base): cls.fetch_many(spec, [(checksum, destination)], base) @@ -748,6 +784,57 @@ def describe( self._write_lock(output, data) return output + def publish_package(self, package_name, environment=".", repository=None, index=None): + """Explicitly preserve one installed package wheel in a configured repository.""" + requirement = _parse_dependency(package_name) + if requirement.specifier and not str(requirement.specifier).startswith("=="): + raise PloadError("remote add accepts a package name or one exact NAME==VERSION") + env = VenvManager(self.config).resolve_existing(environment) + interpreter = env / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + installed = next( + (item for item in probe(interpreter).get("packages", []) + if normalized_name(item["name"]) == normalized_name(requirement.name)), + None, + ) + if not installed: + raise PloadError(f"{requirement.name} is not installed in {environment}") + if requirement.specifier and not requirement.specifier.contains(installed["version"]): + raise PloadError( + f"{requirement.name} {installed['version']} does not satisfy " + f"{requirement.specifier}" + ) + repositories = self.config.settings.get("repositories", {}) + selected_repository = repository or (min(repositories) if repositories else None) + if not selected_repository or selected_repository not in repositories: + raise PloadError("choose a configured repository with --remote") + source_index = public_index( + index or self.config.settings.get("pip_index") or "https://pypi.org/simple" + ) + with tempfile.TemporaryDirectory(prefix="pload-remote-add-") as temporary: + wheel = self._obtain_wheel( + interpreter, installed, Path(temporary), source_index, + ) + checksum = digest(wheel) + artifact = {"filename": wheel.name, "sha256": checksum} + ArtifactRepository.publish( + repositories[selected_repository], wheel, checksum, Path.cwd(), + ) + ArtifactRepository.publish_package_record( + repositories[selected_repository], installed["name"], + installed["version"], artifact, Path.cwd(), + ) + self.cache.mkdir(parents=True, exist_ok=True) + cached = self.cache / wheel.name + if not cached.exists(): + temporary_cache = cached.with_name(cached.name + "." + uuid.uuid4().hex) + shutil.copyfile(wheel, temporary_cache) + temporary_cache.replace(cached) + return { + "package": installed["name"], "version": installed["version"], + "repository": selected_repository, "filename": artifact["filename"], + "sha256": checksum, + } + def _obtain_wheel(self, interpreter, installed, wheel_dir, index): expected_name = normalized_name(installed["name"]) expected_version = installed["version"] @@ -1348,6 +1435,17 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): venvs.remove_venv(target_name) raise + def save_plan(self, manifest_path, plan): + """Persist user-selected routes against the exact current configuration and lock.""" + path, data, current, _ = self._load_state(manifest_path) + if data["environment"].get("dependencies") and not current: + raise PloadError("cannot save routes against a missing or stale lock") + sidecar = lock_path(path) + return write_plan( + path, configuration_digest(data), + file_digest(sidecar) if sidecar.is_file() else "", plan, + ) + @staticmethod def _validate_saved_plan(saved, data): """Bind every executable route to one exact package in the current lock.""" diff --git a/src/pload/help_content.py b/src/pload/help_content.py index 10e5c60..6f1021e 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -433,6 +433,22 @@ for _command in ("add", "list", "remove"): GUIDES[("repo", _command)] = GUIDES[("repo",)] +GUIDES[("remote",)] = r""" +# Back up package artifacts explicitly + +```console +pload remote add torch --from v10 --remote lab +pload remote add numpy==2.1.0 -f ./project/.venv -r disk +``` + +`remote add` finds the exact installed version, reuses its original cached wheel +when available, verifies its SHA-256 identity, and stores it in the selected +content-addressed repository. If the original wheel is unavailable it may fetch +that exact version from `--index`. Planning and applying never upload packages; +publication happens only through this explicit command. +""" +GUIDES[("remote", "add")] = GUIDES[("remote",)] + def render_detailed_help(parser, command_path): console = Console(highlight=False) diff --git a/src/pload/resource_plan.py b/src/pload/resource_plan.py index 6193b31..c505d5c 100644 --- a/src/pload/resource_plan.py +++ b/src/pload/resource_plan.py @@ -2,6 +2,7 @@ import hashlib import json +from copy import deepcopy from pathlib import Path try: @@ -12,6 +13,45 @@ from pload.errors import PloadError +class PlanSelection: + """Mutable route selection with deterministic reset and multi-step undo.""" + + def __init__(self, plan): + self.plan = plan + self._original = deepcopy(plan.get("packages", [])) + self._history = [] + + def routes(self, package_index): + package = self.plan["packages"][package_index] + return ([package["selected"]] if package.get("selected") else []) + list( + package.get("alternatives", []) + ) + + def select(self, package_index, route_index): + routes = self.routes(package_index) + if route_index < 0 or route_index >= len(routes): + raise IndexError("route index out of range") + self._history.append(deepcopy(self.plan["packages"])) + package = self.plan["packages"][package_index] + package["selected"] = routes[route_index] + package["alternatives"] = [ + route for index, route in enumerate(routes) if index != route_index + ] + self.plan["ready"] = all( + item.get("selected") for item in self.plan["packages"] + ) and self.plan["python"].get("status") != "unavailable" + + def undo(self): + if not self._history: + return False + self.plan["packages"] = self._history.pop() + return True + + def reset(self): + self._history.append(deepcopy(self.plan["packages"])) + self.plan["packages"] = deepcopy(self._original) + + def plan_path(configuration_path): return Path(configuration_path).parent / ".pload_plan.toml" diff --git a/tests/test_declarative.py b/tests/test_declarative.py index e3ce952..afdf6e8 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -582,6 +582,34 @@ def test_apply_requires_a_saved_plan_and_never_auto_publishes(tmp_path): assert not (repository / "objects" / checksum).exists() +def test_remote_add_explicitly_publishes_one_installed_package(tmp_path): + repository = tmp_path / "remote" + config_home = tmp_path / "home" + save_settings(config_home, { + "repositories": {"lab": {"kind": "local", "location": str(repository)}}, + }) + config = ConfigManager(home=config_home) + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + source = VenvManager(config).create_venv(name="remote-source") + execute(config.get_pip_command(source) + [ + "install", "--no-index", "--find-links", str(wheel.parent), "pload-demo==1.0", + ]) + manager = DeclarativeEnvironmentManager(config) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + + result = manager.publish_package("pload-demo", "remote-source", "lab") + + assert result["sha256"] == digest(wheel) + assert (repository / "objects" / digest(wheel)).is_file() + record = ( + repository / "packages" / "pload-demo" / "1.0" / f"{wheel.name}.json" + ) + assert record.is_file() + assert '"sha256": "' + digest(wheel) + '"' in record.read_text(encoding="utf-8") + + def test_apply_obeys_selected_route_without_fallback(tmp_path, monkeypatch): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) @@ -697,6 +725,9 @@ def test_declarative_commands_and_shell_integration(capsys): assert parsed.package_sources == ["torch=https://download.pytorch.org/whl/cu121"] assert parser.parse_args(["lock"]).file == "pload.toml" assert parser.parse_args(["apply"]).file == "pload.toml" + assert parser.parse_args(["plan", "--no-ui"]).no_ui is True + remote = parser.parse_args(["remote", "add", "numpy", "-f", "v10", "-r", "lab"]) + assert (remote.package, remote.environment, remote.repository) == ("numpy", "v10", "lab") for command in ("describe", "lock", "plan", "apply"): assert main([command, "-h"]) == 0 for shell in ("bash", "zsh", "fish", "powershell"): diff --git a/tests/test_resource_plan.py b/tests/test_resource_plan.py new file mode 100644 index 0000000..3e55759 --- /dev/null +++ b/tests/test_resource_plan.py @@ -0,0 +1,34 @@ +from pload.resource_plan import PlanSelection + + +def _route(method): + return { + "method": method, "location": "/" + method, "status": "ready", + "cost": [0], "estimated_seconds": 1, + } + + +def test_route_selection_supports_navigation_state_undo_and_reset(): + plan = { + "python": {"status": "ready"}, "ready": True, + "packages": [{ + "name": "demo", "version": "1", "selected": _route("cache"), + "alternatives": [_route("environment-copy"), _route("index-exact")], + }], + } + selection = PlanSelection(plan) + + selection.select(0, 2) + assert plan["packages"][0]["selected"]["method"] == "index-exact" + assert [item["method"] for item in plan["packages"][0]["alternatives"]] == [ + "cache", "environment-copy", + ] + assert selection.undo() is True + assert plan["packages"][0]["selected"]["method"] == "cache" + assert selection.undo() is False + + selection.select(0, 1) + selection.reset() + assert plan["packages"][0]["selected"]["method"] == "cache" + assert selection.undo() is True + assert plan["packages"][0]["selected"]["method"] == "environment-copy" From b8af6188ba1d9ff66c2781f7898467ffea89df5c Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Fri, 25 Sep 2026 17:21:43 +0800 Subject: [PATCH 21/41] Document resource-aware reproduction preview --- CHANGELOG.md | 34 +- README.md | 25 +- docs/declarative-environments.md | 591 +++++++++---------------------- pyproject.toml | 2 +- src/pload/__init__.py | 2 +- src/pload/cli.py | 6 +- src/pload/declarative.py | 26 +- src/pload/help_content.py | 42 +-- tests/test_declarative.py | 6 +- 9 files changed, 249 insertions(+), 485 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f7802d..c0e59fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,36 +2,52 @@ ## Unreleased -Pre-release: `1.1.0a10` +Pre-release: `1.2.0a1` + +### Architecture reset + +- Replace the discarded lock-then-plan prototype with a resource-aware + workflow: `plan` resolves only from Simple API/Core Metadata, inventories all + resources, and writes `.pload_plan.toml`; `apply` executes that exact file. +- Never download wheel bodies during planning. Indexes without PEP 658/714 + metadata fail with an actionable message instead of silently filling cache. +- Discover exact wheels in pload, pip and uv caches and copyable distributions + in compatible pload environments. Recheck SHA-256 or installed `RECORD` + fingerprints during apply. +- Add an interactive per-package route chooser with previous/next navigation, + undo, reset-to-fastest, save and save-and-apply actions. +- Add explicit `pload remote add PACKAGE --from ENV --remote REPO`; apply and + describe never publish package bytes automatically. +- Persist exact artifact URLs and hashes, reject stale or incomplete plans, and + stop on selected-route failure without trying alternatives. ### Added - Keep user intent in a clean portable `pload.toml` and store pload-managed exact dependency, artifact and hash decisions in sibling `.pload_lock.toml`. -- Add `pload describe`, explicit `pload lock`, read-only `pload plan` and - idempotent `pload apply`; downloading cannot be triggered by planning. +- Add `pload describe`, metadata-only `pload plan` and idempotent `pload apply`. - Inventory the local wheel cache, adjacent artifacts, local/SSH content stores, configured indexes and Python installations before choosing a route. - Allow package-specific indexes in `describe`, including CUDA wheel indexes, while keeping credentials out of the portable configuration. - Reuse exact wheel archives by SHA-256, deduplicate local/SSH storage, record detected NVIDIA driver provenance and roll back failed materializations. -- Add complete field-by-field configuration and lock references, enforce agreement between +- Add complete field-by-field configuration, lock and plan references, enforce agreement between desired dependency pins and package locks, and make policy repositories and - `publish_missing_artifacts` drive lookup and automatic cache publication. + repositories drive lookup while publication remains explicit. - Validate locked wheel Python, ABI and platform tags against the requested runtime during planning; incompatible cache entries can no longer be reported as ready, and compatible mode explicitly plans network re-resolution instead. - Accept unpinned dependency intentions such as `numpy` and `pandas>=2`; the - explicit lock command resolves the complete wheel dependency closure, records - exact versions, hashes and tags, and subsequent plans inspect it read-only. + metadata-only planner resolves the complete dependency closure and records + exact versions, hashes, URLs and tags without fetching wheel bodies. - Treat edits that add or change dependencies beside an existing lock as a stale lock to reconcile, not an invalid configuration; re-resolution uses the complete requested set and prefers existing pload cache wheels. - Migrate lock tables embedded by earlier 1.1 previews into `.pload_lock.toml` without another dependency resolution or artifact download. -- Show animated status for explicit locking and artifact-repository inspection - instead of appearing frozen while pip or SSH work is captured in the background. +- Show animated status for metadata resolution and artifact-repository inspection + instead of appearing frozen while network or SSH work is captured. - Refuse `apply` when the lock is absent or stale instead of resolving and downloading implicitly. diff --git a/README.md b/README.md index 10e3ae5..3d8f160 100644 --- a/README.md +++ b/README.md @@ -78,22 +78,27 @@ profile when `--shell` is supplied or already configured. ## Reproduce and share environments -The `1.1` pre-release uses a clean, user-authored `pload.toml` as the portable -source of truth and writes exact resolver output to the sibling -`.pload_lock.toml`. The user declares the desired environment; pload inventories local caches, -configured local/SSH content stores, package indexes and Python installations, -then performs transfers and installation internally: +The `1.2` preview uses one clean, user-authored `pload.toml` as portable intent. +pload manages `.pload_lock.toml` (exact versions and artifacts) and +`.pload_plan.toml` (the selected acquisition route for every package): ```console pload describe v2 -o pload.toml -pload lock pload.toml pload plan pload.toml pload apply pload.toml ``` -`plan` is strictly read-only: it never invokes pip, downloads packages, or -changes either TOML file. Run `lock` explicitly after editing dependencies; -`apply` refuses a missing or stale lock instead of resolving behind your back. +`plan` fetches only index pages and independent package metadata—never wheel +bodies—and inventories pload/pip/uv caches, compatible existing environments, +local/SSH stores and indexes. Its interactive chooser supports previous/next, +undo, reset, save and save-and-apply. `apply` executes exactly the saved plan: +it never re-resolves, changes route, falls back, or uploads packages. + +Back up a special or slow wheel only when you explicitly request it: + +```console +pload remote add torch --from v10 --remote lab +``` See the [declarative environment guide](docs/declarative-environments.md) for the [complete TOML field reference](docs/declarative-environments.md#complete-ploadtoml-field-reference), @@ -119,7 +124,7 @@ pload-install --yes --package-spec "pload==1.0.0" To opt into the declarative-environment pre-release for testing: ```console -pload-install --yes --package-spec "pload==1.1.0a10" +pload-install --yes --package-spec "pload==1.2.0a1" ``` To always follow the newest published release instead of pinning a version, diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index c95e603..c3d7f3c 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -1,30 +1,60 @@ # Declarative environments -The `1.1` pre-release replaces the transport-oriented snapshot experiment with -a small portable configuration plus a pload-managed lock. Users describe the -desired state; pload discovers resources, chooses a deterministic plan and -materializes that state. +The `1.2` preview treats reproduction as resource scheduling, not as a sequence +of commands the user must design. You carry one small `pload.toml`; pload +resolves exact versions, inventories available resources, saves an auditable +route for every package, and executes exactly those routes. + +```text +pload.toml (intent) + │ + ▼ +metadata-only resolution ──► .pload_lock.toml (exact graph and wheel identities) + │ + ▼ +local/remote inventory ─────► .pload_plan.toml (one chosen route per package) + │ + ▼ +pload apply ────────────────► environment matching the lock +``` + +The normal workflow is: ```console -pload describe v2 -o pload.toml -pload lock pload.toml -pload plan pload.toml -pload apply pload.toml +pload plan pload.toml # resolve metadata, inventory and choose routes +pload apply pload.toml # execute only the saved plan ``` -`describe` is the bridge for an existing environment. For a new project, -`pload.toml` may be written by hand with names, ranges or exact -`NAME==VERSION` requirements; `lock` produces the managed lock. +In a terminal, `plan` opens a route chooser. A number selects a route; Enter or +`n` moves forward, `p` goes back, `u` undoes, `r` restores the fastest choices, +`s` saves, and `a` saves then applies. Use `pload plan --no-ui` in CI. Use +`--json` for machine-readable inspection. + +## Three files, one user-owned file + +- `pload.toml` is portable user intent. This is the only file users edit. +- `.pload_lock.toml` is pload-managed exact resolution data. +- `.pload_plan.toml` is pload-managed execution data bound to that exact config + and lock. + +The two dotfiles exist because a request such as `torch` does not identify one +version, dependency closure, wheel build, or byte sequence. The lock records +those facts; the plan records where each exact artifact will come from. Editing +`pload.toml` invalidates both through SHA-256 bindings. Deleting either managed +file is safe: the next online `plan` regenerates it, although a new resolution +may differ if an index changed. -## Two files, two responsibilities +`plan` may request Simple API project pages and independent `.metadata` +files. It never requests a wheel body, invokes `pip download`, copies packages, +or populates a cache. If an index does not expose independent Core Metadata, +planning stops and suggests adding a metadata-capable source. `apply` never +re-resolves, changes route, or publishes artifacts when a selected route fails. -`pload.toml` is the file a user reads and edits. It contains only the desired -Python, direct dependency requirements, target platform, policies and resource -providers. For example: +## Complete `pload.toml` reference ```toml schema = 1 -name = "llm-training" +name = "training" [environment] python = "3.12.7" @@ -39,441 +69,174 @@ nvidia_driver = ["550.90.07"] [policy] reproducibility = "exact" network = "allow" -source_build = "fallback" -publish_missing_artifacts = true +source_build = "forbid" +publish_missing_artifacts = false repositories = ["lab"] -[sources.default] +[sources.pypi] kind = "index" url = "https://pypi.org/simple" +[sources.mirror] +kind = "index" +url = "https://pypi.tuna.tsinghua.edu.cn/simple" + [repositories.lab] kind = "ssh" location = "frpxiaoxin:/home/tibless/pload-cloud" ``` -`pload lock` writes its decisions to `.pload_lock.toml` in the same directory: - -```toml -schema = 1 -configuration = "pload.toml" -configuration_sha256 = "..." - -[environment] -dependencies = ["numpy==2.1.3", "torch==2.5.0+cu121", "..."] - -[[package]] -name = "torch" -version = "2.5.0+cu121" -sources = ["default"] - -[[package.artifact]] -filename = "torch-2.5.0+cu121-cp312-cp312-linux_x86_64.whl" -sha256 = "..." -tags = ["cp312-cp312-linux_x86_64"] -repositories = ["lab"] -``` - -The lock is generated data, not a second user configuration. It records the -complete transitive dependency closure, exact versions, wheel identities, -SHA-256 hashes, compatibility tags and known artifact locations. The -`configuration_sha256` binds it to the current `pload.toml`; editing the user -configuration makes the old lock stale and the next explicit `pload lock` -replaces it atomically. Commit both files when reproducibility matters, but normally edit -only `pload.toml`. - -The lock is necessary for **exact** reproduction because a requirement such as -`torch` or even `torch==2.8.0` does not uniquely identify all transitive versions -or a particular wheel build. Package indexes change over time. The filename and -SHA-256 in the lock let pload prove that a cached, cloud-hosted or downloaded -artifact is the same byte sequence chosen originally. Deleting the lock is safe: -the next online `pload lock` resolves a new one, but the result may differ. -`plan` never regenerates it. - -Python runtimes and wheel bytes remain in caches, indexes or content-addressed -repositories; the two TOML files contain only control information. Credentials -are never stored in either file; SSH uses the existing SSH agent/configuration. -`describe` writes both files automatically. URLs containing embedded -credentials or query-string tokens are rejected; authentication belongs to the -machine, not a shareable environment description. - -## Complete `pload.toml` field reference - -`pload.toml` has five user-facing logical parts. Unknown TOML fields are reserved -for future schema versions and should not be relied upon. The generated lock -format is documented separately below so it can be audited, not hand-maintained. - -### Top-level identity - -| Field | Type | Required | Meaning | -| --- | --- | --- | --- | -| `schema` | integer | yes | File-format version. The current and only accepted value is `1`. It is not the pload package version. | -| `name` | string | yes | Default name used by `pload apply`. It must be 1–100 letters, numbers, dots, underscores or hyphens, starting with a letter or number. `apply --name` may override it without changing the file. | - -### `[environment]`: desired runtime state - -| Field | Type | Default | Meaning and enforcement | -| --- | --- | --- | --- | -| `python` | string | none; required | Exact Python version, currently including the patch component, such as `3.12.7`. The planner reuses a matching discovered interpreter or schedules an installation when network policy allows it. | -| `implementation` | string | `"cpython"` | `sys.implementation.name`, normally `cpython` or `pypy`. `apply` rejects a different implementation. | -| `system` | string | empty | Target `platform.system()`, such as `Linux`, `Darwin` or `Windows`. A non-empty value is a hard compatibility constraint. | -| `machine` | string | empty | Target `platform.machine()`, such as `x86_64`, `AMD64`, `arm64` or `aarch64`. A non-empty value is a hard compatibility constraint. | -| `dependencies` | array of strings | `[]` | Direct package requirements owned by the user. Entries may be names (`numpy`), version constraints (`pandas>=2,<3`) or exact pins (`numpy==2.1.3`). `plan` never replaces this list with transitive dependencies; resolved versions belong to `.pload_lock.toml`. Direct URLs and environment markers are intentionally rejected in schema 1. pip, setuptools and wheel bootstrap packages are omitted by `describe`. | - -The environment section is desired state, not a command sequence. Paths to a -source virtual environment are never stored here, so the configuration remains -portable. The current schema describes one target platform and one exact Python -version; use separate files for different targets until multi-target locks exist. - -For example, this is a valid initial request: - -```toml -dependencies = ["numpy", "pandas>=2,<3"] -``` - -`pload lock` asks pip under the requested interpreter to resolve these requirements -and their transitive dependencies as compatible wheels. It then atomically writes -exact `NAME==VERSION` results, wheel filenames, SHA-256 hashes and tags to -`.pload_lock.toml`; `pload.toml` remains unchanged. Resolution uses all declared -indexes, with `default` as the primary index. It requires network access once; -`lock --offline` rejects unavailable resolver inputs rather than guessing. -`plan` is always read-only and never runs the resolver. - -Editing dependencies after a lock exists is also supported. For example, adding -`"torch"` to a configuration with an existing `.pload_lock.toml` marks that lock -as stale. The next online `pload lock` resolves the complete -requested set—including existing exact constraints—rather than rejecting the -file or silently dropping the new package. The resolver receives the pload wheel -cache as a local candidate source, so compatible cached wheels are preferred and -only missing artifacts need index access. The reconciled complete lock replaces -the old one atomically. - -Configurations produced by pre-`1.1.0a8` previews may still contain embedded -`[[package]]` tables. The first `pload lock` migrates those tables to -`.pload_lock.toml` without downloading again and rewrites `pload.toml` in the -clean declaration-only format. Because older previews replaced the original -direct requirements with the complete closure, they cannot always recover which -packages the user originally typed; those exact requirements remain valid and -can be simplified manually afterward. - -### `[capabilities]`: observed non-Python context - -Each capability is an array of strings. `describe` currently recognizes -`nvidia_driver` and records the driver versions reported by `nvidia-smi`: - -```toml -[capabilities] -nvidia_driver = ["550.90.07"] -``` - -Capabilities are provenance and planning information, not installable Python -artifacts. The current executor does not install, downgrade or require an exact -NVIDIA driver match. CUDA-enabled wheels are locked separately under -`[[package.artifact]]`; drivers and system libraries remain host responsibilities. -Unknown capability keys are preserved by the parser as arrays of strings but do -not affect planning in schema 1. - -### `[policy]`: permitted ways to reach the desired state - -| Field | Allowed values / type | Default | Meaning | -| --- | --- | --- | --- | -| `reproducibility` | `"exact"` or `"compatible"` | `"exact"` | `exact` requires a declared wheel identity and SHA-256 for every package. `compatible` keeps exact versions but may obtain a new artifact build from an index. | -| `network` | `"allow"` or `"offline"` | `"allow"` | Controls internet index access and automatic Python installation. `offline` still permits configured local and SSH repositories. CLI `--offline` can only make a run stricter; it cannot override an offline file to allow internet access. | -| `source_build` | `"fallback"` or `"forbid"` | `"fallback"` | Applies to compatible re-resolution. `forbid` passes `--only-binary=:all:`; `fallback` allows pip to use an sdist when no wheel exists. Exact mode always installs the locked wheel and never rebuilds it. | -| `publish_missing_artifacts` | boolean | `true` | After `apply` obtains exact wheels, publish their SHA-addressed bytes to every repository named by `policy.repositories`. `false` makes those repositories read-only candidates for this configuration. | -| `repositories` | array of strings | `[]` | Logical repository names preferred for lookup and optional automatic publication. Every name must have a matching `[repositories.NAME]` table. This is a candidate set, not a user-visible upload sequence. | - -Command-line `--offline` is intentionally one-way: it prevents a configuration -from unexpectedly using the internet, while a file declaring `network = -"offline"` cannot be weakened accidentally by omitting the flag. - -Changing `environment.python` does not make native wheels portable. During -planning, pload parses every locked wheel filename and compares its Python, ABI -and platform tags with the requested runtime. For example, a cached `cp39-cp39` -wheel is rejected for Python 3.8 even when its checksum is valid. Under `exact`, -that makes the plan unavailable; under `compatible`, the planner may select an -explicit `index-resolve / network` route for that package. Universal wheels such -as `py3-none-any` can still be reused. `apply` never silently treats an -incompatible cached wheel as ready. - -### `[sources.NAME]`: Python package indexes - -`NAME` is a logical identifier used by package locks, for example `default` or -`pytorch-cu121`: - -```toml -[sources.pytorch-cu121] -kind = "index" -url = "https://download.pytorch.org/whl/cu121" -``` +### Top level | Field | Type | Meaning | | --- | --- | --- | -| `kind` | string | Must be `"index"` in schema 1. | -| `url` | string | HTTP(S) Simple API base URL. Embedded usernames, passwords and query parameters are rejected because this file is intended to be shared. Configure authentication on the machine instead. | - -Source names must follow the same safe-name rules as the environment name. A -package lists its allowed source names in `package.sources`. In compatible mode, -the first listed source is the primary resolver source. In exact mode, an index -is accepted only if it returns the declared wheel filename and its bytes match -the declared SHA-256. - -### `[repositories.NAME]`: reusable artifact stores +| `schema` | integer, required | Configuration format; currently `1`. | +| `name` | string, required | Default environment name used by `apply`. Allowed characters are letters, numbers, dots, underscores and hyphens. | -Repositories store immutable bytes by content hash rather than by environment: +### `[environment]` -```toml -[repositories.lab] -kind = "ssh" -location = "frpxiaoxin:/home/tibless/pload-cloud" - -[repositories.shared-disk] -kind = "local" -location = "./pload-artifacts" -``` - -| Field | Allowed values | Meaning | +| Field | Default | Meaning | | --- | --- | --- | -| `kind` | `"local"` or `"ssh"` | Selects filesystem copying or OpenSSH/SFTP transport. | -| `location` | path or `HOST:/absolute/path` | A local absolute path, a path relative to the TOML file's directory, or an SSH alias/host plus absolute remote directory. SSH paths cannot contain spaces or `..`; `/` itself is rejected. | - -The physical layout is `objects/`. SSH credentials, host keys, ports, -proxies and jump hosts remain in normal OpenSSH configuration. The remote host -must provide a POSIX shell and `sha256sum`. Repository names referenced by a -policy or artifact must exist in this file. Missing or unreachable providers are -removed from the plan and another valid route is tried. - -The TOML control file may itself be versioned in Git or hosted on GitHub. Large -wheel bytes should remain in these content-addressed local/SSH repositories rather -than being committed to Git. - -## Complete `.pload_lock.toml` field reference - -The lock's top-level `schema` is currently `1`. `configuration` names the sibling -user configuration and `configuration_sha256` identifies its canonical content. -`[environment].dependencies` contains the complete, sorted exact dependency -closure selected by the resolver. These fields let pload distinguish a reusable -lock from one made stale by editing or renaming the configuration. - -### `[[package]]`: one resolved package - -There is one array entry for every application dependency: +| `python` | required | Exact target Python, including patch version. | +| `implementation` | `"cpython"` | Required interpreter implementation. | +| `system` | empty | Optional hard `platform.system()` constraint. | +| `machine` | empty | Optional hard `platform.machine()` constraint. Common aliases such as `AMD64`/`x86_64` and `aarch64`/`arm64` are normalized. | +| `dependencies` | `[]` | Direct requirements: names, ranges, or exact pins. Direct URLs and environment markers are rejected in schema 1. Transitive packages stay out of this user file. | -| Field | Type | Required | Meaning | -| --- | --- | --- | --- | -| `name` | string | yes | Distribution name from installed metadata. Comparison is normalized according to Python package-name rules, so `python_dateutil` and `python-dateutil` identify the same project. | -| `version` | string | yes | Exact installed version, including a local suffix such as `+cu121` when present. | -| `sources` | array of strings | no | Logical `[sources.NAME]` entries allowed to satisfy this package. Each referenced source must exist. `describe -s PACKAGE=URL` creates a package-specific source when installed metadata cannot reliably reveal it. | -| `artifact` | array of tables | no | Exact wheel identities accepted for this package. Exact policy needs at least one reachable artifact route; compatible policy may omit artifacts and re-resolve the pinned version. | +An unpinned entry is intentional. During `plan`, pload selects the newest +compatible dependency graph using only index metadata, then writes exact +versions and wheel identities to the managed lock. -Users should not add or remove these records manually. Delete `.pload_lock.toml` -and run `pload lock` when a complete re-resolution is wanted. +### `[capabilities]` -### `[[package.artifact]]`: exact wheel identity +Every capability is an array of strings. `nvidia_driver` records observed +driver versions for audit and future scheduling. pload does not install CUDA +drivers. CUDA-enabled Python wheels can still be cached locally or remotely and +reused exactly, which avoids repeatedly downloading large special builds. -An artifact table belongs to the immediately preceding `[[package]]` entry: +### `[policy]` -| Field | Type | Required | Meaning | -| --- | --- | --- | --- | -| `filename` | string | yes | Exact wheel filename, including Python, ABI and platform tags. Source archives are not accepted as exact artifacts. | -| `sha256` | string | yes | Lowercase 64-character SHA-256 digest of the complete wheel. Every cache, repository and index retrieval is checked against it before installation. | -| `tags` | array of strings | no | Wheel compatibility tags extracted from the filename, such as `cp312-cp312-linux_x86_64` or `py3-none-any`. They are recorded for inspection; pip remains the final wheel-compatibility validator. | -| `repositories` | array of strings | no | Repositories known to contain this digest. The planner also checks repositories listed globally in `policy.repositories`, allowing newly cached objects to be reused without rewriting the package entry. | - -The schema permits more than one artifact entry for future multi-platform locks, -but schema 1 `describe` emits one artifact for the described target and the -environment's OS/architecture remain hard constraints. A hash proves byte -identity and detects corruption; it does not establish that a repository owner or -package is trustworthy. - -### Minimal hand-written compatible configuration - -This is the smallest practical file that asks pload to resolve pinned packages -online without claiming byte-for-byte wheel reproducibility: - -```toml -schema = 1 -name = "analysis" - -[environment] -python = "3.12.7" -dependencies = ["numpy==2.1.3", "pandas==2.2.3"] - -[policy] -reproducibility = "compatible" -network = "allow" -source_build = "forbid" -publish_missing_artifacts = false - -[sources.default] -kind = "index" -url = "https://pypi.org/simple" -``` - -`implementation`, `system`, `machine`, `capabilities` and `repositories` are -filled or omitted according to the defaults described above. `.pload_lock.toml` -is generated separately. For durable sharing, commit both files and prefer -`pload describe` so exact package and artifact identity is captured rather than -inferred. - -## Describe an existing environment - -```console -pload describe v2 -pload describe /project/.venv -o project.pload.toml -pload describe /project/.venv/bin/python -o project.pload.toml -``` - -The default `exact` mode obtains one wheel for every application package, -computes its SHA-256, places it in the shared cache and automatically publishes -it to the first configured local/SSH artifact repository. The repository is -declared in `pload.toml`; the artifact-to-repository association is recorded in -`.pload_lock.toml`. - -Use a specific configured repository when necessary: - -```console -pload describe v2 -r lab -``` +| Field | Default | Meaning | +| --- | --- | --- | +| `reproducibility` | `"exact"` | `exact` requires a locked wheel hash or a verified environment-copy route. `compatible` may install the locked version from an index when no exact artifact is known. | +| `network` | `"allow"` | `offline` forbids index access and Python installation; local and configured SSH resources remain usable. CLI `--offline` can only make policy stricter. | +| `source_build` | `"fallback"` | In compatible mode, `forbid` disallows source distributions. Exact wheel routes never build source. | +| `publish_missing_artifacts` | `false` recommended | Retained for schema-1 compatibility but ignored by `apply`. Upload is always explicit through `pload remote add`. | +| `repositories` | `[]` | Repositories to search while planning. Every name must have a matching repository table. This never authorizes upload. | -Installed metadata does not always preserve the original package index. Record a -dedicated source explicitly for packages such as CUDA-enabled PyTorch; repeat the -option for multiple packages: +### `[sources.NAME]` -```console -pload describe v2 \ - -s torch=https://download.pytorch.org/whl/cu121 \ - -s torchvision=https://download.pytorch.org/whl/cu121 -``` +| Field | Meaning | +| --- | --- | +| `kind` | Must be `"index"`. | +| `url` | Public HTTP(S) Simple API base URL. Credentials and query tokens are rejected from portable files. | -If a package has no compatible wheel, exact description stops instead of quietly -claiming reproducibility. `--mode compatible` records exact versions but permits -a future machine to re-resolve their artifacts: +Sources are searched for compatible wheel metadata. A mirror may serve wheel +files but omit PEP 658/714 Core Metadata; in that case add a metadata-capable +source such as PyPI. pload will not hide this limitation by downloading wheels +during analysis. -```console -pload describe v2 --mode compatible -``` +### `[repositories.NAME]` -Compatible mode is useful for exploration, but it is weaker than an artifact lock. +| Field | Meaning | +| --- | --- | +| `kind` | `"local"` or `"ssh"`. | +| `location` | Local path, or `HOST:/absolute/path` using normal OpenSSH configuration. | -## Lock the declared dependencies +Objects live at `objects/` and searchable package records at +`packages///.json`. Credentials remain in the +host's SSH agent/config, never in TOML. -```console -pload lock -pload lock project.pload.toml -pload lock project.pload.toml --offline -``` - -`lock` is the explicit boundary at which pload may invoke pip, resolve the full -dependency closure, obtain wheel bytes, populate the local cache and atomically -write `.pload_lock.toml`. It does not modify `pload.toml`. Run it after changing -Python, dependencies, indexes or any other resolution input. Keeping this as a -separate command prevents a read-only preview from changing the resources it is -supposed to inspect. - -## Plan against available resources +Configure providers separately, then back up selected packages explicitly: ```console -pload plan -pload plan project.pload.toml -pload plan --offline -pload plan --json +pload repo add lab frpxiaoxin:/home/tibless/pload-cloud -t ssh +pload remote add torch --from v10 --remote lab +pload remote add numpy==2.1.0 -f ./project/.venv -r lab ``` -`plan` is strictly read-only. It never invokes pip, downloads a package, fills -the cache, writes a lock or changes the user configuration. If -`.pload_lock.toml` is absent or its configuration digest is stale, the plan is -not ready: direct requirements are shown as `lock-required / pending` and the -user is told to run `pload lock`. This is intentionally different from a route -that only happens to become ready because planning downloaded it. - -With a current lock, the planner inventories: - -1. installed and discoverable Python interpreters; -2. the pload wheel cache; -3. an `artifacts/` directory beside the configuration; -4. content-addressed local and SSH repositories named in the file; -5. package indexes named in the file; -6. target OS, architecture and Python implementation. - -The Python row and package rows describe separate resources. A plan may reuse -every package from cache while still showing `Python: install-python · ... · -network`; in that case the download is the Python runtime itself. `reuse-python · -... · ready` means no runtime download is planned. +`remote add` reuses a matching original wheel when possible. If it is absent, +the command may fetch that exact installed version from `--index`; this is an +explicit storage operation, unlike `plan` and `apply`. -It first rejects candidates that violate hard requirements. Remaining candidates -are ordered lexicographically by reproducibility loss, compatibility risk, -network acquisition, execution cost and a deterministic tie-breaker. This avoids -fragile arbitrary totals where a cheap but incompatible resource could win merely -by accumulating points. +## Managed lock reference -The current first implementation chooses per-package routes and deduplicates -artifacts by SHA-256. A later global optimizer can combine connections, builds and -downloads across packages without changing the file format. +`.pload_lock.toml` contains: -## Apply desired state +| Field | Meaning | +| --- | --- | +| `schema` | Managed lock format, currently `1`. | +| `configuration` | Sibling user configuration filename. | +| `configuration_sha256` | Canonical digest that makes edited configs stale. | +| `[environment].dependencies` | Complete exact transitive closure. | +| `[[package]].name/version` | One exact resolved distribution. | +| `[[package]].sources` | Source that supplied its metadata. | +| `[[package.artifact]].filename` | Exact compatible wheel filename. | +| `sha256` | Required wheel body identity. | +| `tags` | Python/ABI/platform compatibility tags. | +| `url` | Exact artifact URL selected from the Simple API. | +| `size` | Advertised artifact bytes when the index supplies it. | +| `metadata_sha256` | Hash of the independently fetched Core Metadata when supplied. | +| `repositories` | Known repositories containing the object; policy repositories are also checked by hash. | -```console -pload apply -pload apply project.pload.toml -pload apply project.pload.toml -n project-copy -pload apply project.pload.toml --offline -``` - -`apply` performs the internal operations that were previously exposed as export, -push, pull and restore: - -1. require a current `.pload_lock.toml` and refuse implicit re-resolution; -2. resolve or install the declared Python; -3. verify implementation, exact Python version, OS and architecture; -4. reuse exact local artifacts; -5. retrieve missing SHA-addressed artifacts from configured repositories; -6. fall back to an index only when policy permits it; -7. verify every artifact before installation; -8. create and register the environment; -9. install exact packages and run `pip check`; -10. record the applied configuration digest inside the environment. +Resolution honors `Requires-Python`, wheel tags, `Requires-Dist`, dependency +markers, yanked releases, hashes and target Python/platform. A repository +without independent metadata is reported as blocked instead of causing an +analysis-time wheel download. -Running `apply` again is idempotent when the environment already carries the same -configuration digest. If the name exists with different state, pload refuses to -overwrite it. Destructive replacement is deliberately not implicit. If package -installation or verification fails, the newly created environment and registry -entry are rolled back; an existing environment is never modified. +## Managed plan reference -## Resource storage +`.pload_plan.toml` stores configuration and lock hashes, the Python action, +readiness, and one route for every locked package: -Local and SSH artifact repositories store bytes at: - -```text -objects/ -``` +```toml +schema = 1 +configuration = "pload.toml" +configuration_sha256 = "..." +lock_sha256 = "..." +name = "training" +ready = true -Two configurations referencing the same large CUDA wheel therefore store and -transfer one object. `.pload_lock.toml` contains the checksum and logical -repository name; the user does not run upload/download commands. A missing or -unreachable provider is simply removed from the candidate set, and apply tries -the next valid route. - -## Compatibility and boundaries - -- Exact locks are platform-specific unless the wheel is universal. -- A different OS, architecture or exact Python patch version currently requires - a separately described configuration. Multi-target locks are planned. -- Detected NVIDIA driver versions are recorded as capability provenance. Drivers - and system libraries are not installed by pload and are not backed up as if - they were Python artifacts. -- CUDA-enabled Python wheels are ordinary locked artifacts and can be cached in - the content repository. This preserves the large, slow-to-download package even - though the host driver remains the machine administrator's responsibility. -- Native Conda packages require a future Conda provider; they are not silently - converted to pip packages. -- Package installation may execute code. Only use configurations and providers - you trust; checksums prove identity, not trustworthiness. - -## Abandoned snapshot experiment - -The earlier `export → repo push/pull → restore` interface is not the public model -for 1.1. Its useful internals—probing, hashing, caches and content-addressed SSH -storage—remain providers behind `describe`, `plan` and `apply`. Before a stable -release, the experimental format was abandoned and is intentionally not carried -forward. +[[package]] +name = "numpy" +version = "2.2.1" +method = "external-cache" +location = "/Users/me/Library/Caches/pip/.../numpy-2.2.1.whl" +status = "ready" +estimated_seconds = 0.2 + +[package.artifact] +filename = "numpy-2.2.1-cp312-cp312-macosx_14_0_arm64.whl" +sha256 = "..." +url = "https://files.pythonhosted.org/..." +``` + +| Method | Meaning | +| --- | --- | +| `cache` | Exact SHA-matching wheel in pload's cache. | +| `configuration-artifact` | Exact wheel under the configuration's `artifacts/`. | +| `external-cache` | Exact wheel in configured or common pip/uv cache roots. | +| `environment-copy` | Exact installed distribution from a compatible pload environment, guarded by its `RECORD` fingerprint. Packages with files outside site-packages are excluded. | +| `repository` | SHA-addressed object from a local/SSH store. | +| `index-exact` | Exact locked URL, downloaded only during apply and verified by SHA-256. | +| `index-resolve` | Compatible-mode fallback for the already locked version. | + +Automatic ranking minimizes expected time: existing bytes first, compatible +local environments next, repositories next, and indexes last. The chooser can +override it. If the chosen resource changes or disappears, `apply` stops and +asks for a new plan; it never tries another candidate. + +## Current boundaries + +- Wheel reuse requires matching Python, ABI, OS and architecture tags. +- Environment copying additionally requires exact Python implementation, + version, OS, architecture and an unchanged installed `RECORD` fingerprint. +- Conda native libraries, system packages and GPU drivers are context, not + installed by the schema-1 executor. +- Source builds, editable installs and arbitrary direct URLs cannot currently + provide the same byte-for-byte guarantees as locked wheels. +- Index authentication is machine configuration; portable TOML never embeds + secrets. + +These boundaries are reported explicitly. They are never converted into an +unplanned download, fallback route, upload, or silent partial environment. diff --git a/pyproject.toml b/pyproject.toml index 74d753f..ebdf54d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pload" -version = "1.1.0a10" +version = "1.2.0a1" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" diff --git a/src/pload/__init__.py b/src/pload/__init__.py index f7f9421..7bfbef1 100644 --- a/src/pload/__init__.py +++ b/src/pload/__init__.py @@ -1 +1 @@ -__version__ = "1.1.0a10" +__version__ = "1.2.0a1" diff --git a/src/pload/cli.py b/src/pload/cli.py index 0e1014a..9504ae1 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -726,9 +726,9 @@ def render_landing(): table.add_row("pload cfg", "Show where pload stores its data") table.add_row("pload python list", "Find every usable Python interpreter") table.add_row("pload describe v1", "Describe an environment in pload.toml") - table.add_row("pload lock", "Resolve and record exact package artifacts") - table.add_row("pload plan", "Preview without downloading or changing files") - table.add_row("pload apply", "Create exactly what pload.toml declares") + table.add_row("pload plan", "Resolve metadata and choose package routes") + table.add_row("pload apply", "Execute exactly the saved package routes") + table.add_row("pload remote add torch -f v1 -r lab", "Explicitly back up one wheel") table.add_row("pload new", "Open guided environment creation") table.add_row("pload new -n data -v 3.12", "Create a named virtual environment") table.add_row("pload list", "List environments, IDs, and descriptions") diff --git a/src/pload/declarative.py b/src/pload/declarative.py index ce4e6d7..a56ca3b 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -131,7 +131,7 @@ def dump_manifest(data): f"network = {_quoted(policy.get('network', 'allow'))}", f"source_build = {_quoted(policy.get('source_build', 'fallback'))}", "publish_missing_artifacts = " + - ("true" if policy.get("publish_missing_artifacts", True) else "false"), + ("true" if policy.get("publish_missing_artifacts", False) else "false"), ] repositories = policy.get("repositories", []) if repositories: @@ -256,7 +256,7 @@ def load_manifest(path): policy.setdefault("reproducibility", "exact") policy.setdefault("network", "allow") policy.setdefault("source_build", "fallback") - policy.setdefault("publish_missing_artifacts", True) + policy.setdefault("publish_missing_artifacts", False) policy.setdefault("repositories", []) for package in data["package"]: package.setdefault("sources", []) @@ -281,7 +281,7 @@ def validate_manifest(data): raise PloadError("policy.network must be allow or offline") if policy.get("source_build", "fallback") not in {"fallback", "forbid"}: raise PloadError("policy.source_build must be fallback or forbid") - if not isinstance(policy.get("publish_missing_artifacts", True), bool): + if not isinstance(policy.get("publish_missing_artifacts", False), bool): raise PloadError("policy.publish_missing_artifacts must be true or false") policy_repositories = policy.get("repositories", []) if (not isinstance(policy_repositories, list) @@ -716,7 +716,7 @@ def describe( "reproducibility": mode, "network": "allow", "source_build": "fallback", - "publish_missing_artifacts": True, + "publish_missing_artifacts": False, "repositories": [selected_repository] if selected_repository else [], }, "sources": declared_sources, @@ -730,7 +730,6 @@ def describe( "package": [], } self.cache.mkdir(parents=True, exist_ok=True) - publication = [] with tempfile.TemporaryDirectory(prefix="pload-describe-") as temporary: wheel_dir = Path(temporary) for index_number, installed in enumerate(packages, 1): @@ -756,27 +755,12 @@ def describe( raise PloadError(f"conflicting cached artifact: {wheel.name}") if not cached.exists(): shutil.copyfile(wheel, cached) - locations = [] - if selected_repository: - publication.append((cached, checksum)) artifact = { "filename": wheel.name, "sha256": checksum, - "tags": _wheel_tags(wheel.name), "repositories": locations, + "tags": _wheel_tags(wheel.name), "repositories": [], } record["artifact"].append(artifact) data["package"].append(record) - if selected_repository and publication: - if progress: - progress( - f"Publishing {len(publication)} locked artifacts to " - f"{selected_repository}" - ) - ArtifactRepository.publish_many( - repositories[selected_repository], publication, output.parent, - ) - for package in data["package"]: - for artifact in package.get("artifact", []): - artifact["repositories"].append(selected_repository) validate_manifest(data) output.parent.mkdir(parents=True, exist_ok=True) self._write_manifest(output, data) diff --git a/src/pload/help_content.py b/src/pload/help_content.py index 6f1021e..b7c83fb 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -55,15 +55,13 @@ ```console $ pload describe v1 -o pload.toml -$ pload lock pload.toml $ pload plan pload.toml $ pload apply pload.toml ``` -`pload.toml` describes user intent and `.pload_lock.toml` records exact package -artifacts. `plan` is strictly read-only. `lock` is the explicit operation that -resolves dependencies and may download artifacts; `apply` only accepts a current -lock and materializes its selected routes. +`pload.toml` describes user intent. `plan` resolves from independent metadata, +inventories resources and saves one route per package without downloading wheel +bodies. `apply` executes only that saved plan and never changes route or uploads. """, ("new",): r""" # Create a managed environment @@ -356,9 +354,8 @@ ``` The default exact mode locks every installed application package to an actual -wheel and SHA-256. pload reuses its local cache, downloads a missing wheel when -necessary, and publishes it to the selected configured local/SSH repository. -These are internal resource operations: the user receives one TOML file. +wheel and SHA-256. pload reuses its local cache and may obtain a missing wheel, +but does not upload it. Use `pload remote add PACKAGE` for explicit backup. Use `--mode compatible` only when exact wheels do not exist and future re-resolution is acceptable. Native Conda packages are not silently converted. @@ -374,15 +371,14 @@ pload plan project.pload.toml pload plan --offline pload plan --json +pload plan --no-ui ``` -Planning reads the configuration, discovers Python interpreters and checks local -caches, adjacent artifacts and content-addressed repositories. -Incompatible resources are rejected first; valid routes are ordered by exactness, -compatibility risk, transfer, execution cost and a deterministic tie-breaker. -It never invokes pip, downloads packages, writes a lock or changes either TOML -file. A missing or stale lock is reported as `lock-required / pending`; run -`pload lock` explicitly before planning exact resource routes. +Planning resolves missing/stale locks using Simple API pages and independent +Core Metadata, then checks pload/pip/uv caches, compatible environments, +adjacent artifacts and content-addressed repositories. It never downloads a +wheel body. In a terminal the chooser supports previous/next, undo, reset, +save, and save-and-apply; `--no-ui` selects the fastest routes for automation. """ GUIDES[("lock",)] = r""" # Resolve the user declaration into an exact managed lock @@ -393,11 +389,9 @@ pload lock --offline ``` -Locking is an explicit mutating operation. It resolves direct and transitive -dependencies for the configured Python, obtains exact wheels when necessary, -computes SHA-256 identities and atomically writes `.pload_lock.toml` beside the -configuration. `pload.toml` is never rewritten. Run it after changing Python, -dependencies, sources or another resolution input. +This legacy explicit command is retained for preview compatibility. The normal +workflow is now `pload plan`, whose metadata-only resolver writes the managed +lock without obtaining wheel bodies. Prefer `plan` for new configurations. """ GUIDES[("apply",)] = r""" # Materialize the configuration @@ -408,10 +402,10 @@ pload apply --offline ``` -Apply requires a current `.pload_lock.toml`, then installs Python when needed, -obtains exact artifacts through the selected -resource routes, verifies hashes, creates the environment, installs packages and -runs `pip check`. It never silently resolves or updates a stale lock. Reapplying +Apply requires a current `.pload_lock.toml` and `.pload_plan.toml`, then installs +Python when needed, executes each selected route, verifies hashes, creates the +environment, installs packages and runs `pip check`. It never resolves, tries an +alternative route, or uploads package bytes. Reapplying the same configuration is idempotent; an existing environment with different state is never silently overwritten. """ diff --git a/tests/test_declarative.py b/tests/test_declarative.py index afdf6e8..62d3e5f 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -498,10 +498,12 @@ def test_describe_plan_apply_through_content_repository(tmp_path): assert data["sources"]["default"]["url"] == "https://example.invalid/simple" assert locked["package"][0]["sources"] == ["package-pload-demo"] assert data["sources"]["package-pload-demo"]["url"].endswith("/cu121") - assert artifact["repositories"] == ["lab"] + assert artifact["repositories"] == [] + assert not (repository / "objects" / artifact["sha256"]).exists() + manager.publish_package("pload-demo", "source", "lab") assert (repository / "objects" / artifact["sha256"]).is_file() assert any("Locking pload-demo==1.0" in item for item in describe_progress) - assert any("Publishing 1 locked artifact" in item for item in describe_progress) + assert not any("Publishing" in item for item in describe_progress) shutil.rmtree(source_config.home / "cache") target_config = ConfigManager(home=tmp_path / "target-home") From 4b3c93499b0f0f3032c00a7fc22e26e9cf48a6d8 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Sun, 27 Sep 2026 17:05:58 +0800 Subject: [PATCH 22/41] Support resolving multiple root requirements --- src/pload/metadata_resolver.py | 7 ++++++ tests/test_metadata_resolver.py | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/pload/metadata_resolver.py b/src/pload/metadata_resolver.py index 4af06f1..a9e944e 100644 --- a/src/pload/metadata_resolver.py +++ b/src/pload/metadata_resolver.py @@ -245,6 +245,13 @@ def is_satisfied_by(requirement, candidate): def get_dependencies(self, candidate): return self.index.dependencies(candidate, self.marker_environment) + @staticmethod + def narrow_requirement_selection( + identifiers, resolutions, candidates, information, backtrack_causes, + ): + """Keep every active criterion; ordering remains get_preference's job.""" + return identifiers + def resolve_metadata(requirements, sources, environment, supported_tags, fetch=None): """Return exact packages and wheel metadata without fetching any wheel body.""" diff --git a/tests/test_metadata_resolver.py b/tests/test_metadata_resolver.py index f51465a..095e175 100644 --- a/tests/test_metadata_resolver.py +++ b/tests/test_metadata_resolver.py @@ -84,3 +84,42 @@ def fetch(url, accept, maximum): {Tag("py3", "none", "any")}, fetch=fetch, ) assert requested == ["https://index.invalid/simple/root-demo/"] + + +def test_resolver_supports_multiple_root_requirements(): + requested = [] + + def fetch(url, accept, maximum): + requested.append(url) + name = url.rstrip("/").rsplit("/", 1)[-1] + if url.endswith(".metadata"): + version = "1.0" if "first_demo" in url else "2.0" + project = "first-demo" if "first_demo" in url else "second-demo" + return ( + f"Metadata-Version: 2.1\nName: {project}\nVersion: {version}\n\n".encode(), + "application/octet-stream", + ) + distribution = name.replace("-", "_") + version = "1.0" if name == "first-demo" else "2.0" + filename = f"{distribution}-{version}-py3-none-any.whl" + metadata = ( + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n\n".encode() + ) + return ( + _project(filename, ("a" if name == "first-demo" else "b") * 64, + hashlib.sha256(metadata).hexdigest()), + "application/vnd.pypi.simple.v1+json", + ) + + packages = resolve_metadata( + ["first-demo", "second-demo"], + [("default", "https://index.invalid/simple")], + {"python": "3.12.1", "implementation": "cpython", + "system": "Linux", "machine": "x86_64"}, + {Tag("py3", "none", "any")}, fetch=fetch, + ) + + assert [(item["name"], item["version"]) for item in packages] == [ + ("first-demo", "1.0"), ("second-demo", "2.0"), + ] + assert not any(url.endswith(".whl") for url in requested) From d545dea21a85319b2924edca772d0f4cf07ca327 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Sun, 27 Sep 2026 17:20:52 +0800 Subject: [PATCH 23/41] Limit route choices to direct requirements --- CHANGELOG.md | 3 ++- README.md | 5 +++-- docs/declarative-environments.md | 17 ++++++++++------ src/pload/cli.py | 30 +++++++++++++++++++++-------- src/pload/declarative.py | 5 +++++ src/pload/help_content.py | 6 ++++-- src/pload/resource_plan.py | 8 ++++++++ tests/test_declarative.py | 33 ++++++++++++++++++++++++++++++++ tests/test_resource_plan.py | 18 +++++++++++++++++ 9 files changed, 106 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0e59fa..530b42c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,8 @@ Pre-release: `1.2.0a1` in compatible pload environments. Recheck SHA-256 or installed `RECORD` fingerprints during apply. - Add an interactive per-package route chooser with previous/next navigation, - undo, reset-to-fastest, save and save-and-apply actions. + undo, reset-to-fastest, save and save-and-apply actions. Show only direct + user requirements; route transitive dependencies automatically. - Add explicit `pload remote add PACKAGE --from ENV --remote REPO`; apply and describe never publish package bytes automatically. - Persist exact artifact URLs and hashes, reject stale or incomplete plans, and diff --git a/README.md b/README.md index 3d8f160..e0d9098 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,9 @@ pload apply pload.toml `plan` fetches only index pages and independent package metadata—never wheel bodies—and inventories pload/pip/uv caches, compatible existing environments, -local/SSH stores and indexes. Its interactive chooser supports previous/next, -undo, reset, save and save-and-apply. `apply` executes exactly the saved plan: +local/SSH stores and indexes. Its interactive chooser shows only dependencies +written by the user; transitive dependencies are routed automatically. It +supports previous/next, undo, reset, save and save-and-apply. `apply` executes exactly the saved plan: it never re-resolves, changes route, falls back, or uploads packages. Back up a special or slow wheel only when you explicitly request it: diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index c3d7f3c..e71949d 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -25,10 +25,12 @@ pload plan pload.toml # resolve metadata, inventory and choose routes pload apply pload.toml # execute only the saved plan ``` -In a terminal, `plan` opens a route chooser. A number selects a route; Enter or -`n` moves forward, `p` goes back, `u` undoes, `r` restores the fastest choices, -`s` saves, and `a` saves then applies. Use `pload plan --no-ui` in CI. Use -`--json` for machine-readable inspection. +In a terminal, `plan` opens a route chooser for only the direct requirements +the user wrote in `pload.toml`. Transitive dependencies remain exact and +auditable, but pload automatically assigns their fastest valid routes. A number +selects a route; Enter or `n` moves forward, `p` goes back, `u` undoes, `r` +restores the fastest choices, `s` saves, and `a` saves then applies. Use +`pload plan --no-ui` in CI. Use `--json` for machine-readable inspection. ## Three files, one user-owned file @@ -223,8 +225,11 @@ url = "https://files.pythonhosted.org/..." Automatic ranking minimizes expected time: existing bytes first, compatible local environments next, repositories next, and indexes last. The chooser can -override it. If the chosen resource changes or disappears, `apply` stops and -asks for a new plan; it never tries another candidate. +override it for direct requirements. Transitive requirements use automatic +ranking so installing `torch` does not ask the user to decide separately for +`filelock`, `sympy`, and every other implementation detail. If any chosen +resource changes or disappears, `apply` stops and asks for a new plan; it never +tries another candidate. ## Current boundaries diff --git a/src/pload/cli.py b/src/pload/cli.py index 9504ae1..0a5492c 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -491,7 +491,11 @@ def print_declarative_plan(plan): table.add_column("METHOD", style="cyan", no_wrap=True) table.add_column("STATUS", no_wrap=True) table.add_column("RESOURCE") - for package in plan["packages"]: + direct_packages = [ + package for package in plan["packages"] if package.get("direct", True) + ] + transitive_count = len(plan["packages"]) - len(direct_packages) + for package in direct_packages: selected = package["selected"] if selected: table.add_row(package["name"], package["version"], selected["method"], @@ -505,6 +509,11 @@ def print_declarative_plan(plan): table.add_row(package["name"], package["version"], "—", "unavailable", "No resource satisfies the configuration") console.print(table) + if transitive_count: + console.print( + f"[dim]+ {transitive_count} transitive dependencies planned " + "automatically using their fastest valid routes[/]" + ) console.print( "[dim]Saved acquisition choices:[/] " f"[cyan]{plan['plan_path']}[/]" @@ -517,17 +526,22 @@ def choose_declarative_routes(plan): console = Console(highlight=False) selection = PlanSelection(plan) - if not plan["packages"]: + decision_indices = selection.decision_indices() + if not decision_indices: return "save" cursor = 0 while True: packages = plan["packages"] - package = packages[cursor] - routes = selection.routes(cursor) + package_index = decision_indices[cursor] + package = packages[package_index] + routes = selection.routes(package_index) console.clear() console.print(Panel.fit( f"[bold green]{package['name']}[/]==[yellow]{package['version']}[/]\n" - f"Package [bold]{cursor + 1}[/] of [bold]{len(packages)}[/]", + f"Direct requirement [bold]{cursor + 1}[/] of " + f"[bold]{len(decision_indices)}[/]\n" + f"[dim]{len(packages) - len(decision_indices)} transitive dependencies " + "will be planned automatically[/]", title="[bold cyan]Choose acquisition route[/]", border_style="blue", )) table = Table(box=box.ROUNDED, header_style="bold cyan", border_style="blue") @@ -550,13 +564,13 @@ def choose_declarative_routes(plan): command = console.input("[bold cyan]route> [/]").strip().lower() if command.isdigit(): try: - selection.select(cursor, int(command) - 1) + selection.select(package_index, int(command) - 1) except IndexError: continue - if cursor < len(packages) - 1: + if cursor < len(decision_indices) - 1: cursor += 1 elif command in {"", "n", "next"}: - cursor = min(cursor + 1, len(packages) - 1) + cursor = min(cursor + 1, len(decision_indices) - 1) elif command in {"p", "previous", "back"}: cursor = max(cursor - 1, 0) elif command in {"u", "undo"}: diff --git a/src/pload/declarative.py b/src/pload/declarative.py index a56ca3b..ff9bc6b 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -1051,6 +1051,9 @@ def lock(self, manifest_path, offline=False, progress=None): def plan(self, manifest_path, offline=False, progress=None): path, data, current, legacy = self._load_state(manifest_path) dependencies = data["environment"].get("dependencies", []) + direct_names = { + normalized_name(_parse_dependency(value).name) for value in dependencies + } lock_required = bool(dependencies) and not current sidecar = lock_path(path) lock = { @@ -1107,6 +1110,7 @@ def plan(self, manifest_path, offline=False, progress=None): pending.append({ "name": requirement.name, "version": exact.group(2) if exact else str(requirement.specifier) or "unresolved", + "direct": True, "selected": self._candidate( "lock-required", sidecar, "pending", (0, 0, 0, 0) ), @@ -1214,6 +1218,7 @@ def plan(self, manifest_path, offline=False, progress=None): candidates.sort(key=lambda item: item["cost"]) plans.append({ "name": package["name"], "version": package["version"], + "direct": normalized_name(package["name"]) in direct_names, "selected": candidates[0] if candidates else None, "alternatives": candidates[1:], "rejections": rejections, diff --git a/src/pload/help_content.py b/src/pload/help_content.py index b7c83fb..aa7f7a5 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -377,8 +377,10 @@ Planning resolves missing/stale locks using Simple API pages and independent Core Metadata, then checks pload/pip/uv caches, compatible environments, adjacent artifacts and content-addressed repositories. It never downloads a -wheel body. In a terminal the chooser supports previous/next, undo, reset, -save, and save-and-apply; `--no-ui` selects the fastest routes for automation. +wheel body. The chooser shows only requirements written in `pload.toml` and +automatically assigns routes to their transitive dependencies. It supports +previous/next, undo, reset, save, and save-and-apply; `--no-ui` selects the +fastest routes for automation. """ GUIDES[("lock",)] = r""" # Resolve the user declaration into an exact managed lock diff --git a/src/pload/resource_plan.py b/src/pload/resource_plan.py index c505d5c..63a30cc 100644 --- a/src/pload/resource_plan.py +++ b/src/pload/resource_plan.py @@ -27,6 +27,13 @@ def routes(self, package_index): package.get("alternatives", []) ) + def decision_indices(self): + """Only direct user requirements need interactive route decisions.""" + return [ + index for index, package in enumerate(self.plan["packages"]) + if package.get("direct", True) + ] + def select(self, package_index, route_index): routes = self.routes(package_index) if route_index < 0 or route_index >= len(routes): @@ -88,6 +95,7 @@ def dump_plan(configuration_path, configuration_sha256, lock_sha256, plan): "[[package]]", f"name = {_quoted(package['name'])}", f"version = {_quoted(package['version'])}", + f"direct = {'true' if package.get('direct', True) else 'false'}", f"method = {_quoted(selected.get('method', 'unavailable'))}", f"location = {_quoted(selected.get('location', ''))}", f"status = {_quoted(selected.get('status', 'unavailable'))}", diff --git a/tests/test_declarative.py b/tests/test_declarative.py index 62d3e5f..aed76cf 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -422,6 +422,39 @@ def fake_metadata(requirements, sources, environment, supported_tags): assert output == "42" +def test_plan_marks_only_user_declared_packages_as_direct(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + root_wheel = tiny_wheel(tmp_path / "fixture", distribution="root_demo") + child_wheel = tiny_wheel(tmp_path / "fixture", distribution="child_demo") + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": ["root-demo"], + }) + data["package"] = [] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data, with_lock=False) + + def fake_metadata(requirements, sources, environment, supported_tags): + return [ + {"name": name, "version": "1.0", "sources": ["default"], + "artifact": [{"filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": []}]} + for name, wheel in (("child-demo", child_wheel), ("root-demo", root_wheel)) + ] + + monkeypatch.setattr(declarative_module, "resolve_metadata", fake_metadata) + plan = DeclarativeEnvironmentManager(config).plan(manifest) + + assert {item["name"]: item["direct"] for item in plan["packages"]} == { + "child-demo": False, "root-demo": True, + } + + def test_plan_refreshes_stale_lock_using_only_metadata(tmp_path, monkeypatch): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) diff --git a/tests/test_resource_plan.py b/tests/test_resource_plan.py index 3e55759..81f9249 100644 --- a/tests/test_resource_plan.py +++ b/tests/test_resource_plan.py @@ -32,3 +32,21 @@ def test_route_selection_supports_navigation_state_undo_and_reset(): assert plan["packages"][0]["selected"]["method"] == "cache" assert selection.undo() is True assert plan["packages"][0]["selected"]["method"] == "environment-copy" + + +def test_only_direct_requirements_are_interactive_decisions(): + plan = { + "python": {"status": "ready"}, "ready": True, + "packages": [ + {"name": "torch", "direct": True, "selected": _route("cache"), + "alternatives": []}, + {"name": "filelock", "direct": False, "selected": _route("cache"), + "alternatives": [_route("index-exact")]}, + {"name": "numpy", "direct": True, "selected": _route("cache"), + "alternatives": []}, + ], + } + + selection = PlanSelection(plan) + + assert selection.decision_indices() == [0, 2] From 10ab7d291ba4f4324a17ef694822ffd2966b5a2b Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Sun, 27 Sep 2026 17:48:02 +0800 Subject: [PATCH 24/41] Unify terminal interaction design --- CHANGELOG.md | 6 + README.md | 24 ++- docs/terminal-experience.md | 48 ++++++ pyproject.toml | 1 + src/pload/cli.py | 230 ++++++++++++++--------------- src/pload/display.py | 42 +++--- src/pload/help_content.py | 18 +-- src/pload/installer.py | 243 ++++++++++++++++++++----------- src/pload/managers/color.py | 42 ------ src/pload/managers/dependency.py | 4 +- src/pload/managers/venv.py | 13 +- src/pload/ui.py | 221 ++++++++++++++++++++++++++++ tests/test_cli.py | 15 +- tests/test_installer.py | 19 ++- tests/test_ui.py | 77 ++++++++++ 15 files changed, 713 insertions(+), 290 deletions(-) create mode 100644 docs/terminal-experience.md delete mode 100644 src/pload/managers/color.py create mode 100644 src/pload/ui.py create mode 100644 tests/test_ui.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 530b42c..f4e5592 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,12 @@ Pre-release: `1.2.0a1` ### Added +- Introduce one terminal design system for prompts, tables, progress, success, + warning and error output. Guided creation and route planning now use familiar + cursor-key menus with numbered non-TTY fallbacks instead of bespoke command + loops. +- Respect `NO_COLOR` consistently and keep animation limited to real terminals; + set `PLOAD_NO_PROGRESS=1` to replace spinners with plain status text. - Keep user intent in a clean portable `pload.toml` and store pload-managed exact dependency, artifact and hash decisions in sibling `.pload_lock.toml`. - Add `pload describe`, metadata-only `pload plan` and idempotent `pload apply`. diff --git a/README.md b/README.md index e0d9098..b128ecb 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ complete parameter reference and `-h -d` for practical examples and effects. - [Discover and install Python](#discover-and-install-python) - [Create and activate environments](#create-and-activate-environments) - [Help and aliases](#help-and-aliases) +- [Terminal experience](#terminal-experience) - [Isolation and directory layout](#isolation-and-directory-layout) - [Command reference](#command-reference) - [Development](#development) @@ -52,10 +53,11 @@ pipx install pload pload-install ``` -The installer uses colored numbered choices for the pload data directory, +The installer uses consistent cursor-key choices for the pload data directory, launcher directory, virtual-environment directory, Python directory, runtime -source, pip index, and optional shell integration. Press Enter to accept a -marked default. +source, pip index, and optional shell integration. Use `↑`/`↓` to move and +Enter to accept the highlighted value. Without a terminal UI, the same prompts +fall back to numbered choices. After installation it prints an ASCII welcome screen with copyable next steps. The same screen is shown whenever you run bare `pload` (see the screenshot at @@ -92,7 +94,8 @@ pload apply pload.toml bodies—and inventories pload/pip/uv caches, compatible existing environments, local/SSH stores and indexes. Its interactive chooser shows only dependencies written by the user; transitive dependencies are routed automatically. It -supports previous/next, undo, reset, save and save-and-apply. `apply` executes exactly the saved plan: +supports previous/next, undo, reset, save and save-and-apply through a +cursor-key menu. `apply` executes exactly the saved plan: it never re-resolves, changes route, falls back, or uploads packages. Back up a special or slow wheel only when you explicitly request it: @@ -383,6 +386,19 @@ pload python install 3.12 pload py p 3.12 ``` +## Terminal experience + +All interactive screens share the same keyboard and visual language. Use +`↑`/`↓` and Enter in guided creation, setup, and package-route selection; +recommended values start highlighted. Status, success, warning, and error +messages use the same symbols and colors everywhere, while compact border-light +tables keep lists readable. + +`NO_COLOR=1` disables color. `PLOAD_NO_PROGRESS=1` replaces animated progress +with plain status lines. Every guided workflow also has an explicit flag-based +form for scripts. See the [terminal experience specification](docs/terminal-experience.md) +for the complete interaction and output contract. + ## Isolation and directory layout Put all pload-managed data under one root: diff --git a/docs/terminal-experience.md b/docs/terminal-experience.md new file mode 100644 index 0000000..81a3b1f --- /dev/null +++ b/docs/terminal-experience.md @@ -0,0 +1,48 @@ +# Terminal experience + +pload uses one interaction language across installation, environment creation, +planning, configuration, and destructive actions. The goal is predictable +behaviour, not decoration. + +## Interaction model + +- A normal command prints a compact result and exits. It never opens a full + screen interface unexpectedly. +- A command that needs a choice uses `↑`/`↓` and Enter. The active row is + prefixed with `❯`; the recommended value is selected initially. +- `pload new` is guided when no creation options are supplied. Every guided + operation also has flags for scripts and automation. +- Destructive actions use a conventional `y/N` confirmation. `--yes` is the + explicit non-interactive alternative. +- `Ctrl-C` cancels without a traceback. + +## Visual language + +| Treatment | Meaning | +| --- | --- | +| cyan | current focus, headings, and resources | +| green / `✓` | completed successfully | +| yellow / `!` | warning or required attention | +| red / `error:` | command failure | +| dim text | paths, provenance, and secondary explanation | + +Tables are compact and border-light so rows remain readable at normal terminal +widths. Progress animation appears only on a TTY. `NO_COLOR=1` disables colour, +and `PLOAD_NO_PROGRESS=1` replaces spinners with plain text. + +## Output contract + +Primary results remain suitable for copying or piping. JSON modes write only +JSON to stdout. Progress and diagnostics use stderr where a command exposes +machine-readable output. Redirected input uses explicit flags or numbered +fallbacks instead of emitting terminal control sequences. + +## Design references + +The design follows [Atuin's compact keyboard-driven interaction](https://docs.atuin.sh/main/reference/search/), +[uv's concise help and TTY-aware progress](https://docs.astral.sh/uv/getting-started/help/), +[GitHub CLI's explicit prompt and accessibility controls](https://cli.github.com/manual/gh_help_environment), +[Cargo's automatic terminal capability detection](https://doc.rust-lang.org/cargo/reference/config.html#termprogresswhen), +and the human-first principles of the [Command Line Interface Guidelines](https://clig.dev/). +pload intentionally uses these conventions without turning every command into +a full-screen TUI. diff --git a/pyproject.toml b/pyproject.toml index ebdf54d..8f7fd95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ ] dependencies = [ "packaging>=23.2,<27", + "questionary>=2.0.1,<3", "rich>=13.7,<15", "resolvelib>=1.1,<1.2", "tomli>=2; python_version < '3.11'", diff --git a/src/pload/cli.py b/src/pload/cli.py index 0a5492c..4f72c84 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -4,13 +4,13 @@ import sys from pathlib import Path +from questionary import Choice from rich import box -from rich.console import Console from rich.panel import Panel from rich.table import Table from rich.text import Text -from pload import __version__ +from pload import __version__, ui from pload.display import print_environment_table, print_python_table from pload.errors import PloadError from pload.help_content import render_detailed_help @@ -55,7 +55,7 @@ def build_parser(): - parser = argparse.ArgumentParser( + parser = ui.ArgumentParser( prog="pload", description=( "Create, activate, and remove Python virtual environments without tying " @@ -472,25 +472,28 @@ def _global_options_table(root_parser): def print_declarative_plan(plan): - console = Console(highlight=False) + console = ui.console() if plan["lock"]["required"]: console.print( f"[bold yellow]Lock: {plan['lock']['status']}[/] · metadata was unavailable " "offline; run [bold green]pload plan[/] with network access" ) python = plan["python"] - console.print(Panel.fit( - f"[bold]{plan['name']}[/]\nPython: [cyan]{python['method']}[/] · " - f"{python['location']} · [bold]{python['status']}[/]", - title="[bold cyan]Environment plan[/]", - border_style="blue", - )) - table = Table(box=box.ROUNDED, header_style="bold cyan", border_style="blue") - table.add_column("PACKAGE", style="bold green", no_wrap=True) - table.add_column("VERSION", style="yellow", no_wrap=True) - table.add_column("METHOD", style="cyan", no_wrap=True) - table.add_column("STATUS", no_wrap=True) - table.add_column("RESOURCE") + ui.heading( + f"Plan · {plan['name']}", + f"Python {python['method']} · {python['status']}", + output=console, + ) + console.print(f"[dim]{python['location']}[/]", overflow="ellipsis", no_wrap=True) + table = ui.table( + ("PACKAGE", "bold", {"width": 15, "overflow": "ellipsis", "no_wrap": True}), + ("VERSION", "yellow", {"width": 10, "no_wrap": True}), + ("SOURCE", "cyan", {"width": 12, "overflow": "ellipsis", "no_wrap": True}), + ("STATUS", "", {"width": 10, "no_wrap": True}), + ("RESOURCE", "dim", {"min_width": 14, "ratio": 1, + "overflow": "ellipsis", "no_wrap": True}), + expand=True, + ) direct_packages = [ package for package in plan["packages"] if package.get("direct", True) ] @@ -521,10 +524,10 @@ def print_declarative_plan(plan): def choose_declarative_routes(plan): - """Navigate every package route with backtracking and undo in a terminal.""" + """Choose direct-package routes with familiar cursor-key menus.""" from pload.resource_plan import PlanSelection - console = Console(highlight=False) + console = ui.console() selection = PlanSelection(plan) decision_indices = selection.decision_indices() if not decision_indices: @@ -535,53 +538,53 @@ def choose_declarative_routes(plan): package_index = decision_indices[cursor] package = packages[package_index] routes = selection.routes(package_index) - console.clear() - console.print(Panel.fit( - f"[bold green]{package['name']}[/]==[yellow]{package['version']}[/]\n" - f"Direct requirement [bold]{cursor + 1}[/] of " - f"[bold]{len(decision_indices)}[/]\n" - f"[dim]{len(packages) - len(decision_indices)} transitive dependencies " - "will be planned automatically[/]", - title="[bold cyan]Choose acquisition route[/]", border_style="blue", - )) - table = Table(box=box.ROUNDED, header_style="bold cyan", border_style="blue") - table.add_column("#", justify="right", style="bold yellow") - table.add_column("METHOD", style="green") - table.add_column("TIME", justify="right") - table.add_column("RESOURCE") + ui.heading( + f"Choose source · {package['name']}=={package['version']}", + f"Direct requirement {cursor + 1}/{len(decision_indices)} · " + f"{len(packages) - len(decision_indices)} dependencies handled automatically", + output=console, + ) + choices = [] for index, route in enumerate(routes, 1): - marker = "✓" if index == 1 else str(index) seconds = route.get("estimated_seconds", 0) - table.add_row(marker, route["method"], f"~{seconds:g}s", route["location"]) - if routes: - console.print(table) - else: - console.print("[bold red]No valid route is currently available.[/]") - console.print( - "[dim]number choose · Enter/n next · p previous · u undo · " - "r reset fastest · s save · a save & apply · q quit[/]" + recommended = " recommended" if index == 1 else "" + choices.append(Choice( + f"{route['method']:<18} ~{seconds:g}s {route['location']}{recommended}", + value=("route", index - 1), + )) + if cursor: + choices.append(Choice("← Previous package", value=("previous", None))) + if cursor < len(decision_indices) - 1: + choices.append(Choice("→ Keep current choice and continue", value=("next", None))) + choices.extend([ + Choice("↶ Undo last change", value=("undo", None)), + Choice("↺ Reset all to recommended", value=("reset", None)), + Choice("✓ Save plan", value=("save", None)), + Choice("▶ Save and apply", value=("apply", None)), + Choice("× Cancel", value=("quit", None)), + ]) + command, value = ui.select( + "Select an acquisition source", + choices, + default=("route", 0) if routes else ("quit", None), ) - command = console.input("[bold cyan]route> [/]").strip().lower() - if command.isdigit(): - try: - selection.select(package_index, int(command) - 1) - except IndexError: - continue + if command == "route": + selection.select(package_index, value) if cursor < len(decision_indices) - 1: cursor += 1 - elif command in {"", "n", "next"}: + elif command == "next": cursor = min(cursor + 1, len(decision_indices) - 1) - elif command in {"p", "previous", "back"}: + elif command == "previous": cursor = max(cursor - 1, 0) - elif command in {"u", "undo"}: + elif command == "undo": selection.undo() - elif command in {"r", "reset"}: + elif command == "reset": selection.reset() - elif command in {"s", "save"}: + elif command == "save": return "save" - elif command in {"a", "apply"}: + elif command == "apply": return "apply" - elif command in {"q", "quit"}: + elif command == "quit": return "quit" @@ -604,10 +607,11 @@ def _print_arrow_table(console, title, table): def _brief_help(parser, command_path, root_parser=None): - console = Console(highlight=False) + console = ui.console() title = f"pload {__version__}" if not command_path else "pload " + " ".join(command_path) description = parser.description or "Command-line help" usage = " ".join(parser.format_usage().split()) + usage = re.sub(r"\{[^}]+\} \.\.\.", "", usage) if command_path and usage.startswith(f"usage: {parser.prog}"): usage = usage.replace( f"usage: {parser.prog}", @@ -723,16 +727,8 @@ def render_help(argv): def render_landing(): """Show the no-argument welcome screen and the smallest useful command tour.""" - console = Console(highlight=False) - art = Text( - """ ____ _ _ - | _ \\| | ___ __ _ __| | - | |_) | |/ _ \\ / _` |/ _` | - | __/| | (_) | (_| | (_| | - |_| |_|\\___/ \\__,_|\\__,_|""", - style="bold cyan", - ) - console.print(art) + console = ui.console() + ui.logo(output=console) console.print(f"[bold white]pload {__version__}[/] [dim]Python environments, kept simple.[/]") table = Table(box=box.SIMPLE, header_style="bold cyan", show_edge=False) table.add_column("COMMAND", style="bold green", no_wrap=True) @@ -802,34 +798,40 @@ def shell_script(shell): def _guided_new(config): """Collect useful creation choices when ``pload new`` is run by itself.""" - console = Console(highlight=False) - console.print(Panel.fit( - "Choose an interpreter and an optional name. Press Enter to accept defaults.", - title="[bold cyan]Guided environment creation[/]", - border_style="blue", - )) + console = ui.console() + ui.heading( + "Create an environment", + "Review each value, then create. Nothing is changed until the final step.", + output=console, + ) runtimes = PythonManager(config).discover() if runtimes: - console.print("[dim]Use a Python ID, alias, version, or interpreter path:[/]") - print_python_table(runtimes) + runtime_choices = [ + Choice( + f"{runtime.id or '—':<5} {runtime.version:<10} " + f"{runtime.source:<10} {runtime.path}", + value=runtime.id or runtime.alias or str(runtime.path), + ) + for runtime in runtimes + ] + runtime_choices.append(Choice("Enter a version or path…", value="__custom__")) + version = ui.select( + "Python interpreter", + runtime_choices, + default=runtime_choices[0].value, + ) + if version == "__custom__": + version = ui.text("Python version or executable path", default="current") else: console.print("[yellow]No Python was discovered; the current Python is the default.[/]") - - def ask(label, default=""): - suffix = f" [{default}]" if default else "" - value = input(f"{label}{suffix}: ").strip() - return value or default - - version = ask("Python ID, alias, version, or path", "current") + version = "current" _, suggested_name = config.resolve_venv_path(version=version) - name = ask("Environment name (blank generates one)") - if not name: - console.print(f"[dim]Generated name: [bold]{suggested_name}[/][/dim]") - description = ask("Description (optional)") - packages = ask("Packages, separated by spaces (optional)") + name = ui.text("Environment name", default=suggested_name) + description = ui.text("Description (optional)") + packages = ui.text("Packages, separated by spaces (optional)") return { "python_version": version, - "name": name or None, + "name": name, "description": description or None, "requirements": packages.split() or None, } @@ -868,13 +870,13 @@ def run(argv=None): result = manager.publish_package( args.package, args.environment, args.repository, args.index, ) - Console(highlight=False).print( - f"[bold green]✓ Backed up {result['package']}=={result['version']}[/] " - f"to [cyan]{result['repository']}[/]\n" - f"[dim]{result['filename']} · sha256:{result['sha256']}[/]" + ui.success( + f"Backed up {result['package']}=={result['version']} " + f"to {result['repository']}", + detail=f"{result['filename']} · sha256:{result['sha256']}", ) elif command == "describe": - progress_console = Console(highlight=False) + progress_console = ui.console() print(manager.describe( args.source, args.output, args.name, args.mode, args.repository, args.package_sources, @@ -883,38 +885,29 @@ def run(argv=None): ), )) elif command == "lock": - progress_console = Console(highlight=False, stderr=True) - with progress_console.status( - "[cyan]Reading configuration…[/]", spinner="dots", - ) as status: + with ui.status("Reading configuration…") as update_status: result = manager.lock( args.file, args.offline, - progress=lambda message: status.update(f"[cyan]{message}…[/]"), + progress=lambda message: update_status(f"{message}…"), ) count = len(result["resolved"]) action = "Wrote" if result["updated"] else "Reused" - Console(highlight=False).print( - f"[bold green]✓ {action} {count} locked packages[/] in " - f"[cyan]{result['path']}[/]" + ui.success( + f"{action} {count} locked packages", + detail=str(result["path"]), ) elif command == "plan": if args.json: plan = manager.plan(args.file, args.offline) print(json.dumps(plan, ensure_ascii=False, indent=2)) else: - progress_console = Console(highlight=False, stderr=True) - with progress_console.status( - "[cyan]Reading configuration and available resources…[/]", - spinner="dots", - ) as status: + with ui.status("Reading configuration and available resources…") as update_status: plan = manager.plan( args.file, args.offline, - progress=lambda message: status.update( - f"[cyan]{message}…[/]" - ), + progress=lambda message: update_status(f"{message}…"), ) action = "save" - chooser_console = Console(highlight=False) + chooser_console = ui.console() if chooser_console.is_terminal and not args.no_ui: action = choose_declarative_routes(plan) if action != "quit": @@ -923,7 +916,7 @@ def run(argv=None): if action == "apply": chooser_console.print(manager.apply(args.file, offline=args.offline)) else: - progress_console = Console(highlight=False) + progress_console = ui.console() print(manager.apply( args.file, args.name, args.offline, progress=lambda message: progress_console.print( @@ -969,11 +962,15 @@ def run(argv=None): names = list(dict.fromkeys(names)) if not names: raise PloadError("no environments selected") + if not args.yes: + ui.heading("Remove environments", "\n".join(f" {name}" for name in names)) + if not ui.confirm( + f"Remove {len(names)} environment{'s' if len(names) != 1 else ''}?", + default=False, + ): + ui.warning("Removal cancelled") + return 0 for name in names: - if not args.yes: - confirmation = input(f"Remove {name!r}? Type its name to confirm: ") - if confirmation != name: - raise PloadError("removal cancelled") venvs.remove_venv(name, project_dir=args.project_dir) return 0 @@ -1026,7 +1023,7 @@ def run(argv=None): if not args.key or args.value is None: raise PloadError("usage: pload cfg set SETTING VALUE") path = _set_config_value(config, args.key, args.value) - print(f"[*] Updated configuration: {path}") + ui.success("Updated configuration", detail=str(path)) return 0 effective = dict(config.settings) effective.update({ @@ -1044,8 +1041,11 @@ def run(argv=None): def main(argv=None): try: return run(argv) + except (KeyboardInterrupt, EOFError): + ui.warning("Cancelled") + return 130 except (PloadError, PythonNotFoundError, re.error, OSError) as exc: - print(f"pload: error: {exc}", file=sys.stderr) + ui.error(str(exc)) return 1 diff --git a/src/pload/display.py b/src/pload/display.py index f1b3bd1..84e4084 100644 --- a/src/pload/display.py +++ b/src/pload/display.py @@ -1,19 +1,16 @@ -from rich import box -from rich.console import Console -from rich.table import Table - - -def _console(): - return Console(highlight=False) +from pload import ui def print_environment_table(environments): - table = Table(box=box.ROUNDED, header_style="bold cyan", border_style="blue") - table.add_column("ID", style="bold green", no_wrap=True) - table.add_column("NAME", style="bold", no_wrap=True) - table.add_column("PYTHON", style="yellow", no_wrap=True) - table.add_column("DESCRIPTION") - table.add_column("PATH", style="dim", overflow="fold") + table = ui.table( + ("ID", "bold cyan", {"width": 4, "no_wrap": True}), + ("NAME", "bold", {"width": 16, "overflow": "ellipsis", "no_wrap": True}), + ("PYTHON", "yellow", {"width": 8, "no_wrap": True}), + ("DESCRIPTION", "", {"width": 18, "overflow": "ellipsis", "no_wrap": True}), + ("PATH", "dim", {"min_width": 12, "ratio": 1, + "overflow": "ellipsis", "no_wrap": True}), + expand=True, + ) for item in environments: table.add_row( item.get("id", "-"), @@ -22,16 +19,19 @@ def print_environment_table(environments): item.get("description") or "—", item.get("path", "-"), ) - _console().print(table) + ui.console().print(table) def print_python_table(runtimes): - table = Table(box=box.ROUNDED, header_style="bold cyan", border_style="blue") - table.add_column("ID", style="bold green", no_wrap=True) - table.add_column("ALIAS", style="bold cyan", no_wrap=True) - table.add_column("VERSION", style="bold green", no_wrap=True) - table.add_column("TYPE", style="yellow", no_wrap=True) - table.add_column("PATH", style="dim", overflow="fold") + table = ui.table( + ("ID", "bold cyan", {"width": 5, "no_wrap": True}), + ("ALIAS", "cyan", {"width": 20, "overflow": "ellipsis", "no_wrap": True}), + ("VERSION", "green", {"width": 10, "no_wrap": True}), + ("TYPE", "yellow", {"width": 10, "no_wrap": True}), + ("PATH", "dim", {"min_width": 14, "ratio": 1, + "overflow": "ellipsis", "no_wrap": True}), + expand=True, + ) for runtime in runtimes: table.add_row( runtime.id or "-", @@ -40,4 +40,4 @@ def print_python_table(runtimes): runtime.source, str(runtime.path), ) - _console().print(table) + ui.console().print(table) diff --git a/src/pload/help_content.py b/src/pload/help_content.py index aa7f7a5..54f80ab 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -1,7 +1,6 @@ -from rich import box -from rich.console import Console from rich.markdown import Markdown -from rich.table import Table + +from pload import ui GUIDES = { (): r""" @@ -447,7 +446,7 @@ def render_detailed_help(parser, command_path): - console = Console(highlight=False) + console = ui.console() content = GUIDES.get(tuple(command_path), GUIDES[()]) console.print(Markdown(content.strip())) @@ -458,14 +457,11 @@ def render_detailed_help(parser, command_path): label = ", ".join(action.option_strings) if action.option_strings else action.dest rows.append((label, action.help or "")) if rows: - table = Table( - title="Related options", - box=box.ROUNDED, - header_style="bold cyan", - border_style="blue", + console.print("\n[bold cyan]Options[/]") + table = ui.table( + ("OPTION", "cyan", {"no_wrap": True}), + ("MEANING", "", {}), ) - table.add_column("OPTION", style="bold green", no_wrap=True) - table.add_column("MEANING") for label, help_text in rows: table.add_row(label, help_text) console.print(table) diff --git a/src/pload/installer.py b/src/pload/installer.py index 0cfceba..7b58bb2 100644 --- a/src/pload/installer.py +++ b/src/pload/installer.py @@ -6,16 +6,15 @@ from argparse import Namespace from pathlib import Path +from questionary import Choice from rich import box -from rich.console import Console from rich.markdown import Markdown from rich.panel import Panel from rich.table import Table from rich.text import Text -from pload import __version__ +from pload import __version__, ui from pload.errors import PloadError -from pload.managers.color import Colors from pload.settings import ( USTC_PYTHON_MIRROR, default_bin_dir, @@ -44,7 +43,7 @@ def build_parser(): - parser = argparse.ArgumentParser( + parser = ui.ArgumentParser( prog="pload-install", description=( "Install pload into its own private runtime, create a stable executable in a " @@ -104,7 +103,7 @@ def build_parser(): def render_help(detailed=False): parser = build_parser() if detailed: - console = Console(highlight=False) + console = ui.console() console.print(Markdown(r""" # Install pload without tying it to a project environment @@ -120,19 +119,17 @@ def render_help(detailed=False): managed virtual environment directory [/mnt/tools/pload/venvs]: /mnt/venvs managed Python directory [/mnt/tools/pload/pythons]: /mnt/python -Choose the Python runtime download source - 1) official (default) - 2) ustc - 3) custom -Enter a number [1]: 2 - -Choose the Python package index used to install pload and uv - 1) official (default) - 2) tsinghua - 3) ustc - 4) aliyun - 5) custom -Enter a number [1]: 3 +? Choose the Python runtime download source (↑/↓ move • enter select) + official Astral's official releases +❯ ustc USTC mirror in China + custom Your own mirror + +? Choose the Python package index (↑/↓ move • enter select) + official Official PyPI + tsinghua Tsinghua University mirror +❯ ustc USTC mirror + aliyun Alibaba Cloud mirror + custom Private compatible index ``` ## Resulting layout @@ -178,21 +175,18 @@ def render_help(detailed=False): `--yes` reuses explicit values and saved defaults. It does not edit a shell profile unless `--shell` is supplied or already saved in the configuration. """.strip())) - table = Table( - title="Related options", - box=box.ROUNDED, - header_style="bold cyan", - border_style="blue", + console.print("\n[bold cyan]Options[/]") + table = ui.table( + ("OPTION", "cyan", {"no_wrap": True}), + ("MEANING", "", {}), ) - table.add_column("OPTION", style="bold green", no_wrap=True) - table.add_column("MEANING") for action in parser._actions: if action.dest == "help" or not action.option_strings: continue table.add_row(", ".join(action.option_strings), action.help or "") console.print(table) return - console = Console(highlight=False) + console = ui.console() usage = " ".join(parser.format_usage().split()) usage_text = Text() for index, token in enumerate(usage.split()): @@ -217,7 +211,7 @@ def render_help(detailed=False): table = Table(box=box.SIMPLE, header_style="bold cyan", show_edge=False) table.add_column("COMMAND", style="bold green", no_wrap=True) table.add_column("PURPOSE") - table.add_row("pload-install", "Start the colored, numbered guided setup") + table.add_row("pload-install", "Start the cursor-key guided setup") table.add_row("pload-install --yes", "Reuse defaults without interactive questions") table.add_row( "pload-install --yes --home PATH", @@ -241,49 +235,132 @@ def render_help(detailed=False): def ask(prompt, default, non_interactive=False): if non_interactive: return str(default) - answer = input(f"{Colors.cyan(prompt)} [{Colors.green(default)}]: ").strip() - return answer or str(default) + return ui.text(prompt, default=str(default)) def ask_choice(prompt, choices, default, non_interactive=False): if non_interactive: return default items = [item if isinstance(item, tuple) else (item, "") for item in choices] - keys = [item[0] for item in items] - default_index = keys.index(default) if default in keys else 0 - while True: - print(f"\n{Colors.bold(prompt)}") - for index, (key, description) in enumerate(items, 1): - marker = Colors.green(" (default)") if index - 1 == default_index else "" - print(f" {Colors.cyan(index)}) {Colors.bold(key)}{marker}") - if description: - print(f" {description}") - answer = input( - f"{Colors.cyan('Enter a number')} [{Colors.green(default_index + 1)}]: " - ).strip().lower() - if not answer: - return keys[default_index] - if answer.isdigit() and 1 <= int(answer) <= len(items): - return keys[int(answer) - 1] - if answer in keys: - return answer - print(Colors.yellow(f"Please enter 1-{len(items)}.")) + choices = [ + Choice(f"{key:<10} {description}", value=key) + for key, description in items + ] + return ui.select(prompt, choices, default=default) def print_welcome(): - print(Colors.bold("\npload guided setup")) - print( - "This installer keeps pload itself, downloaded Python runtimes, and virtual " - "environments independent. Press Enter to accept any recommended default." + output = ui.console() + ui.heading( + "Set up pload", + "Choose where pload, Python runtimes, and environments live. " + "Press Enter to accept the recommended value.", + output=output, ) - print(Colors.cyan("\nDirectory layout")) - print(" pload home configuration, private runtime, state, and caches") - print(" executable bin stable `pload` command; add this directory to PATH") - print(" environments every managed project environment") - print(" Python runtimes Python versions downloaded through uv") + output.print("[dim]You can revisit these choices later with `pload cfg -t`.[/]") + + +def review_settings(args): + """Edit setup values from one revisitable settings screen.""" + home = Path(args.home or Path.home() / ".pload").expanduser().resolve() + existing = load_settings(home) + python = existing.get("python", {}) + values = { + "home": str(home), + "bin_dir": str(Path(args.bin_dir or existing.get("bin_dir") or default_bin_dir())), + "venvs_dir": str(Path(args.venvs_dir or existing.get("venvs_dir") or home / "venvs")), + "python_dir": str(Path(args.python_dir or python.get("install_dir") or home / "pythons")), + "source": args.source or python.get("source", "official"), + "mirror_url": args.mirror_url or python.get("mirror"), + "pip_source": args.pip_source or existing.get("pip_source", "official"), + "pip_index": args.pip_index or existing.get("pip_index"), + "shell": args.shell or existing.get("shell") or detect_shell(), + } + labels = { + "home": "pload data", + "bin_dir": "executable bin", + "venvs_dir": "environments", + "python_dir": "Python runtimes", + "source": "Python source", + "pip_source": "package index", + "shell": "shell integration", + } + path_fields = {"home", "bin_dir", "venvs_dir", "python_dir"} + while True: + choices = [ + Choice(f"{labels[key]:<19} {values[key]}", value=key) + for key in labels + ] + choices.extend([ + Choice("✓ Save configuration", value="__save__"), + Choice("× Cancel", value="__cancel__"), + ]) + field = ui.select( + "Review setup", + choices, + default="__save__", + instruction="↑/↓ move • enter edit", + ) + if field == "__save__": + break + if field == "__cancel__": + raise KeyboardInterrupt + if field in path_fields: + values[field] = ui.text(labels[field], default=values[field]) + elif field == "source": + values[field] = ui.select( + "Python runtime source", + [Choice(f"{key:<10} {description}", value=key) + for key, description in PYTHON_SOURCE_CHOICES], + default=values[field], + ) + if values[field] == "custom": + values["mirror_url"] = ui.text( + "Custom mirror base URL", + default=values["mirror_url"] or "file:///path/to/mirror", + ) + elif field == "pip_source": + values[field] = ui.select( + "Python package index", + [Choice(f"{key:<10} {description}", value=key) + for key, description in PIP_SOURCE_CHOICES], + default=values[field], + ) + if values[field] == "custom": + values["pip_index"] = ui.text( + "Custom package index URL", + default=values["pip_index"] or "https://example.com/simple", + ) + elif field == "shell": + detected = detect_shell() + shells = list(dict.fromkeys([detected, "bash", "zsh", "fish", "powershell"])) + values[field] = ui.select( + "Shell integration", + [Choice( + f"{shell:<12} " + f"{'detected shell' if shell == detected else 'update profile'}", + value=shell, + ) for shell in shells] + + [Choice("none print manual instructions", value="none")], + default=values[field], + ) + + args.home = values["home"] + args.bin_dir = values["bin_dir"] + args.venvs_dir = values["venvs_dir"] + args.python_dir = values["python_dir"] + args.source = values["source"] + args.mirror_url = values["mirror_url"] + args.pip_source = values["pip_source"] + args.pip_index = values["pip_index"] + args.shell = values["shell"] + args.yes = True + return args def collect_settings(args): + if not args.yes and ui.is_interactive(): + args = review_settings(args) home = Path(ask("pload home", args.home or Path.home() / ".pload", args.yes)).expanduser().resolve() existing = load_settings(home) bin_dir = Path( @@ -545,30 +622,23 @@ def configure_shell(settings): def print_welcome_screen(version, launcher, settings, profile=None): """Show a friendly post-install landing screen with copyable next steps.""" - print() - print(Colors.cyan(" ____ _ _ ")) - print(Colors.cyan(" | _ \\| | ___ __ _ __| |")) - print(Colors.cyan(" | |_) | |/ _ \\ / _` |/ _` |")) - print(Colors.cyan(" | __/| | (_) | (_| | (_| |")) - print(Colors.cyan(" |_| |_|\\___/ \\__,_|\\__,_|")) - print(Colors.bold(f"\n pload {version} is ready")) - print(" Your private runtime and launcher are installed.") - print() - print(Colors.bold(" Next steps")) - print(f" {Colors.green('1')} Check configuration: pload cfg") - print(f" {Colors.green('2')} Find Python versions: pload python list") - print(f" {Colors.green('3')} Create an environment: pload new -n data -v 3.12") - print(f" {Colors.green('4')} Read the walkthrough: pload -h -d") - print() - print(Colors.bold(" Installed paths")) - print(f" launcher {launcher}") - print(f" data {settings['home']}") - print(f" environments {settings['venvs_dir']}") + output = ui.console() + ui.logo(output=output) + output.print() + ui.success(f"pload {version} is ready", detail="Private runtime and launcher installed") + ui.heading("Next steps", output=output) + output.print(" [cyan]1[/] Check configuration [bold]pload cfg[/]") + output.print(" [cyan]2[/] Find Python versions [bold]pload python list[/]") + output.print(" [cyan]3[/] Create an environment [bold]pload new -n data -v 3.12[/]") + output.print(" [cyan]4[/] Read the walkthrough [bold]pload -h -d[/]") + ui.heading("Installed paths", output=output) + output.print(f" launcher [dim]{launcher}[/]") + output.print(f" data [dim]{settings['home']}[/]") + output.print(f" environments [dim]{settings['venvs_dir']}[/]") if profile: - print(f" shell updated {profile}") + output.print(f" shell [dim]updated {profile}[/]") else: - print(f" shell add {settings['bin_dir']} to PATH, then start a new shell") - print() + output.print(f" shell [dim]add {settings['bin_dir']} to PATH[/]") def run(argv=None): @@ -582,18 +652,18 @@ def run(argv=None): settings = collect_settings(args) path = save_settings(settings["home"], settings) if args.no_runtime_install: - print(f"[*] Wrote configuration: {path}") + ui.success("Wrote configuration", detail=str(path)) return 0 python = install_private_runtime(settings) launcher = write_launcher(settings, python) profile = configure_shell(settings) installed_version = installed_pload_version(python) - print(f"[*] Installed pload {installed_version}: {launcher}") - print(f"[*] Configuration: {path}") + ui.success(f"Installed pload {installed_version}", detail=str(launcher)) + ui.success("Saved configuration", detail=str(path)) if profile: - print(f"[*] Updated shell profile: {profile}") + ui.success("Updated shell profile", detail=str(profile)) else: - print(f"[!] Add {settings['bin_dir']} to PATH, then run: pload shell-init ") + ui.warning(f"Add {settings['bin_dir']} to PATH, then run: pload shell-init ") print_welcome_screen(installed_version, launcher, settings, profile) return 0 @@ -601,8 +671,11 @@ def run(argv=None): def main(argv=None): try: return run(argv) + except (KeyboardInterrupt, EOFError): + ui.warning("Cancelled") + return 130 except (PloadError, OSError) as exc: - print(f"pload-install: error: {exc}", file=sys.stderr) + ui.error(str(exc)) return 1 diff --git a/src/pload/managers/color.py b/src/pload/managers/color.py deleted file mode 100644 index cc653e6..0000000 --- a/src/pload/managers/color.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import sys - - -class Colors: - """Small color helper that stays quiet when output is redirected.""" - - enabled = sys.stdout.isatty() and os.environ.get("NO_COLOR") is None - - @classmethod - def _paint(cls, value, color): - text = str(value) - if not cls.enabled: - return text - return f"\033[1;{color}m{text}\033[0m" - - @classmethod - def green(cls, value): - return cls._paint(value, "32") - - @classmethod - def red(cls, value): - return cls._paint(value, "31") - - @classmethod - def yellow(cls, value): - return cls._paint(value, "33") - - @classmethod - def cyan(cls, value): - return cls._paint(value, "36") - - @classmethod - def blue(cls, value): - return cls._paint(value, "34") - - @classmethod - def bold(cls, value): - text = str(value) - if not cls.enabled: - return text - return f"\033[1m{text}\033[0m" diff --git a/src/pload/managers/dependency.py b/src/pload/managers/dependency.py index 7713767..c7c7361 100644 --- a/src/pload/managers/dependency.py +++ b/src/pload/managers/dependency.py @@ -1,7 +1,7 @@ import subprocess +from pload import ui from pload.errors import PloadError -from pload.managers.color import Colors class DependencyManager: @@ -19,7 +19,7 @@ def install_dependencies(self, venv_path, requirements, channel=None): if channel: command += ["--index-url", channel] - print(f"[*] Installing: {' '.join(Colors.green(item) for item in requirements)}") + ui.info(f"Installing {' '.join(requirements)}") process = subprocess.run(command, check=False) if process.returncode != 0: raise PloadError("dependency installation failed") diff --git a/src/pload/managers/venv.py b/src/pload/managers/venv.py index b139e84..a6d9576 100644 --- a/src/pload/managers/venv.py +++ b/src/pload/managers/venv.py @@ -5,11 +5,10 @@ from datetime import datetime, timezone from pathlib import Path -from rich.console import Console from rich.text import Text +from pload import ui from pload.errors import PloadError -from pload.managers.color import Colors class VenvManager: @@ -42,7 +41,7 @@ def create_venv( python_exe = self.config.get_python_path(version) target_path.parent.mkdir(parents=True, exist_ok=True) command = [str(python_exe), "-m", "venv", str(target_path)] - console = Console(highlight=False) + console = ui.console() if console.is_terminal: status = Text.assemble( ("Creating ", "bold cyan"), @@ -55,7 +54,7 @@ def create_venv( command, capture_output=True, text=True, check=False ) else: - print(f"[*] Creating {Colors.green(display_name)} at {Colors.green(target_path)}") + ui.info(f"Creating {display_name} at {target_path}") process = subprocess.run( command, capture_output=True, text=True, check=False ) @@ -71,7 +70,7 @@ def create_venv( (display_name, "bold"), )) else: - print(f"[*] Created {Colors.green(display_name)}") + ui.success(f"Created {display_name}") entry = self.register_environment( target_path, name=display_name, @@ -84,7 +83,7 @@ def create_venv( (entry["id"], "bold cyan"), )) else: - print(f"[*] Assigned {Colors.cyan(entry['id'])}") + ui.success(f"Assigned {entry['id']}") return target_path @property @@ -272,7 +271,7 @@ def remove_venv(self, venv_name, project_dir=None): if Path(entry.get("path", "")).expanduser().resolve() != target_path ] self._write_registry(data) - print(f"[*] Removed {Colors.green(target_path)}") + ui.success(f"Removed {target_path}") def activation_script(self, venv_name, shell=None, project_dir=None): path = self.resolve_existing(venv_name, project_dir=project_dir) diff --git a/src/pload/ui.py b/src/pload/ui.py new file mode 100644 index 0000000..5909d86 --- /dev/null +++ b/src/pload/ui.py @@ -0,0 +1,221 @@ +"""One terminal language for every pload command. + +Keep presentation decisions here so commands cannot invent their own colours, +symbols, tables, or prompt behaviour. Rich owns output; Questionary owns +interactive input. Both automatically degrade when output is redirected. +""" + +from __future__ import annotations + +import argparse +import difflib +import os +import re +import sys +from contextlib import contextmanager + +import questionary +from prompt_toolkit.styles import Style as PromptStyle +from rich import box +from rich.console import Console +from rich.table import Table +from rich.text import Text + +ACCENT = "cyan" +SUCCESS = "green" +WARNING = "yellow" +DANGER = "red" +MUTED = "dim" + +PROMPT_STYLE = PromptStyle.from_dict( + { + "qmark": "fg:#22d3ee bold", + "question": "bold", + "answer": "fg:#34d399 bold", + "pointer": "fg:#22d3ee bold", + "highlighted": "fg:#22d3ee bold", + "selected": "fg:#34d399", + "instruction": "fg:#7c8499", + "text": "", + "disabled": "fg:#7c8499 italic", + } +) +PLAIN_PROMPT_STYLE = PromptStyle.from_dict( + {name: "" for name in ( + "qmark", "question", "answer", "pointer", "highlighted", + "selected", "instruction", "text", "disabled", + )} +) + + +def _prompt_style(): + return PLAIN_PROMPT_STYLE if os.environ.get("NO_COLOR") is not None else PROMPT_STYLE + + +def _color_system(): + return None if os.environ.get("NO_COLOR") is not None else "auto" + + +class ArgumentParser(argparse.ArgumentParser): + """Argparse with the same compact error language as the rest of pload.""" + + def error(self, message): + suggestion = None + if "invalid choice:" in message and " (choose from " in message: + summary, choices = message.split(" (choose from ", 1) + candidates = [ + item.strip().strip("'\"") + for item in choices.rstrip(")").split(",") + ] + invalid = re.findall(r"invalid choice: '([^']+)'", summary) + if invalid: + matches = difflib.get_close_matches( + invalid[0], candidates, n=1, cutoff=0.5, + ) + suggestion = matches[0] if matches else None + message = summary + error(message) + if suggestion: + console(stderr=True).print(f"[dim]Did you mean '{suggestion}'?[/]") + console(stderr=True).print(f"[dim]Try '{self.prog} -h' for help.[/]") + raise SystemExit(2) + + +def console(*, stderr=False): + return Console(stderr=stderr, highlight=False, color_system=_color_system()) + + +def is_interactive(): + return sys.stdin.isatty() and sys.stdout.isatty() + + +def table(*columns, expand=False): + """Build the compact table used throughout pload.""" + result = Table( + box=box.SIMPLE_HEAD, + header_style="bold cyan", + show_edge=False, + pad_edge=False, + expand=expand, + ) + for heading, style, kwargs in columns: + result.add_column(heading, style=style, **kwargs) + return result + + +def heading(title, subtitle=None, *, output=None): + output = output or console() + output.print() + output.print(Text(title, style="bold cyan")) + if subtitle: + output.print(Text(subtitle, style="dim")) + + +def logo(*, output=None): + """Render the single pload wordmark used by landing and setup screens.""" + output = output or console() + output.print(Text( + """ ____ _ _ + | _ \\| | ___ __ _ __| | + | |_) | |/ _ \\ / _` |/ _` | + | __/| | (_) | (_| | (_| | + |_| |_|\\___/ \\__,_|\\__,_|""", + style="bold cyan", + )) + + +def success(message, *, detail=None, output=None): + output = output or console() + output.print(Text.assemble(("✓ ", "bold green"), (message, "white"))) + if detail: + output.print(f" [dim]{detail}[/]") + + +def info(message, *, output=None): + (output or console()).print(Text.assemble(("• ", "cyan"), (message, "white"))) + + +def warning(message, *, output=None): + (output or console(stderr=True)).print( + Text.assemble(("! ", "bold yellow"), (message, "yellow")) + ) + + +def error(message, *, output=None): + (output or console(stderr=True)).print( + Text.assemble(("error: ", "bold red"), (message, "red")) + ) + + +@contextmanager +def status(message): + output = console(stderr=True) + if output.is_terminal and os.environ.get("PLOAD_NO_PROGRESS") is None: + with output.status(f"[cyan]{message}[/]", spinner="dots", spinner_style="cyan") as live: + yield lambda value: live.update(f"[cyan]{value}[/]") + else: + output.print(f"[dim]{message}[/]") + yield lambda value: None + + +def select(message, choices, *, default=None, instruction="↑/↓ move • enter select"): + """Choose with normal cursor keys, with a numbered fallback for tests/pipes.""" + normalized = [ + item if isinstance(item, questionary.Choice) else questionary.Choice(str(item), value=item) + for item in choices + ] + if is_interactive(): + return questionary.select( + message, + choices=normalized, + default=default, + instruction=instruction, + pointer="❯", + qmark="?", + style=_prompt_style(), + use_shortcuts=False, + use_arrow_keys=True, + ).unsafe_ask() + + default_index = 0 + values = [item.value for item in normalized] + if default in values: + default_index = values.index(default) + print(f"\n{message}") + for index, item in enumerate(normalized, 1): + suffix = " (default)" if index - 1 == default_index else "" + print(f" {index}) {item.title}{suffix}") + while True: + answer = input(f"Choose [1-{len(normalized)}] [{default_index + 1}]: ").strip() + if not answer: + return normalized[default_index].value + if answer.isdigit() and 1 <= int(answer) <= len(normalized): + return normalized[int(answer) - 1].value + + +def text(message, *, default=""): + if is_interactive(): + return questionary.text( + message, + default=str(default), + qmark="?", + style=_prompt_style(), + ).unsafe_ask().strip() + suffix = f" [{default}]" if default else "" + value = input(f"{message}{suffix}: ").strip() + return value or str(default) + + +def confirm(message, *, default=False): + if is_interactive(): + return questionary.confirm( + message, + default=default, + qmark="?", + style=_prompt_style(), + ).unsafe_ask() + suffix = "Y/n" if default else "y/N" + answer = input(f"{message} [{suffix}]: ").strip().lower() + if not answer: + return default + return answer in {"y", "yes"} diff --git a/tests/test_cli.py b/tests/test_cli.py index 9171c87..312c3c1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -124,6 +124,19 @@ def test_simple_commands_have_no_alias_but_long_options_have_short_forms(): raise AssertionError("new must not have a command alias") +def test_parser_errors_are_compact_and_actionable(capsys): + with pytest.raises(SystemExit) as error: + build_parser().parse_args(["lisst"]) + + output = capsys.readouterr().err + assert error.value.code == 2 + assert "error:" in output + assert "invalid choice" in output + assert "Did you mean 'list'?" in output + assert "Try 'pload -h' for help." in output + assert not output.startswith("usage:") + + def test_description_short_option_does_not_conflict_with_detailed_help(): args = build_parser().parse_args(["new", "-m", "Data tools"]) @@ -152,7 +165,7 @@ def test_brief_and_detailed_help_are_distinct(capsys): assert "$ pload new -n data" in detailed assert "What changes on disk" in detailed assert "PLOAD_HOME/runtime" in detailed - assert "Related options" in detailed + assert "Options" in detailed def test_command_specific_detailed_help_uses_d_flag(capsys): diff --git a/tests/test_installer.py b/tests/test_installer.py index ed13827..6ee9d44 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -93,6 +93,21 @@ def test_guided_choice_accepts_numeric_selection(monkeypatch, capsys): assert "Alibaba Cloud" in output +def test_settings_screen_can_revisit_and_edit_any_value(tmp_path, monkeypatch): + answers = iter(["bin_dir", "__save__"]) + monkeypatch.setattr(installer.ui, "select", lambda *args, **kwargs: next(answers)) + monkeypatch.setattr( + installer.ui, + "text", + lambda message, default="": str(tmp_path / "custom-bin"), + ) + + args = installer.review_settings(installer_args(tmp_path, yes=False)) + + assert args.bin_dir == str(tmp_path / "custom-bin") + assert args.yes is True + + def test_config_only_install_writes_reusable_configuration(tmp_path): result = run([ "--yes", @@ -181,7 +196,7 @@ def test_shell_configuration_updates_existing_managed_block(monkeypatch, tmp_pat def test_installer_has_brief_and_detailed_help(capsys): assert run(["-h"]) == 0 brief = capsys.readouterr().out - assert "colored, numbered guided setup" in brief + assert "cursor-key guided setup" in brief assert "pload-install -h -d" in brief assert "--downloads-json-url" in brief assert "--home, -H" in brief @@ -190,7 +205,7 @@ def test_installer_has_brief_and_detailed_help(capsys): assert run(["-h", "-d"]) == 0 detailed = capsys.readouterr().out assert "What an interactive run looks like" in detailed - assert "Enter a number" in detailed + assert "↑/↓ move" in detailed assert "Resulting layout" in detailed assert "What the installer changes" in detailed assert "--downloads-json-url" in detailed diff --git a/tests/test_ui.py b/tests/test_ui.py new file mode 100644 index 0000000..695e4f6 --- /dev/null +++ b/tests/test_ui.py @@ -0,0 +1,77 @@ +from questionary import Choice + +from pload import ui +from pload.cli import choose_declarative_routes + + +def test_select_has_numbered_fallback_when_terminal_ui_is_unavailable(monkeypatch, capsys): + monkeypatch.setattr(ui, "is_interactive", lambda: False) + monkeypatch.setattr("builtins.input", lambda prompt: "2") + + selected = ui.select( + "Choose a source", + [Choice("Local cache", value="cache"), Choice("PyPI", value="index")], + default="cache", + ) + + assert selected == "index" + assert "1) Local cache (default)" in capsys.readouterr().out + + +def test_route_chooser_only_visits_direct_packages_and_supports_backtracking(monkeypatch): + plan = { + "python": {"status": "available"}, + "packages": [ + { + "name": "torch", + "version": "2.8.0", + "direct": True, + "selected": { + "method": "index", + "location": "https://pypi.org/simple", + "status": "available", + "estimated_seconds": 10.0, + }, + "alternatives": [{ + "method": "cache", + "location": "/cache/torch.whl", + "status": "available", + "estimated_seconds": 0.1, + }], + }, + { + "name": "typing-extensions", + "version": "4.15.0", + "direct": False, + "selected": { + "method": "cache", + "location": "/cache/typing_extensions.whl", + "status": "available", + "estimated_seconds": 0.1, + }, + "alternatives": [], + }, + ], + } + answers = iter([("route", 1), ("save", None)]) + prompts = [] + + def choose(message, choices, **kwargs): + prompts.append((message, choices)) + return next(answers) + + monkeypatch.setattr("pload.cli.ui.select", choose) + + assert choose_declarative_routes(plan) == "save" + assert plan["packages"][0]["selected"]["method"] == "cache" + assert plan["packages"][1]["selected"]["method"] == "cache" + assert len(prompts) == 2 + assert all("typing-extensions" not in message for message, _ in prompts) + + +def test_no_color_disables_rich_ansi_sequences(monkeypatch): + monkeypatch.setenv("NO_COLOR", "1") + output = ui.console() + + assert output.color_system is None + assert ui._prompt_style() is ui.PLAIN_PROMPT_STYLE From 1d3f08e9e50ebbe1d0857a39d532508dc43f1853 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Sun, 27 Sep 2026 17:52:27 +0800 Subject: [PATCH 25/41] Fall back to ASCII on legacy terminals --- src/pload/cli.py | 20 +++++++++++++------- src/pload/installer.py | 6 +++--- src/pload/ui.py | 22 +++++++++++++++++++--- tests/test_ui.py | 6 ++++++ 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/pload/cli.py b/src/pload/cli.py index 4f72c84..6343ef5 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -553,15 +553,21 @@ def choose_declarative_routes(plan): value=("route", index - 1), )) if cursor: - choices.append(Choice("← Previous package", value=("previous", None))) + choices.append(Choice( + f"{ui.glyph('←', '<-')} Previous package", + value=("previous", None), + )) if cursor < len(decision_indices) - 1: - choices.append(Choice("→ Keep current choice and continue", value=("next", None))) + choices.append(Choice( + f"{ui.glyph('→', '->')} Keep current choice and continue", + value=("next", None), + )) choices.extend([ - Choice("↶ Undo last change", value=("undo", None)), - Choice("↺ Reset all to recommended", value=("reset", None)), - Choice("✓ Save plan", value=("save", None)), - Choice("▶ Save and apply", value=("apply", None)), - Choice("× Cancel", value=("quit", None)), + Choice(f"{ui.glyph('↶', '<-')} Undo last change", value=("undo", None)), + Choice(f"{ui.glyph('↺', 'R')} Reset all to recommended", value=("reset", None)), + Choice(f"{ui.glyph('✓', '[ok]')} Save plan", value=("save", None)), + Choice(f"{ui.glyph('▶', '>')} Save and apply", value=("apply", None)), + Choice(f"{ui.glyph('×', 'x')} Cancel", value=("quit", None)), ]) command, value = ui.select( "Select an acquisition source", diff --git a/src/pload/installer.py b/src/pload/installer.py index 7b58bb2..57998f2 100644 --- a/src/pload/installer.py +++ b/src/pload/installer.py @@ -292,14 +292,14 @@ def review_settings(args): for key in labels ] choices.extend([ - Choice("✓ Save configuration", value="__save__"), - Choice("× Cancel", value="__cancel__"), + Choice(f"{ui.glyph('✓', '[ok]')} Save configuration", value="__save__"), + Choice(f"{ui.glyph('×', 'x')} Cancel", value="__cancel__"), ]) field = ui.select( "Review setup", choices, default="__save__", - instruction="↑/↓ move • enter edit", + instruction=ui.navigation_hint("edit"), ) if field == "__save__": break diff --git a/src/pload/ui.py b/src/pload/ui.py index 5909d86..64d7289 100644 --- a/src/pload/ui.py +++ b/src/pload/ui.py @@ -56,6 +56,20 @@ def _color_system(): return None if os.environ.get("NO_COLOR") is not None else "auto" +def glyph(value, fallback, *, encoding=None): + """Use Unicode only when the active terminal can actually encode it.""" + encoding = encoding or getattr(sys.stdout, "encoding", None) or "utf-8" + try: + value.encode(encoding) + except (LookupError, UnicodeEncodeError): + return fallback + return value + + +def navigation_hint(action="select"): + return f"{glyph('↑/↓', 'up/down')} move • enter {action}" + + class ArgumentParser(argparse.ArgumentParser): """Argparse with the same compact error language as the rest of pload.""" @@ -126,7 +140,8 @@ def logo(*, output=None): def success(message, *, detail=None, output=None): output = output or console() - output.print(Text.assemble(("✓ ", "bold green"), (message, "white"))) + mark = glyph("✓ ", "[ok] ", encoding=output.encoding) + output.print(Text.assemble((mark, "bold green"), (message, "white"))) if detail: output.print(f" [dim]{detail}[/]") @@ -158,19 +173,20 @@ def status(message): yield lambda value: None -def select(message, choices, *, default=None, instruction="↑/↓ move • enter select"): +def select(message, choices, *, default=None, instruction=None): """Choose with normal cursor keys, with a numbered fallback for tests/pipes.""" normalized = [ item if isinstance(item, questionary.Choice) else questionary.Choice(str(item), value=item) for item in choices ] if is_interactive(): + instruction = instruction or navigation_hint() return questionary.select( message, choices=normalized, default=default, instruction=instruction, - pointer="❯", + pointer=glyph("❯", ">"), qmark="?", style=_prompt_style(), use_shortcuts=False, diff --git a/tests/test_ui.py b/tests/test_ui.py index 695e4f6..1839553 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -75,3 +75,9 @@ def test_no_color_disables_rich_ansi_sequences(monkeypatch): assert output.color_system is None assert ui._prompt_style() is ui.PLAIN_PROMPT_STYLE + + +def test_symbols_fall_back_for_legacy_windows_encodings(): + assert ui.glyph("✓", "[ok]", encoding="cp1252") == "[ok]" + assert ui.glyph("❯", ">", encoding="cp1252") == ">" + assert ui.glyph("✓", "[ok]", encoding="utf-8") == "✓" From 46ced49401277520daacd74e8dfc2494d6687028 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Sun, 27 Sep 2026 18:05:29 +0800 Subject: [PATCH 26/41] Move planner actions to keyboard shortcuts --- README.md | 3 ++- docs/terminal-experience.md | 3 +++ src/pload/cli.py | 35 ++++++++++++++++++----------------- src/pload/ui.py | 17 ++++++++++++++--- tests/test_ui.py | 15 +++++++++++++++ 5 files changed, 52 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index b128ecb..7325a65 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,8 @@ bodies—and inventories pload/pip/uv caches, compatible existing environments, local/SSH stores and indexes. Its interactive chooser shows only dependencies written by the user; transitive dependencies are routed automatically. It supports previous/next, undo, reset, save and save-and-apply through a -cursor-key menu. `apply` executes exactly the saved plan: +compact shortcut row (`p/n/u/r/s/a/q`); the selection list itself contains only +real package sources. `apply` executes exactly the saved plan: it never re-resolves, changes route, falls back, or uploads packages. Back up a special or slow wheel only when you explicitly request it: diff --git a/docs/terminal-experience.md b/docs/terminal-experience.md index 81a3b1f..42a4c75 100644 --- a/docs/terminal-experience.md +++ b/docs/terminal-experience.md @@ -10,6 +10,9 @@ behaviour, not decoration. screen interface unexpectedly. - A command that needs a choice uses `↑`/`↓` and Enter. The active row is prefixed with `❯`; the recommended value is selected initially. +- Package planning keeps navigation commands out of the source list. Use the + visible first-letter shortcuts: `p` previous, `n` next, `u` undo, `r` reset, + `s` save, `a` apply, and `q` quit. - `pload new` is guided when no creation options are supplied. Every guided operation also has flags for scripts and automation. - Destructive actions use a conventional `y/N` confirmation. `--yes` is the diff --git a/src/pload/cli.py b/src/pload/cli.py index 6343ef5..b77ad6d 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -552,27 +552,28 @@ def choose_declarative_routes(plan): f"{route['method']:<18} ~{seconds:g}s {route['location']}{recommended}", value=("route", index - 1), )) - if cursor: - choices.append(Choice( - f"{ui.glyph('←', '<-')} Previous package", - value=("previous", None), - )) - if cursor < len(decision_indices) - 1: - choices.append(Choice( - f"{ui.glyph('→', '->')} Keep current choice and continue", - value=("next", None), - )) - choices.extend([ - Choice(f"{ui.glyph('↶', '<-')} Undo last change", value=("undo", None)), - Choice(f"{ui.glyph('↺', 'R')} Reset all to recommended", value=("reset", None)), - Choice(f"{ui.glyph('✓', '[ok]')} Save plan", value=("save", None)), - Choice(f"{ui.glyph('▶', '>')} Save and apply", value=("apply", None)), - Choice(f"{ui.glyph('×', 'x')} Cancel", value=("quit", None)), - ]) + if not choices: + ui.warning(f"No valid source is available for {package['name']}") + return "quit" + actions = { + "p": ("previous", None), + "n": ("next", None), + "u": ("undo", None), + "r": ("reset", None), + "s": ("save", None), + "a": ("apply", None), + "q": ("quit", None), + } + console.print( + "[dim]p previous · n next · u undo · r reset · " + "s save · a apply · q quit[/]" + ) command, value = ui.select( "Select an acquisition source", choices, default=("route", 0) if routes else ("quit", None), + instruction=ui.navigation_hint("choose"), + hotkeys=actions, ) if command == "route": selection.select(package_index, value) diff --git a/src/pload/ui.py b/src/pload/ui.py index 64d7289..9ba5320 100644 --- a/src/pload/ui.py +++ b/src/pload/ui.py @@ -173,15 +173,16 @@ def status(message): yield lambda value: None -def select(message, choices, *, default=None, instruction=None): +def select(message, choices, *, default=None, instruction=None, hotkeys=None): """Choose with normal cursor keys, with a numbered fallback for tests/pipes.""" + hotkeys = {str(key).lower(): value for key, value in (hotkeys or {}).items()} normalized = [ item if isinstance(item, questionary.Choice) else questionary.Choice(str(item), value=item) for item in choices ] if is_interactive(): instruction = instruction or navigation_hint() - return questionary.select( + question = questionary.select( message, choices=normalized, default=default, @@ -191,7 +192,15 @@ def select(message, choices, *, default=None, instruction=None): style=_prompt_style(), use_shortcuts=False, use_arrow_keys=True, - ).unsafe_ask() + ) + for key, value in hotkeys.items(): + def register(binding, result): + @question.application.key_bindings.add(binding, eager=True) + def choose_action(event): + event.app.exit(result=result) + + register(key, value) + return question.unsafe_ask() default_index = 0 values = [item.value for item in normalized] @@ -205,6 +214,8 @@ def select(message, choices, *, default=None, instruction=None): answer = input(f"Choose [1-{len(normalized)}] [{default_index + 1}]: ").strip() if not answer: return normalized[default_index].value + if answer.lower() in hotkeys: + return hotkeys[answer.lower()] if answer.isdigit() and 1 <= int(answer) <= len(normalized): return normalized[int(answer) - 1].value diff --git a/tests/test_ui.py b/tests/test_ui.py index 1839553..0ae0ed8 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -18,6 +18,19 @@ def test_select_has_numbered_fallback_when_terminal_ui_is_unavailable(monkeypatc assert "1) Local cache (default)" in capsys.readouterr().out +def test_select_fallback_accepts_action_hotkeys(monkeypatch): + monkeypatch.setattr(ui, "is_interactive", lambda: False) + monkeypatch.setattr("builtins.input", lambda prompt: "p") + + selected = ui.select( + "Choose a source", + [Choice("Local cache", value="cache")], + hotkeys={"p": ("previous", None)}, + ) + + assert selected == ("previous", None) + + def test_route_chooser_only_visits_direct_packages_and_supports_backtracking(monkeypatch): plan = { "python": {"status": "available"}, @@ -67,6 +80,8 @@ def choose(message, choices, **kwargs): assert plan["packages"][1]["selected"]["method"] == "cache" assert len(prompts) == 2 assert all("typing-extensions" not in message for message, _ in prompts) + displayed = [choice.title for _, choices in prompts for choice in choices] + assert all("Previous" not in title and "Save" not in title for title in displayed) def test_no_color_disables_rich_ansi_sequences(monkeypatch): From 42bdbf405eb823d2b3ea7b74d3e694cc617d6631 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Sun, 27 Sep 2026 18:16:49 +0800 Subject: [PATCH 27/41] Make guided package installs resilient --- README.md | 13 +++- docs/terminal-experience.md | 4 + src/pload/cli.py | 38 +++++++++- src/pload/help_content.py | 13 +++- src/pload/managers/dependency.py | 124 ++++++++++++++++++++++++++++--- tests/test_cli.py | 3 +- tests/test_dependency.py | 85 +++++++++++++++++++++ 7 files changed, 260 insertions(+), 20 deletions(-) create mode 100644 tests/test_dependency.py diff --git a/README.md b/README.md index 7325a65..c3515cd 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,10 @@ pload v1 # activate an environment by ID Run `pload new` without options to start the guided creator. It lists detected Python runtimes, then asks for the interpreter, environment name, description, -and optional packages. Press Enter to accept the suggested value at each step. +and optional packages. When packages are present, choose **Automatic** to let +pload prefer a compatible local wheel cache and otherwise use the configured +index, or **Custom** to choose the source of each top-level package yourself. +Press Enter to accept the suggested value at each step. ## Configure later @@ -307,6 +310,7 @@ Create a named environment with a description: pload new # guided creation pload new -n data -v 3.12 -m "Data analysis" pload new -n web -v 3.12 -r fastapi uvicorn -m "Web API" +pload new -n lab -r numpy torch -s custom # choose each top-level package source pload new -v 3.12 --message "Temporary data tools" # auto-named safely pload list pload v1 @@ -337,6 +341,13 @@ The table includes ID, name, Python version, description, and full path. A spinner is shown in interactive terminals; redirected and CI output uses stable ordinary log lines. +Package requests are installed one at a time. A misspelled, unavailable, or +invalid request is reported and skipped, while later requests still run and +successful installations remain in the environment. The command returns a +failure status at the end when any requested package failed, with one summary +listing those packages. Source selection applies only to packages explicitly +entered by the user; pip continues to resolve their transitive dependencies. + ## Help and aliases Brief help is a complete reference. It starts with an emphasized `USAGE` panel, diff --git a/docs/terminal-experience.md b/docs/terminal-experience.md index 42a4c75..e185558 100644 --- a/docs/terminal-experience.md +++ b/docs/terminal-experience.md @@ -15,6 +15,10 @@ behaviour, not decoration. `s` save, `a` apply, and `q` quit. - `pload new` is guided when no creation options are supplied. Every guided operation also has flags for scripts and automation. +- When guided creation includes packages, it asks once whether pload should + choose the fastest source or the user wants custom per-package choices. The + custom list contains only real cache/index routes. Transitive dependencies + remain automatic, and one failed request does not prevent later requests. - Destructive actions use a conventional `y/N` confirmation. `--yes` is the explicit non-interactive alternative. - `Ctrl-C` cancels without a traceback. diff --git a/src/pload/cli.py b/src/pload/cli.py index b77ad6d..985be61 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -360,6 +360,10 @@ def build_parser(): def add_packages(parser): parser.add_argument("--channel", "-c", help="Python package index URL") parser.add_argument("--requirements", "-r", nargs="+", help="packages to install") + parser.add_argument( + "--package-strategy", "-s", choices=["auto", "custom"], + help="choose package sources automatically or interactively (default: auto)", + ) def _set_config_value(config, key, value): @@ -836,11 +840,29 @@ def _guided_new(config): name = ui.text("Environment name", default=suggested_name) description = ui.text("Description (optional)") packages = ui.text("Packages, separated by spaces (optional)") + requirements = packages.split() or None + package_strategy = None + if requirements: + package_strategy = ui.select( + "Package source strategy", + [ + Choice( + "Automatic prefer compatible cache, then the configured index", + value="auto", + ), + Choice( + "Custom choose a source for each package", + value="custom", + ), + ], + default="auto", + ) return { "python_version": version, "name": name, "description": description or None, - "requirements": packages.split() or None, + "requirements": requirements, + "package_strategy": package_strategy, } @@ -933,19 +955,25 @@ def run(argv=None): return 0 if command == "new": - if not any((args.python_version, args.name, args.path, args.description, args.requirements, args.channel)): + if not any(( + args.python_version, args.name, args.path, args.description, + args.requirements, args.channel, args.package_strategy, + )): guided = _guided_new(config) args.python_version = guided["python_version"] args.name = guided["name"] args.description = guided["description"] args.requirements = guided["requirements"] + args.package_strategy = guided["package_strategy"] path = venvs.create_venv( version=args.python_version, target=args.path, name=args.name, description=args.description, ) - dependencies.install_dependencies(path, args.requirements, args.channel) + dependencies.install_dependencies( + path, args.requirements, args.channel, args.package_strategy or "auto", + ) return 0 if command == "init": @@ -958,7 +986,9 @@ def run(argv=None): target=args.venv_dir, description=args.description, ) - dependencies.install_dependencies(path, args.requirements, args.channel) + dependencies.install_dependencies( + path, args.requirements, args.channel, args.package_strategy or "auto", + ) return 0 if command == "rm": diff --git a/src/pload/help_content.py b/src/pload/help_content.py index 54f80ab..5f9d90c 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -83,6 +83,12 @@ guided creator. Use `--message` (or `-m`) for the human description; it never affects the generated directory name. +When packages are requested, `--package-strategy auto` (the default) prefers +a compatible wheel already in pload's cache and otherwise uses the configured +package index. `--package-strategy custom` (or `-s custom`) opens one source +choice for each package written on the command line. Dependencies introduced +by those packages remain automatic, so the chooser stays short. + ## Choose an exact location or interpreter ```console @@ -100,8 +106,11 @@ - Creates one virtual-environment directory and one registry entry. - Does not modify the system Python or activate the environment automatically. - If Python cannot create the environment, the incomplete directory is removed. -- If later package installation fails, the valid environment remains so it can - be inspected or removed with `pload rm`. +- Requested packages are attempted one at a time. A bad request is reported, + later requests still run, and successful installations remain available. +- If any package failed, pload returns a failure status after all requests and + lists every failure. The valid environment remains so it can be inspected or + removed with `pload rm`. """, ("init",): r""" # Create an environment for the current project diff --git a/src/pload/managers/dependency.py b/src/pload/managers/dependency.py index c7c7361..cae6cbf 100644 --- a/src/pload/managers/dependency.py +++ b/src/pload/managers/dependency.py @@ -1,25 +1,125 @@ import subprocess +from pathlib import Path + +from packaging.requirements import InvalidRequirement, Requirement +from packaging.utils import InvalidWheelFilename, canonicalize_name, parse_wheel_filename +from questionary import Choice from pload import ui from pload.errors import PloadError +from pload.managers.platform import ConfigManager + + +def _compatible_tags(venv_path): + """Describe the target runtime, rather than the Python running pload.""" + from pload.declarative import _target_wheel_tags + from pload.snapshots import probe + + return _target_wheel_tags(probe(ConfigManager.venv_python(venv_path))) class DependencyManager: def __init__(self, config): self.config = config - def install_dependencies(self, venv_path, requirements, channel=None): + def install_dependencies(self, venv_path, requirements, channel=None, strategy="auto"): if not requirements: return + if strategy not in {"auto", "custom"}: + raise PloadError(f"unknown package strategy: {strategy}") + + wheel_cache = self.config.home / "cache" / "wheels" + supported_tags = _compatible_tags(venv_path) if wheel_cache.is_dir() else set() + index = channel or self.config.settings.get("pip_index") + failures = [] + installed = [] + + for value in requirements: + try: + requirement = Requirement(value) + except InvalidRequirement: + ui.warning(f"Skipping invalid package requirement: {value}") + failures.append((value, "invalid requirement")) + continue + + routes = self._routes(requirement, wheel_cache, supported_tags, index) + selected = routes[0] + if strategy == "custom" and len(routes) > 1: + selected = ui.select( + f"Source for {value}", + [Choice(route["label"], value=route) for route in routes], + default=routes[0], + ) + + ui.info(f"Installing {value} from {selected['summary']}") + command = self.config.get_pip_command(venv_path) + ["install"] + if selected["kind"] == "cache": + command.append(str(selected["path"])) + command += ["--find-links", str(wheel_cache)] + else: + command.append(value) + if index: + command += ["--index-url", index] + try: + process = subprocess.run(command, check=False) + except OSError as exc: + failures.append((value, str(exc))) + ui.warning(f"Could not install {value}; continuing with the next package") + continue + if process.returncode != 0: + failures.append((value, f"{selected['summary']} failed")) + ui.warning(f"Could not install {value}; continuing with the next package") + continue + installed.append(value) + + if installed: + ui.success( + f"Installed {len(installed)} requested package" + f"{'s' if len(installed) != 1 else ''}", + detail=", ".join(installed), + ) + if failures: + details = "; ".join(f"{name} ({reason})" for name, reason in failures) + raise PloadError( + "some packages were not installed; successful packages were kept: " + details + ) + + @staticmethod + def _routes(requirement, wheel_cache, supported_tags, index): + matches = [] + # Installing a wheel path would discard URL, marker, or extras semantics. + # Let pip interpret those requirement forms from the configured index. + if (wheel_cache.is_dir() and not requirement.url + and not requirement.marker and not requirement.extras): + for wheel in wheel_cache.glob("*.whl"): + try: + name, version, _, tags = parse_wheel_filename(wheel.name) + except InvalidWheelFilename: + continue + if canonicalize_name(name) != canonicalize_name(requirement.name): + continue + if requirement.specifier and not requirement.specifier.contains( + version, prereleases=True, + ): + continue + if not tags.intersection(supported_tags): + continue + matches.append((version, wheel)) + matches.sort(key=lambda item: (item[0], item[1].name), reverse=True) - command = self.config.get_pip_command(venv_path) + ["install"] + list(requirements) - wheels = self.config.home / "cache" / "wheels" - if wheels.is_dir(): - command += ["--find-links", str(wheels)] - if channel: - command += ["--index-url", channel] - - ui.info(f"Installing {' '.join(requirements)}") - process = subprocess.run(command, check=False) - if process.returncode != 0: - raise PloadError("dependency installation failed") + routes = [ + { + "kind": "cache", + "path": Path(wheel), + "summary": f"pload cache ({wheel.name})", + "label": f"Cache {wheel.name}", + } + for _, wheel in matches + ] + location = index or "pip/default index" + routes.append({ + "kind": "index", + "summary": f"package index ({location})", + "label": f"Index {location}", + }) + return routes diff --git a/tests/test_cli.py b/tests/test_cli.py index 312c3c1..d107b26 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -242,6 +242,7 @@ def test_new_without_options_uses_guided_creation(tmp_path, monkeypatch): "name": "guided", "description": "Created interactively", "requirements": None, + "package_strategy": None, }, ) monkeypatch.setattr( @@ -250,7 +251,7 @@ def test_new_without_options_uses_guided_creation(tmp_path, monkeypatch): ) monkeypatch.setattr( "pload.cli.DependencyManager.install_dependencies", - lambda self, path, requirements, channel: None, + lambda self, path, requirements, channel, strategy: None, ) assert main(["-H", str(tmp_path / "home"), "new"]) == 0 diff --git a/tests/test_dependency.py b/tests/test_dependency.py new file mode 100644 index 0000000..741237c --- /dev/null +++ b/tests/test_dependency.py @@ -0,0 +1,85 @@ +from subprocess import CompletedProcess + +import pytest +from packaging.tags import Tag + +from pload.errors import PloadError +from pload.managers.dependency import DependencyManager +from pload.managers.platform import ConfigManager + + +def test_failed_package_does_not_stop_later_requirements(tmp_path, monkeypatch): + manager = DependencyManager(ConfigManager(home=tmp_path / "home")) + commands = [] + return_codes = iter([1, 0]) + + def run(command, check): + commands.append(command) + return CompletedProcess(command, next(return_codes)) + + monkeypatch.setattr("pload.managers.dependency.subprocess.run", run) + + with pytest.raises(PloadError, match="successful packages were kept"): + manager.install_dependencies( + tmp_path / "environment", + ["not a requirement ???", "missing-package", "working-package"], + ) + + assert len(commands) == 2 + assert "missing-package" in commands[0] + assert "working-package" in commands[1] + + +def test_automatic_strategy_prefers_compatible_pload_cache(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + cache = config.home / "cache" / "wheels" + cache.mkdir(parents=True) + older = cache / "demo-1.0-py3-none-any.whl" + newest = cache / "demo-2.0-py3-none-any.whl" + older.touch() + newest.touch() + captured = [] + + monkeypatch.setattr( + "pload.managers.dependency._compatible_tags", + lambda path: {Tag("py3", "none", "any")}, + ) + monkeypatch.setattr( + "pload.managers.dependency.subprocess.run", + lambda command, check: captured.append(command) or CompletedProcess(command, 0), + ) + + DependencyManager(config).install_dependencies( + tmp_path / "environment", ["demo>=1"], strategy="auto", + ) + + assert str(newest) in captured[0] + assert str(older) not in captured[0] + + +def test_custom_strategy_can_choose_index_instead_of_cache(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + cache = config.home / "cache" / "wheels" + cache.mkdir(parents=True) + (cache / "demo-1.0-py3-none-any.whl").touch() + captured = [] + + monkeypatch.setattr( + "pload.managers.dependency._compatible_tags", + lambda path: {Tag("py3", "none", "any")}, + ) + monkeypatch.setattr( + "pload.managers.dependency.ui.select", + lambda message, choices, **kwargs: choices[-1].value, + ) + monkeypatch.setattr( + "pload.managers.dependency.subprocess.run", + lambda command, check: captured.append(command) or CompletedProcess(command, 0), + ) + + DependencyManager(config).install_dependencies( + tmp_path / "environment", ["demo>=1"], strategy="custom", + ) + + assert "demo>=1" in captured[0] + assert not any(item.endswith(".whl") for item in captured[0]) From c201ebaff7ffa1383b01b69b386c7bf289ee5e0a Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Sun, 27 Sep 2026 18:26:28 +0800 Subject: [PATCH 28/41] Execute saved acquisition routes exactly --- docs/declarative-environments.md | 6 +- src/pload/cli.py | 20 +++--- src/pload/declarative.py | 105 ++++++++++++++++++++++--------- src/pload/help_content.py | 6 +- tests/test_declarative.py | 14 ++--- 5 files changed, 100 insertions(+), 51 deletions(-) diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index e71949d..6a15616 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -229,7 +229,11 @@ override it for direct requirements. Transitive requirements use automatic ranking so installing `torch` does not ask the user to decide separately for `filelock`, `sympy`, and every other implementation detail. If any chosen resource changes or disappears, `apply` stops and asks for a new plan; it never -tries another candidate. +tries another candidate. The selected method is authoritative: choosing +`index-exact` performs and verifies that network download even when an identical +wheel is already present in pload's cache. Apply then installs all verified +artifacts locally with networking disabled, so `Processing .../cache/wheels/...` +is the installation phase rather than evidence that the selected route changed. ## Current boundaries diff --git a/src/pload/cli.py b/src/pload/cli.py index 985be61..40372d0 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -943,15 +943,19 @@ def run(argv=None): plan["plan_path"] = str(manager.save_plan(args.file, plan)) print_declarative_plan(plan) if action == "apply": - chooser_console.print(manager.apply(args.file, offline=args.offline)) + with ui.status("Applying the saved acquisition plan…") as update_status: + restored = manager.apply( + args.file, offline=args.offline, + progress=update_status, + ) + chooser_console.print(restored) else: - progress_console = ui.console() - print(manager.apply( - args.file, args.name, args.offline, - progress=lambda message: progress_console.print( - Text("• " + message, style="cyan") - ), - )) + with ui.status("Applying the saved acquisition plan…") as update_status: + restored = manager.apply( + args.file, args.name, args.offline, + progress=update_status, + ) + print(restored) return 0 if command == "new": diff --git a/src/pload/declarative.py b/src/pload/declarative.py index ff9bc6b..8dceb36 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -37,6 +37,15 @@ SHA256 = re.compile(r"[0-9a-f]{64}") +def _human_size(value): + size = float(value or 0) + units = ("B", "KiB", "MiB", "GiB") + for unit in units: + if size < 1024 or unit == units[-1]: + return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + + def normalized_name(value): return re.sub(r"[-_.]+", "-", value).lower() @@ -1327,29 +1336,6 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): compatible = [] environment_copies = [] self.cache.mkdir(parents=True, exist_ok=True) - repository_fetches = {} - for package_plan in plan["packages"]: - selected = package_plan["selected"] - artifact = selected.get("artifact") - if artifact and selected["method"] == "repository": - cached = self.cache / artifact["filename"] - if not (cached.is_file() and digest(cached) == artifact["sha256"]): - repository_fetches.setdefault(selected["location"], []).append( - (artifact["sha256"], cached) - ) - for repo_name, fetches in repository_fetches.items(): - if progress: - progress(f"Fetching {len(fetches)} artifacts from {repo_name}") - try: - ArtifactRepository.fetch_many( - data["repositories"][repo_name], fetches, path.parent, - ) - except PloadError as exc: - if progress: - progress( - "Batch fetch failed; retrying the same selected repository " - f"route individually: {exc}" - ) with tempfile.TemporaryDirectory(prefix="pload-apply-") as temporary: stage = Path(temporary) for index_number, package_plan in enumerate(plan["packages"], 1): @@ -1362,18 +1348,60 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): artifact = selected.get("artifact") if artifact: cached = self.cache / artifact["filename"] - if not (cached.is_file() and digest(cached) == artifact["sha256"]): + label = f"{package_plan['name']}=={package_plan['version']}" + method = selected["method"] + if method == "cache": + source = Path(selected["location"]) + if progress: + progress( + f"[{index_number}/{len(plan['packages'])}] " + f"Using selected cache for {label}" + ) + if not source.is_file() or digest(source) != artifact["sha256"]: + raise PloadError( + f"planned route failed for {label} (cache): " + "selected cache artifact changed or disappeared; " + "run pload plan to choose another route" + ) + install_artifact = source + else: + action = { + "index-exact": "Downloading", + "repository": "Fetching", + "configuration-artifact": "Copying", + "external-cache": "Copying", + }.get(method, "Acquiring") + size = artifact.get("size") + size_note = f" ({_human_size(size)})" if size else "" + if progress: + progress( + f"[{index_number}/{len(plan['packages'])}] {action} " + f"{label} via {method}:{selected['location']}{size_note}" + ) + acquired_dir = stage / f"artifact-{index_number}" + acquired_dir.mkdir() + acquired = acquired_dir / artifact["filename"] try: - self._acquire(path, data, package_plan, selected, cached, stage) + self._acquire( + path, data, package_plan, selected, acquired, stage, + progress=progress, + ) except (OSError, PloadError) as exc: raise PloadError( f"planned route failed for {package_plan['name']}==" - f"{package_plan['version']} ({selected['method']}): {exc}; " + f"{package_plan['version']} ({method}): {exc}; " "run pload plan to choose another route" ) from exc - if digest(cached) != artifact["sha256"]: - raise PloadError("planned artifact checksum mismatch") - artifacts.append(cached.resolve().as_uri() + "#sha256=" + artifact["sha256"]) + if not acquired.is_file() or digest(acquired) != artifact["sha256"]: + raise PloadError("planned artifact checksum mismatch") + pending = cached.with_name(cached.name + "." + uuid.uuid4().hex) + shutil.copyfile(acquired, pending) + pending.replace(cached) + install_artifact = cached + artifacts.append( + install_artifact.resolve().as_uri() + + "#sha256=" + artifact["sha256"] + ) else: if selected["method"] == "environment-copy": environment_copies.append(package_plan) @@ -1533,7 +1561,9 @@ def _copy_distribution(self, package, target_environment, stage): raise PloadError("environment package contains an unsafe path") bundle.extractall(site_packages) - def _acquire(self, path, data, package, selected, destination, stage): + def _acquire( + self, path, data, package, selected, destination, stage, progress=None, + ): artifact = selected["artifact"] if selected["method"] == "configuration-artifact": shutil.copyfile(selected["location"], destination) @@ -1555,7 +1585,20 @@ def _acquire(self, path, data, package, selected, destination, stage): artifact["url"], headers={"User-Agent": "pload-apply/1"}, ) with urlopen(request, timeout=60) as response, temporary.open("wb") as output: - shutil.copyfileobj(response, output) + received = 0 + expected = artifact.get("size") + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + output.write(chunk) + received += len(chunk) + if progress: + total = f" / {_human_size(expected)}" if expected else "" + progress( + f"Downloading {package['name']}=={package['version']}: " + f"{_human_size(received)}{total}" + ) except OSError as exc: raise PloadError(f"cannot download planned artifact: {exc}") from exc if digest(temporary) != artifact["sha256"]: diff --git a/src/pload/help_content.py b/src/pload/help_content.py index 5f9d90c..c132c9f 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -415,8 +415,10 @@ Apply requires a current `.pload_lock.toml` and `.pload_plan.toml`, then installs Python when needed, executes each selected route, verifies hashes, creates the environment, installs packages and runs `pip check`. It never resolves, tries an -alternative route, or uploads package bytes. Reapplying -the same configuration is idempotent; an +alternative route, or uploads package bytes. A selected `index-exact` route +downloads from the locked URL even if the same wheel is already cached; the +verified wheel is then installed locally with package networking disabled. +Reapplying the same configuration is idempotent; an existing environment with different state is never silently overwritten. """ GUIDES[("repo",)] = r""" diff --git a/tests/test_declarative.py b/tests/test_declarative.py index aed76cf..1f9601c 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -673,16 +673,8 @@ def test_apply_obeys_selected_route_without_fallback(tmp_path, monkeypatch): manager.plan(manifest) (manager.cache / wheel.name).unlink() - calls = [] - - def fail_selected(*args): - calls.append(args[3]["method"]) - raise PloadError("selected cache vanished") - - monkeypatch.setattr(manager, "_acquire", fail_selected) with pytest.raises(PloadError, match="planned route failed.*cache"): manager.apply(manifest) - assert calls == ["cache"] def test_apply_downloads_the_exact_url_saved_by_plan(tmp_path, monkeypatch): @@ -712,6 +704,8 @@ def test_apply_downloads_the_exact_url_saved_by_plan(tmp_path, monkeypatch): write_configuration(manifest, data) manager = DeclarativeEnvironmentManager(config) assert manager.plan(manifest)["packages"][0]["selected"]["method"] == "index-exact" + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) requested = [] def fake_urlopen(request, timeout): @@ -719,8 +713,10 @@ def fake_urlopen(request, timeout): return io.BytesIO(wheel.read_bytes()) monkeypatch.setattr(declarative_module, "urlopen", fake_urlopen) - restored = manager.apply(manifest) + progress = [] + restored = manager.apply(manifest, progress=progress.append) assert requested == [artifact_url] + assert any("Downloading pload-demo==1.0" in item for item in progress) assert execute([ config.get_pip_command(restored)[0], "-c", "import pload_demo; print(pload_demo.answer)", From 6b606748a87141d69391e05ee8ac00f19d85b12b Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Sun, 27 Sep 2026 18:52:50 +0800 Subject: [PATCH 29/41] Audit plan execution and measure route costs --- docs/declarative-environments.md | 34 +++- src/pload/cli.py | 25 ++- src/pload/declarative.py | 310 +++++++++++++++++++++++++------ src/pload/help_content.py | 18 +- src/pload/metadata_resolver.py | 27 ++- src/pload/resource_plan.py | 7 +- src/pload/snapshots.py | 36 ++++ tests/test_declarative.py | 224 +++++++++++++++++++++- tests/test_metadata_resolver.py | 38 ++++ 9 files changed, 626 insertions(+), 93 deletions(-) diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index 6a15616..ea27177 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -27,9 +27,10 @@ pload apply pload.toml # execute only the saved plan In a terminal, `plan` opens a route chooser for only the direct requirements the user wrote in `pload.toml`. Transitive dependencies remain exact and -auditable, but pload automatically assigns their fastest valid routes. A number -selects a route; Enter or `n` moves forward, `p` goes back, `u` undoes, `r` -restores the fastest choices, `s` saves, and `a` saves then applies. Use +auditable, but pload automatically assigns their lowest-cost valid routes from +the measured data available. Arrow keys and Enter select a real route; `n` +moves forward, `p` goes back, `u` undoes, `r` restores the recommended choices, +`s` saves, and `a` saves then applies. Use `pload plan --no-ui` in CI. Use `--json` for machine-readable inspection. ## Three files, one user-owned file @@ -205,7 +206,7 @@ version = "2.2.1" method = "external-cache" location = "/Users/me/Library/Caches/pip/.../numpy-2.2.1.whl" status = "ready" -estimated_seconds = 0.2 +measurement = "read 1.1 GiB/s" [package.artifact] filename = "numpy-2.2.1-cp312-cp312-macosx_14_0_arm64.whl" @@ -223,8 +224,14 @@ url = "https://files.pythonhosted.org/..." | `index-exact` | Exact locked URL, downloaded only during apply and verified by SHA-256. | | `index-resolve` | Compatible-mode fallback for the already locked version. | -Automatic ranking minimizes expected time: existing bytes first, compatible -local environments next, repositories next, and indexes last. The chooser can +The chooser never prints invented duration estimates. During each plan it +measures local artifact verification throughput, TCP connection RTT for index +hosts, and the actual availability-check RTT for repositories. These readings +appear as values such as `read 1.1 GiB/s`, `RTT 75 ms`, or `RTT unavailable`. +Because RTT is not transfer bandwidth, pload does not turn it into a fictional +download duration. Automatic ranking uses locality first and the current +measurements within each route class: pload cache, adjacent/external artifacts, +compatible local environments, then repository/index routes. The chooser can override it for direct requirements. Transitive requirements use automatic ranking so installing `torch` does not ask the user to decide separately for `filelock`, `sympy`, and every other implementation detail. If any chosen @@ -235,6 +242,21 @@ wheel is already present in pload's cache. Apply then installs all verified artifacts locally with networking disabled, so `Processing .../cache/wheels/...` is the installation phase rather than evidence that the selected route changed. +### What planning reads and writes + +With a current lock, planning only inventories possible routes. It indexes each +external cache once, inspects only matching-Python environments and only the +locked package names, checks configured repositories, and measures endpoint +RTT. The analysis API does not create or overwrite `.pload_plan.toml`. +Interactive Save/Apply and `--no-ui` persist choices; quitting or `--json` does +not. + +An unpinned configuration with a missing or stale lock has one additional job: +it must determine exact versions and the transitive graph before routes have a +meaning. That first resolution reads Simple API pages and independently served +Core Metadata and updates `.pload_lock.toml`; it still never downloads wheel +bodies. Later plans reuse the current lock and skip metadata resolution. + ## Current boundaries - Wheel reuse requires matching Python, ABI, OS and architecture tags. diff --git a/src/pload/cli.py b/src/pload/cli.py index 40372d0..3f0bb24 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -318,7 +318,7 @@ def build_parser(): plan.add_argument("--json", "-j", action="store_true", help="emit machine-readable JSON") plan.add_argument( "--no-ui", "-N", action="store_true", - help="select the fastest routes without opening the interactive route chooser", + help="save recommended measured routes without opening the interactive chooser", ) apply = subparsers.add_parser( "apply", description="Materialize the desired environment from available resources.", @@ -519,12 +519,15 @@ def print_declarative_plan(plan): if transitive_count: console.print( f"[dim]+ {transitive_count} transitive dependencies planned " - "automatically using their fastest valid routes[/]" + "automatically using their lowest-cost valid routes[/]" ) - console.print( - "[dim]Saved acquisition choices:[/] " - f"[cyan]{plan['plan_path']}[/]" - ) + if plan.get("saved"): + console.print( + "[dim]Saved acquisition choices:[/] " + f"[cyan]{plan['plan_path']}[/]" + ) + else: + console.print("[dim]Acquisition choices were not saved.[/]") def choose_declarative_routes(plan): @@ -550,10 +553,11 @@ def choose_declarative_routes(plan): ) choices = [] for index, route in enumerate(routes, 1): - seconds = route.get("estimated_seconds", 0) + measurement = route.get("measurement", "not measured") recommended = " recommended" if index == 1 else "" choices.append(Choice( - f"{route['method']:<18} ~{seconds:g}s {route['location']}{recommended}", + f"{route['method']:<18} {measurement:<20} " + f"{route['location']}{recommended}", value=("route", index - 1), )) if not choices: @@ -939,8 +943,9 @@ def run(argv=None): chooser_console = ui.console() if chooser_console.is_terminal and not args.no_ui: action = choose_declarative_routes(plan) - if action != "quit": - plan["plan_path"] = str(manager.save_plan(args.file, plan)) + if action != "quit": + plan["plan_path"] = str(manager.save_plan(args.file, plan)) + plan["saved"] = True print_declarative_plan(plan) if action == "apply": with ui.status("Applying the saved acquisition plan…") as update_status: diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 8dceb36..646048a 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -7,10 +7,13 @@ import re import shlex import shutil +import socket import sys import tempfile +import time import uuid import zipfile +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from urllib.parse import urlsplit from urllib.request import Request, urlopen @@ -29,8 +32,16 @@ from pload.managers.pyversion import PythonManager from pload.managers.venv import VenvManager from pload.metadata_resolver import resolve_metadata -from pload.resource_plan import file_digest, load_plan, write_plan -from pload.snapshots import RepositoryManager, digest, execute, probe, public_index, safe_name +from pload.resource_plan import file_digest, load_plan, plan_path, write_plan +from pload.snapshots import ( + RepositoryManager, + digest, + execute, + probe, + probe_resources, + public_index, + safe_name, +) PIN = re.compile(r"([A-Za-z0-9][A-Za-z0-9_.-]*)==([A-Za-z0-9][A-Za-z0-9_.+!-]*)") WHEEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+!-]*\.whl") @@ -46,6 +57,10 @@ def _human_size(value): size /= 1024 +def _duration(value): + return f"{value * 1000:.0f} ms" if value < 1 else f"{value:.2f} s" + + def normalized_name(value): return re.sub(r"[-_.]+", "-", value).lower() @@ -620,26 +635,57 @@ def _external_cache_roots(self): return list(dict.fromkeys(roots)) @staticmethod - def _find_external_artifact(roots, artifact): - """Find an exact wheel in known caches; matching a name alone is insufficient.""" + def _index_external_artifacts(roots, filenames): + """Walk every external cache once instead of once per locked package.""" + wanted = set(filenames) + found = {filename: [] for filename in wanted} for root in roots: try: - matches = root.rglob(artifact["filename"]) - for candidate in matches: - if candidate.is_file() and digest(candidate) == artifact["sha256"]: - return candidate + for candidate in root.rglob("*.whl"): + if candidate.name in wanted and candidate.is_file(): + found[candidate.name].append(candidate) except OSError: continue - return None + return found + + @staticmethod + def _verify_local_artifact(path, checksum): + started = time.perf_counter() + try: + valid = Path(path).is_file() and digest(path) == checksum + except OSError: + valid = False + elapsed = max(time.perf_counter() - started, 1e-6) + if not valid: + return False, elapsed, None + size = Path(path).stat().st_size + rate = size / elapsed + return True, elapsed, f"read {_human_size(rate)}/s" + + @staticmethod + def _network_rtt(url): + parsed = urlsplit(url) + if not parsed.hostname: + return None + port = parsed.port or (443 if parsed.scheme == "https" else 80) + started = time.perf_counter() + try: + with socket.create_connection((parsed.hostname, port), timeout=2): + pass + except OSError: + return None + return time.perf_counter() - started - def _environment_resources(self, expected, progress=None): + def _environment_resources(self, expected, package_names, progress=None): """Index exact, safely copyable distributions in compatible pload environments.""" resources = {} for environment in VenvManager(self.config).environments(register_discovered=False): + if environment.get("python") != expected.get("python"): + continue root = Path(environment["path"]) interpreter = root / ("Scripts/python.exe" if os.name == "nt" else "bin/python") try: - runtime = probe(interpreter) + runtime = probe_resources(interpreter, package_names) except PloadError: continue if any(runtime.get(key) != expected.get(key) for key in @@ -1130,10 +1176,8 @@ def plan(self, manifest_path, offline=False, progress=None): "path": str(path), "name": data["name"], "python": python_action, "packages": pending, "ready": False, "lock": lock, } - result["plan_path"] = str(write_plan( - path, configuration_digest(data), - file_digest(sidecar) if sidecar.is_file() else "", result, - )) + result["plan_path"] = str(plan_path(path)) + result["saved"] = False return result repository_checksums = {} for package in packages: @@ -1149,19 +1193,75 @@ def plan(self, manifest_path, offline=False, progress=None): artifact["sha256"] ) repository_objects = {} - for repo_name, checksums in repository_checksums.items(): - if progress: - progress(f"Checking artifact repository {repo_name}") + repository_rtt = {} + repository_pool = ThreadPoolExecutor(max_workers=max(1, len(repository_checksums))) + repository_futures = {} + + def check_repository(repo_name, checksums): + started = time.perf_counter() try: - repository_objects[repo_name] = ArtifactRepository.contains_many( + objects = ArtifactRepository.contains_many( repositories[repo_name], checksums, path.parent, ) except PloadError: - repository_objects[repo_name] = set() + objects = set() + return objects, time.perf_counter() - started + + for repo_name, checksums in repository_checksums.items(): + if progress: + progress(f"Checking artifact repository {repo_name}") + repository_futures[repo_name] = repository_pool.submit( + check_repository, repo_name, checksums, + ) external_cache_roots = self._external_cache_roots() + compatible_filenames = { + artifact["filename"] + for package in packages + for artifact in package.get("artifact", []) + if _artifact_is_compatible(artifact, supported_tags) + } + external_artifacts = self._index_external_artifacts( + external_cache_roots, compatible_filenames, + ) environment_resources = self._environment_resources( - data["environment"], progress=progress, + data["environment"], + [package["name"] for package in packages], + progress=progress, ) + endpoint_urls = {} + if network_allowed: + urls = [ + artifact.get("url") + for package in packages for artifact in package.get("artifact", []) + if artifact.get("url") + ] + [ + source.get("url") for source in data.get("sources", {}).values() + if source.get("kind") == "index" and source.get("url") + ] + for url in urls: + parsed = urlsplit(url) + key = (parsed.scheme, parsed.hostname, parsed.port) + endpoint_urls.setdefault(key, url) + endpoint_rtt = {} + if endpoint_urls: + with ThreadPoolExecutor(max_workers=min(4, len(endpoint_urls))) as pool: + measured = pool.map(self._network_rtt, endpoint_urls.values()) + endpoint_rtt = dict(zip(endpoint_urls, measured)) + for repo_name, future in repository_futures.items(): + repository_objects[repo_name], repository_rtt[repo_name] = future.result() + repository_pool.shutdown() + + def network_measurement(url): + parsed = urlsplit(url or "") + latency = endpoint_rtt.get((parsed.scheme, parsed.hostname, parsed.port)) + return (f"RTT {_duration(latency)}" if latency is not None + else "RTT unavailable") + + def network_cost(url): + parsed = urlsplit(url or "") + latency = endpoint_rtt.get((parsed.scheme, parsed.hostname, parsed.port)) + return latency if latency is not None else float("inf") + plans = [] for package in packages: candidates = [] @@ -1180,19 +1280,32 @@ def plan(self, manifest_path, offline=False, progress=None): continue cached = self.cache / artifact["filename"] adjacent = path.parent / "artifacts" / artifact["filename"] - if cached.is_file() and digest(cached) == artifact["sha256"]: - candidates.append(self._candidate( - "cache", cached, "ready", (0, 0, 0, 0), artifact - )) - if adjacent.is_file() and digest(adjacent) == artifact["sha256"]: + valid, elapsed, measurement = self._verify_local_artifact( + cached, artifact["sha256"], + ) + if valid: candidates.append(self._candidate( - "configuration-artifact", adjacent, "ready", (0, 0, 0, 1), artifact + "cache", cached, "ready", (0, elapsed, 0), artifact, + measurement, )) - external = self._find_external_artifact(external_cache_roots, artifact) - if external: + valid, elapsed, measurement = self._verify_local_artifact( + adjacent, artifact["sha256"], + ) + if valid: candidates.append(self._candidate( - "external-cache", external, "ready", (0, 0, 0, 2), artifact + "configuration-artifact", adjacent, "ready", (1, elapsed, 1), + artifact, measurement, )) + for external in external_artifacts.get(artifact["filename"], []): + valid, elapsed, measurement = self._verify_local_artifact( + external, artifact["sha256"], + ) + if valid: + candidates.append(self._candidate( + "external-cache", external, "ready", (1, elapsed, 2), + artifact, measurement, + )) + break repo_names = dict.fromkeys( artifact.get("repositories", []) + preferred_repositories ) @@ -1201,7 +1314,9 @@ def plan(self, manifest_path, offline=False, progress=None): if (spec and artifact["sha256"] in repository_objects.get(repo_name, set())): candidates.append(self._candidate( - "repository", repo_name, "remote", (0, 0, 1, repo_name), artifact + "repository", repo_name, "remote", + (3, repository_rtt[repo_name], repo_name), artifact, + f"RTT {_duration(repository_rtt[repo_name])}", )) if network_allowed: for source_name in package.get("sources", []): @@ -1209,20 +1324,26 @@ def plan(self, manifest_path, offline=False, progress=None): if source and source.get("kind") == "index": candidates.append(self._candidate( "index-exact", source_name, "network", - (0, 0, 2, source_name), artifact, + (3, network_cost( + artifact.get("url") or source["url"] + ), source_name), + artifact, + network_measurement(artifact.get("url") or source["url"]), )) environment_key = (normalized_name(package["name"]), package["version"]) for resource in environment_resources.get(environment_key, []): candidate = self._candidate( "environment-copy", resource["path"], "ready", - (0, 0, 0, 3, resource["id"]), + (2, 0, resource["id"]), measurement="local package", ) candidate["fingerprint"] = resource["fingerprint"] candidates.append(candidate) if policy.get("reproducibility") == "compatible" and network_allowed: source_name = (package.get("sources") or ["default"])[0] candidates.append(self._candidate( - "index-resolve", source_name, "network", (1, 1, 2, source_name) + "index-resolve", source_name, "network", + (3, network_cost(data["sources"][source_name]["url"]), source_name), + measurement=network_measurement(data["sources"][source_name]["url"]), )) candidates.sort(key=lambda item: item["cost"]) plans.append({ @@ -1236,25 +1357,18 @@ def plan(self, manifest_path, offline=False, progress=None): and all(item["selected"] for item in plans)) result = {"path": str(path), "name": data["name"], "python": python_action, "packages": plans, "ready": ready, "lock": lock} - result["plan_path"] = str(write_plan( - path, configuration_digest(data), file_digest(sidecar), result, - )) + result["plan_path"] = str(plan_path(path)) + result["saved"] = False return result @staticmethod - def _candidate(method, location, status, cost, artifact=None): + def _candidate( + method, location, status, cost, artifact=None, measurement=None, + ): result = {"method": method, "location": str(location), "status": status, "cost": list(cost)} - result["estimated_seconds"] = { - "cache": 0.05, - "configuration-artifact": 0.1, - "external-cache": 0.2, - "environment-copy": 0.5, - "repository": 5.0, - "index-exact": 30.0, - "index-resolve": 45.0, - "lock-required": 0.0, - }.get(method, 0.0) + if measurement: + result["measurement"] = measurement if artifact: result["artifact"] = artifact return result @@ -1315,11 +1429,23 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): label += f" ({item['rejections'][0]['reason']})" details.append(label) raise PloadError("no valid reproduction route for: " + ", ".join(details)) - if plan["python"]["method"] == "install-python": + python_method = plan["python"].get("method") + if offline and python_method == "install-python": + raise PloadError("offline apply forbids the selected Python network route") + if python_method == "install-python": if plan["python"]["status"] == "unavailable": raise PloadError("required Python is unavailable while offline") - PythonManager(self.config).install_python(data["environment"]["python"]) - interpreter = self.config.get_python_path(data["environment"]["python"]) + interpreter = Path(PythonManager(self.config).install_python( + data["environment"]["python"] + )).resolve() + elif python_method == "reuse-python": + interpreter = Path(plan["python"]["location"]).expanduser().resolve() + if not interpreter.is_file(): + raise PloadError( + "planned Python interpreter changed or disappeared; run pload plan again" + ) + else: + raise PloadError(f"unsupported Python acquisition method: {python_method}") self._check_runtime(interpreter, data["environment"]) target_name = name or data["name"] expected_state = manifest_digest(data) @@ -1345,6 +1471,11 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): f"{package_plan['name']}=={package_plan['version']}" ) selected = package_plan["selected"] + if offline and selected["method"] in {"index-exact", "index-resolve"}: + raise PloadError( + f"offline apply forbids the selected network route for " + f"{package_plan['name']}=={package_plan['version']}" + ) artifact = selected.get("artifact") if artifact: cached = self.cache / artifact["filename"] @@ -1384,7 +1515,7 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): try: self._acquire( path, data, package_plan, selected, acquired, stage, - progress=progress, + interpreter=interpreter, progress=progress, ) except (OSError, PloadError) as exc: raise PloadError( @@ -1424,7 +1555,7 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): for package_plan in compatible: source_name = package_plan["selected"]["location"] source = data.get("sources", {}).get(source_name, {}) - command = pip + ["install", "--no-deps"] + command = pip + ["install", "--no-deps", "--no-cache-dir"] if data["policy"].get("source_build") == "forbid": command += ["--only-binary=:all:"] if source.get("url"): @@ -1466,6 +1597,23 @@ def save_plan(self, manifest_path, plan): @staticmethod def _validate_saved_plan(saved, data): """Bind every executable route to one exact package in the current lock.""" + python = saved.get("python") + if not isinstance(python, dict): + raise PloadError("acquisition plan has no Python route") + if python.get("method") not in {"reuse-python", "install-python"}: + raise PloadError("acquisition plan has an unsupported Python route") + if python.get("status") not in {"ready", "network", "unavailable"}: + raise PloadError("acquisition plan has an invalid Python route status") + if python.get("method") == "install-python" and str(python.get("location")) != str( + data["environment"]["python"] + ): + raise PloadError("planned Python installation does not match the configuration") + expected_python_status = { + "reuse-python": {"ready"}, + "install-python": {"network", "unavailable"}, + }[python["method"]] + if python["status"] not in expected_python_status: + raise PloadError("Python route status does not match its acquisition method") locked = { (normalized_name(package["name"]), package["version"]): package for package in DeclarativeEnvironmentManager._locked_packages(data) @@ -1488,24 +1636,66 @@ def _validate_saved_plan(saved, data): method = package.get("method") if method not in allowed_methods: raise PloadError(f"unsupported method in acquisition plan: {method}") + expected_status = { + "cache": "ready", + "configuration-artifact": "ready", + "external-cache": "ready", + "environment-copy": "ready", + "repository": "remote", + "index-exact": "network", + "index-resolve": "network", + "unavailable": "unavailable", + }[method] + if package.get("status") != expected_status: + raise PloadError( + f"route status does not match method {method} for " + f"{package.get('name')}=={package.get('version')}" + ) artifact = package.get("artifact") if method in artifact_methods: - allowed = { - (item["filename"], item["sha256"]) + allowed_artifacts = { + (item["filename"], item["sha256"]): item for item in locked[key].get("artifact", []) } identity = ( artifact.get("filename"), artifact.get("sha256") ) if isinstance(artifact, dict) else None - if identity not in allowed: + if identity not in allowed_artifacts: raise PloadError( f"planned artifact is not present in the lock for " f"{package['name']}=={package['version']}" ) + locked_artifact = allowed_artifacts[identity] + if artifact.get("url", "") != locked_artifact.get("url", ""): + raise PloadError( + f"planned artifact URL does not match the lock for " + f"{package['name']}=={package['version']}" + ) elif artifact: raise PloadError(f"method {method} must not contain a wheel artifact") if method == "environment-copy" and not package.get("fingerprint"): raise PloadError("environment-copy route has no package fingerprint") + if method in {"index-exact", "index-resolve"}: + source_name = package.get("location") + if source_name not in locked[key].get("sources", []): + raise PloadError( + f"planned index is not locked for {package['name']}==" + f"{package['version']}" + ) + source = data.get("sources", {}).get(source_name) + if not source or source.get("kind") != "index": + raise PloadError(f"planned package index is unavailable: {source_name}") + if method == "repository": + repository = package.get("location") + if repository not in data.get("repositories", {}): + raise PloadError(f"planned repository is unavailable: {repository}") + allowed_repositories = set(locked_artifact.get("repositories", [])) + allowed_repositories.update(data["policy"].get("repositories", [])) + if repository not in allowed_repositories: + raise PloadError( + f"planned repository is not permitted for " + f"{package['name']}=={package['version']}" + ) if (method == "index-resolve" and data["policy"].get("reproducibility") != "compatible"): raise PloadError("index-resolve is forbidden by exact reproducibility policy") @@ -1562,7 +1752,8 @@ def _copy_distribution(self, package, target_environment, stage): bundle.extractall(site_packages) def _acquire( - self, path, data, package, selected, destination, stage, progress=None, + self, path, data, package, selected, destination, stage, interpreter, + progress=None, ): artifact = selected["artifact"] if selected["method"] == "configuration-artifact": @@ -1609,9 +1800,10 @@ def _acquire( download = stage / normalized_name(package["name"]) download.mkdir(exist_ok=True) execute([ - str(self.config.get_python_path(data["environment"]["python"])), + str(interpreter), "-m", "pip", "download", "--only-binary=:all:", "--no-deps", - "--dest", str(download), "--index-url", source["url"], + "--no-cache-dir", "--dest", str(download), + "--index-url", source["url"], f"{package['name']}=={package['version']}", ]) candidate = download / artifact["filename"] diff --git a/src/pload/help_content.py b/src/pload/help_content.py index c132c9f..993d96a 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -58,9 +58,11 @@ $ pload apply pload.toml ``` -`pload.toml` describes user intent. `plan` resolves from independent metadata, -inventories resources and saves one route per package without downloading wheel -bodies. `apply` executes only that saved plan and never changes route or uploads. +`pload.toml` describes user intent. `plan` resolves from independent metadata +when its managed lock is stale, then inventories resources without downloading +wheel bodies. Analysis does not overwrite saved choices; Save, Apply, or +`--no-ui` writes the plan. `apply` executes only that saved plan and never +changes route or uploads. """, ("new",): r""" # Create a managed environment @@ -385,10 +387,16 @@ Planning resolves missing/stale locks using Simple API pages and independent Core Metadata, then checks pload/pip/uv caches, compatible environments, adjacent artifacts and content-addressed repositories. It never downloads a -wheel body. The chooser shows only requirements written in `pload.toml` and +wheel body. A current lock skips metadata resolution. Cache roots are indexed +once and only matching-Python environments are inspected. The chooser reports +measured local read throughput and network/repository RTT instead of fixed time +guesses. Analysis does not overwrite `.pload_plan.toml`; Save, Apply, and +`--no-ui` persist choices, while quit and `--json` do not. + +The chooser shows only requirements written in `pload.toml` and automatically assigns routes to their transitive dependencies. It supports previous/next, undo, reset, save, and save-and-apply; `--no-ui` selects the -fastest routes for automation. +lowest-cost measured routes for automation. """ GUIDES[("lock",)] = r""" # Resolve the user declaration into an exact managed lock diff --git a/src/pload/metadata_resolver.py b/src/pload/metadata_resolver.py index a9e944e..2e666ff 100644 --- a/src/pload/metadata_resolver.py +++ b/src/pload/metadata_resolver.py @@ -4,6 +4,7 @@ import json import platform import sys +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from email.parser import BytesParser from email.policy import compat32 @@ -104,6 +105,15 @@ def __init__(self, sources, supported_tags, python_version, fetch=None): self.fetch = fetch or _read_url self.project_cache = {} self.dependency_cache = {} + self.source_metadata_capability = {} + + def prefetch(self, requirements): + names = sorted({canonicalize_name(item.name) for item in requirements}) + missing = [name for name in names if name not in self.project_cache] + if not missing: + return + with ThreadPoolExecutor(max_workers=min(8, len(missing))) as pool: + list(pool.map(self.candidates, missing)) def candidates(self, name): key = canonicalize_name(name) @@ -113,6 +123,8 @@ def candidates(self, name): metadata_missing = [] source_errors = [] for source_name, base_url in self.sources: + if self.source_metadata_capability.get(source_name) is False: + continue project_url = base_url.rstrip("/") + "/" + key + "/" try: body, content_type = self.fetch( @@ -135,6 +147,8 @@ def candidates(self, name): except UnicodeDecodeError as exc: raise PloadError(f"invalid Simple API HTML response for {name}") from exc files = parser.files + source_versions = {} + source_has_metadata = False for record in files: filename = record.get("filename", "") try: @@ -162,6 +176,7 @@ def candidates(self, name): if metadata_hashes is None: metadata_missing.append(filename) continue + source_has_metadata = True url = urljoin(project_url, record["url"]) candidate = MetadataCandidate( name=key, version=version, source=source_name, @@ -169,9 +184,15 @@ def candidates(self, name): sha256=sha256, size=int(record.get("size") or 0), metadata_sha256=metadata_hashes.get("sha256", ""), ) - current = by_version.get(version) + current = source_versions.get(version) if current is None or (candidate.size or sys.maxsize) < (current.size or sys.maxsize): - by_version[version] = candidate + source_versions[version] = candidate + self.source_metadata_capability[source_name] = source_has_metadata + if source_versions: + # Preserve source priority for duplicate versions while still + # retaining versions that exist only on a later fallback index. + for version, candidate in source_versions.items(): + by_version.setdefault(version, candidate) if not by_version and metadata_missing: raise PloadError( f"index has compatible files for {name}, but does not expose independent " @@ -215,6 +236,7 @@ def dependencies(self, candidate, marker_environment): continue requirements.append(requirement) self.dependency_cache[candidate] = requirements + self.prefetch(requirements) return requirements @@ -267,6 +289,7 @@ def resolve_metadata(requirements, sources, environment, supported_tags, fetch=N "extra": "", }) index = MetadataIndex(sources, supported_tags, python, fetch=fetch) + index.prefetch(parsed) try: result = Resolver(_Provider(index, marker_environment), BaseReporter()).resolve(parsed) except ResolutionImpossible as exc: diff --git a/src/pload/resource_plan.py b/src/pload/resource_plan.py index 63a30cc..94e74c1 100644 --- a/src/pload/resource_plan.py +++ b/src/pload/resource_plan.py @@ -99,8 +99,13 @@ def dump_plan(configuration_path, configuration_sha256, lock_sha256, plan): f"method = {_quoted(selected.get('method', 'unavailable'))}", f"location = {_quoted(selected.get('location', ''))}", f"status = {_quoted(selected.get('status', 'unavailable'))}", - f"estimated_seconds = {float(selected.get('estimated_seconds', 0.0))}", ] + if selected.get("measurement"): + lines.append(f"measurement = {_quoted(selected['measurement'])}") + if selected.get("estimated_seconds") is not None: + lines.append( + f"estimated_seconds = {float(selected['estimated_seconds'])}" + ) if not selected: lines.append(f"reason = {_quoted(reason)}") if selected.get("fingerprint"): diff --git a/src/pload/snapshots.py b/src/pload/snapshots.py index 1ebca15..6aacb5f 100644 --- a/src/pload/snapshots.py +++ b/src/pload/snapshots.py @@ -59,6 +59,34 @@ 'accelerators': accelerators, 'packages': packages})) """ +RESOURCE_PROBE = r""" +import hashlib, json, os, platform, sys +from importlib import metadata +requested = json.loads(sys.argv[1]) +packages = [] +for requested_name in requested: + try: + dist = metadata.distribution(requested_name) + except metadata.PackageNotFoundError: + continue + name = dist.metadata.get('Name') or requested_name + record = dist.read_text('RECORD') or '' + paths = [line.split(',', 1)[0].replace('\\', '/') for line in record.splitlines() + if line.strip()] + copyable = bool(paths) and all( + path and not path.startswith('/') and + os.path.normpath(path).replace('\\', '/') != '..' and + not os.path.normpath(path).replace('\\', '/').startswith('../') + for path in paths + ) + packages.append({'name': name, 'version': dist.version, + 'record_sha256': hashlib.sha256(record.encode()).hexdigest(), + 'copyable': copyable}) +print(json.dumps({'python': platform.python_version(), + 'implementation': sys.implementation.name, 'system': platform.system(), + 'machine': platform.machine(), 'packages': packages})) +""" + def execute(command, capture=True, cwd=None, timeout=None, input_text=None): try: @@ -105,6 +133,14 @@ def probe(python): return json.loads(execute([str(python), "-c", PROBE])) +def probe_resources(python, package_names): + """Inspect only locked packages needed by planning, without hashing file bodies.""" + return json.loads(execute([ + str(python), "-c", RESOURCE_PROBE, + json.dumps(sorted(set(package_names))), + ])) + + def public_index(url): """Store useful provenance without passwords, tokens or URL query strings.""" if not url: diff --git a/tests/test_declarative.py b/tests/test_declarative.py index 1f9601c..f522ce1 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -79,6 +79,12 @@ def write_configuration(path, data, with_lock=True): lock_path(path).write_text(dump_lock(path, locked), encoding="utf-8") +def plan_and_save(manager, path, **kwargs): + plan = manager.plan(path, **kwargs) + manager.save_plan(path, plan) + return plan + + def test_manifest_roundtrip(tmp_path): data = simple_manifest() path = tmp_path / "pload.toml" @@ -169,6 +175,36 @@ def test_exact_configuration_requires_an_artifact_route(tmp_path): assert manager.plan(path)["ready"] is False +def test_plan_analysis_does_not_create_or_overwrite_saved_choices(tmp_path): + path = tmp_path / "pload.toml" + write_configuration(path, simple_manifest()) + manager = DeclarativeEnvironmentManager(ConfigManager(home=tmp_path / "home")) + + plan = manager.plan(path) + saved_path = Path(plan["plan_path"]) + assert not saved_path.exists() + + manager.save_plan(path, plan) + before = saved_path.read_bytes() + manager.plan(path) + assert saved_path.read_bytes() == before + + +def test_plan_uses_measurements_instead_of_fixed_fake_times(tmp_path, monkeypatch): + path = tmp_path / "pload.toml" + write_configuration(path, simple_manifest()) + monkeypatch.setattr( + DeclarativeEnvironmentManager, "_network_rtt", staticmethod(lambda url: 0.123), + ) + + route = DeclarativeEnvironmentManager( + ConfigManager(home=tmp_path / "home") + ).plan(path)["packages"][0]["selected"] + + assert route["measurement"] == "RTT 123 ms" + assert "estimated_seconds" not in route + + def test_changed_python_rejects_incompatible_cached_wheel_before_apply(tmp_path, monkeypatch): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) @@ -196,7 +232,7 @@ def test_changed_python_rejects_incompatible_cached_wheel_before_apply(tmp_path, manager.cache.mkdir(parents=True) shutil.copyfile(wheel, manager.cache / wheel.name) - plan = manager.plan(manifest) + plan = plan_and_save(manager, manifest) package = plan["packages"][0] assert package["selected"] is None assert package["rejections"][0]["artifact"] == wheel.name @@ -247,7 +283,7 @@ def test_cache_plan_applies_without_package_network_access(tmp_path, monkeypatch manager = DeclarativeEnvironmentManager(config) manager.cache.mkdir(parents=True) shutil.copyfile(wheel, manager.cache / wheel.name) - assert manager.plan(manifest)["packages"][0]["selected"]["method"] == "cache" + assert plan_and_save(manager, manifest)["packages"][0]["selected"]["method"] == "cache" original_execute = declarative_module.execute @@ -294,11 +330,19 @@ def test_plan_discovers_and_apply_uses_exact_external_cache(tmp_path, monkeypatc write_configuration(manifest, data) manager = DeclarativeEnvironmentManager(config) - selected = manager.plan(manifest)["packages"][0]["selected"] + selected = plan_and_save(manager, manifest)["packages"][0]["selected"] assert selected["method"] == "external-cache" assert selected["location"] == str(wheel.resolve()) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) original_execute = declarative_module.execute + original_acquire = manager._acquire + acquisition_methods = [] + + def record_acquisition(*args, **kwargs): + acquisition_methods.append(args[3]["method"]) + return original_acquire(*args, **kwargs) def reject_download(command, *args, **kwargs): if "download" in command: @@ -306,7 +350,9 @@ def reject_download(command, *args, **kwargs): return original_execute(command, *args, **kwargs) monkeypatch.setattr(declarative_module, "execute", reject_download) + monkeypatch.setattr(manager, "_acquire", record_acquisition) restored = manager.apply(manifest) + assert acquisition_methods == ["external-cache"] assert digest(manager.cache / wheel.name) == digest(wheel) assert original_execute([ config.get_pip_command(restored)[0], "-c", @@ -341,7 +387,7 @@ def test_plan_reuses_package_from_compatible_pload_environment(tmp_path, monkeyp write_configuration(manifest, data) manager = DeclarativeEnvironmentManager(config) - selected = manager.plan(manifest)["packages"][0]["selected"] + selected = plan_and_save(manager, manifest)["packages"][0]["selected"] assert selected["method"] == "environment-copy" assert selected["location"] == str(source) assert len(selected["fingerprint"]) == 64 @@ -413,7 +459,7 @@ def fake_metadata(requirements, sources, environment, supported_tags): assert len(resolver_calls) == 1 manager.cache.mkdir(parents=True) shutil.copyfile(wheel, manager.cache / wheel.name) - manager.plan(manifest) + plan_and_save(manager, manifest) restored = manager.apply(manifest) assert len(resolver_calls) == 1 output = original_execute([ @@ -502,7 +548,7 @@ def fake_metadata(requirements, sources, environment, supported_tags): assert {item["name"] for item in locked["package"]} == {"extra-demo", "pload-demo"} -def test_describe_plan_apply_through_content_repository(tmp_path): +def test_describe_plan_apply_through_content_repository(tmp_path, monkeypatch): source_config = ConfigManager(home=tmp_path / "source-home") repository = tmp_path / "artifact-store" save_settings(source_config.home, { @@ -542,10 +588,23 @@ def test_describe_plan_apply_through_content_repository(tmp_path): target_config = ConfigManager(home=tmp_path / "target-home") target_config.get_python_path = lambda version=None: Path(sys.executable) target = DeclarativeEnvironmentManager(target_config) - plan = target.plan(manifest) + plan = plan_and_save(target, manifest) assert plan["packages"][0]["selected"]["method"] == "repository" + target.cache.mkdir(parents=True) + shutil.copyfile(wheel, target.cache / artifact["filename"]) + original_fetch = declarative_module.ArtifactRepository.fetch.__func__ + fetches = [] + + def record_fetch(cls, *args, **kwargs): + fetches.append(args[1]) + return original_fetch(cls, *args, **kwargs) + + monkeypatch.setattr( + declarative_module.ArtifactRepository, "fetch", classmethod(record_fetch), + ) apply_progress = [] restored = target.apply(manifest, progress=apply_progress.append) + assert fetches == [artifact["sha256"]] output = execute([target_config.get_pip_command(restored)[0], "-c", "import pload_demo; print(pload_demo.answer)"]) assert output == "42" @@ -612,7 +671,7 @@ def test_apply_requires_a_saved_plan_and_never_auto_publishes(tmp_path): manager = DeclarativeEnvironmentManager(config) with pytest.raises(PloadError, match="plan is missing"): manager.apply(manifest) - manager.plan(manifest) + plan_and_save(manager, manifest) manager.apply(manifest) assert not (repository / "objects" / checksum).exists() @@ -670,7 +729,8 @@ def test_apply_obeys_selected_route_without_fallback(tmp_path, monkeypatch): manager = DeclarativeEnvironmentManager(config) manager.cache.mkdir(parents=True) shutil.copyfile(wheel, manager.cache / wheel.name) - manager.plan(manifest) + wheel.unlink() + plan_and_save(manager, manifest) (manager.cache / wheel.name).unlink() with pytest.raises(PloadError, match="planned route failed.*cache"): @@ -703,7 +763,7 @@ def test_apply_downloads_the_exact_url_saved_by_plan(tmp_path, monkeypatch): manifest.parent.mkdir() write_configuration(manifest, data) manager = DeclarativeEnvironmentManager(config) - assert manager.plan(manifest)["packages"][0]["selected"]["method"] == "index-exact" + assert plan_and_save(manager, manifest)["packages"][0]["selected"]["method"] == "index-exact" manager.cache.mkdir(parents=True) shutil.copyfile(wheel, manager.cache / wheel.name) requested = [] @@ -723,6 +783,80 @@ def fake_urlopen(request, timeout): ]) == "42" +def test_offline_apply_rejects_saved_network_route(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + }) + data["package"][0]["artifact"] = [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + "url": "https://example.invalid/" + wheel.name, + }] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + plan_and_save(manager, manifest) + monkeypatch.setattr( + declarative_module, "urlopen", + lambda *args, **kwargs: pytest.fail("offline apply opened the network"), + ) + + with pytest.raises(PloadError, match="offline apply forbids.*network route"): + manager.apply(manifest, offline=True) + + +def test_apply_refuses_to_replace_missing_planned_python(tmp_path): + planned_python = tmp_path / "planned-python" + planned_python.write_text("placeholder", encoding="utf-8") + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: planned_python + manifest = tmp_path / "pload.toml" + data = simple_manifest("compatible") + data["environment"]["dependencies"] = [] + data["package"] = [] + write_configuration(manifest, data, with_lock=False) + manager = DeclarativeEnvironmentManager(config) + plan_and_save(manager, manifest) + planned_python.unlink() + + with pytest.raises(PloadError, match="planned Python interpreter.*disappeared"): + manager.apply(manifest) + + +def test_compatible_index_route_disables_pip_cache(tmp_path, monkeypatch): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + manifest = tmp_path / "pload.toml" + write_configuration(manifest, simple_manifest("compatible")) + manager = DeclarativeEnvironmentManager(config) + plan_and_save(manager, manifest) + monkeypatch.setattr(manager, "_check_runtime", lambda *args: None) + environment = tmp_path / "environment" + + def create_environment(*args, **kwargs): + environment.mkdir() + return environment + + monkeypatch.setattr(VenvManager, "create_venv", create_environment) + commands = [] + monkeypatch.setattr( + declarative_module, "execute", + lambda command, *args, **kwargs: commands.append(command) or "", + ) + + manager.apply(manifest) + + route_command = next(command for command in commands if "demo==1.0" in command) + assert "--no-cache-dir" in route_command + + def test_apply_rejects_plan_that_omits_a_locked_package(tmp_path): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) @@ -747,6 +881,76 @@ def test_apply_rejects_plan_that_omits_a_locked_package(tmp_path): manager.apply(manifest) +def test_apply_rejects_tampered_route_status_and_artifact_url(tmp_path): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + }) + wheel = tiny_wheel(tmp_path / "fixture") + data["package"][0]["artifact"] = [{ + "filename": wheel.name, + "sha256": digest(wheel), + "tags": ["py3-none-any"], + "repositories": [], + "url": "https://locked.invalid/demo.whl", + }] + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + generated = manager.plan(manifest) + + generated["packages"][0]["selected"]["status"] = "ready" + manager.save_plan(manifest, generated) + with pytest.raises(PloadError, match="route status does not match"): + manager.apply(manifest) + + generated = manager.plan(manifest) + generated["packages"][0]["selected"]["artifact"]["url"] = ( + "https://changed.invalid/demo.whl" + ) + manager.save_plan(manifest, generated) + with pytest.raises(PloadError, match="artifact URL does not match"): + manager.apply(manifest) + + +def test_legacy_exact_download_uses_planned_python_and_disables_pip_cache( + tmp_path, monkeypatch, +): + config = ConfigManager(home=tmp_path / "home") + manager = DeclarativeEnvironmentManager(config) + wheel = tiny_wheel(tmp_path / "fixture") + planned_python = tmp_path / "planned-python" + command_seen = [] + + def fake_execute(command, *args, **kwargs): + command_seen.append(command) + destination = Path(command[command.index("--dest") + 1]) + shutil.copyfile(wheel, destination / wheel.name) + return "" + + monkeypatch.setattr(declarative_module, "execute", fake_execute) + output = tmp_path / "result.whl" + manager._acquire( + tmp_path / "pload.toml", + {"sources": {"default": {"url": "https://index.invalid/simple"}}}, + {"name": "pload-demo", "version": "1.0"}, + { + "method": "index-exact", "location": "default", + "artifact": {"filename": wheel.name, "sha256": digest(wheel)}, + }, + output, tmp_path, interpreter=planned_python, + ) + + assert command_seen[0][:3] == [str(planned_python), "-m", "pip"] + assert "--no-cache-dir" in command_seen[0] + assert output.read_bytes() == wheel.read_bytes() + + def test_declarative_commands_and_shell_integration(capsys): parser = build_parser() assert parser.parse_args(["describe", "v1", "-o", "env.toml"]).output == "env.toml" diff --git a/tests/test_metadata_resolver.py b/tests/test_metadata_resolver.py index 095e175..8bdcc55 100644 --- a/tests/test_metadata_resolver.py +++ b/tests/test_metadata_resolver.py @@ -123,3 +123,41 @@ def fetch(url, accept, maximum): ("first-demo", "1.0"), ("second-demo", "2.0"), ] assert not any(url.endswith(".whl") for url in requested) + + +def test_later_source_can_supply_version_missing_from_preferred_source(): + first_metadata = b"Metadata-Version: 2.1\nName: root-demo\nVersion: 1.0\n\n" + later_metadata = b"Metadata-Version: 2.1\nName: root-demo\nVersion: 2.0\n\n" + first_wheel = "root_demo-1.0-py3-none-any.whl" + later_wheel = "root_demo-2.0-py3-none-any.whl" + responses = { + "https://first.invalid/simple/root-demo/": _project( + first_wheel, "a" * 64, hashlib.sha256(first_metadata).hexdigest(), + ), + "https://later.invalid/simple/root-demo/": _project( + later_wheel, "b" * 64, hashlib.sha256(later_metadata).hexdigest(), + ), + "https://files.invalid/" + first_wheel + ".metadata": first_metadata, + "https://files.invalid/" + later_wheel + ".metadata": later_metadata, + } + + def fetch(url, accept, maximum): + return responses[url], ( + "application/vnd.pypi.simple.v1+json" + if url.endswith("/") else "application/octet-stream" + ) + + packages = resolve_metadata( + ["root-demo>=2"], + [ + ("preferred", "https://first.invalid/simple"), + ("fallback", "https://later.invalid/simple"), + ], + {"python": "3.12.1", "implementation": "cpython", + "system": "Linux", "machine": "x86_64"}, + {Tag("py3", "none", "any")}, fetch=fetch, + ) + + assert [(item["version"], item["sources"]) for item in packages] == [ + ("2.0", ["fallback"]), + ] From 992b8427dca99a9461ba4c4e03e0e3c2fb61b198 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Sun, 27 Sep 2026 19:07:59 +0800 Subject: [PATCH 30/41] Add automatic planning and transparent apply output --- README.md | 6 +- docs/declarative-environments.md | 35 ++++++- src/pload/cli.py | 55 ++++++++--- src/pload/declarative.py | 161 ++++++++++++++++++++++++++----- src/pload/help_content.py | 27 ++++-- tests/test_cli.py | 35 +++++++ tests/test_declarative.py | 87 ++++++++++++++++- 7 files changed, 355 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index c3515cd..1fb0ca4 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,11 @@ written by the user; transitive dependencies are routed automatically. It supports previous/next, undo, reset, save and save-and-apply through a compact shortcut row (`p/n/u/r/s/a/q`); the selection list itself contains only real package sources. `apply` executes exactly the saved plan: -it never re-resolves, changes route, falls back, or uploads packages. +it never re-resolves, changes route, falls back, or uploads packages. Set +`[plan].mode = "auto"` in `pload.toml` to save recommended routes without the +chooser; strict per-package exceptions live under `[plan.packages.NAME]`. +`apply` prints the saved plan first, then package headings, exact source +URLs/paths and real download byte progress. Back up a special or slow wheel only when you explicitly request it: diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index ea27177..1c5660b 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -25,8 +25,9 @@ pload plan pload.toml # resolve metadata, inventory and choose routes pload apply pload.toml # execute only the saved plan ``` -In a terminal, `plan` opens a route chooser for only the direct requirements -the user wrote in `pload.toml`. Transitive dependencies remain exact and +By default, `plan` opens a route chooser for only the direct requirements the +user wrote in `pload.toml`. Set `[plan].mode = "auto"` to save the recommended +routes without opening the chooser. Transitive dependencies remain exact and auditable, but pload automatically assigns their lowest-cost valid routes from the measured data available. Arrow keys and Enter select a real route; `n` moves forward, `p` goes back, `u` undoes, `r` restores the recommended choices, @@ -76,6 +77,13 @@ source_build = "forbid" publish_missing_artifacts = false repositories = ["lab"] +[plan] +mode = "auto" + +[plan.packages.torch] +method = "repository" +location = "lab" + [sources.pypi] kind = "index" url = "https://pypi.org/simple" @@ -127,6 +135,19 @@ reused exactly, which avoids repeatedly downloading large special builds. | `publish_missing_artifacts` | `false` recommended | Retained for schema-1 compatibility but ignored by `apply`. Upload is always explicit through `pload remote add`. | | `repositories` | `[]` | Repositories to search while planning. Every name must have a matching repository table. This never authorizes upload. | +### `[plan]` and `[plan.packages.NAME]` + +| Field | Default | Meaning | +| --- | --- | --- | +| `mode` | `"interactive"` | `interactive` opens the route chooser; `auto` immediately saves the recommended measured routes. | +| `plan.packages.NAME.method` | optional | Require one acquisition method for a special package, such as `repository`, `index-exact`, `cache`, or `environment-copy`. | +| `plan.packages.NAME.location` | optional | Further require a specific source, repository, environment, or path. | + +Most configurations only need `mode = "auto"`. Package overrides are strict: +if the requested route is not currently available, planning reports the package +as unavailable instead of silently choosing another route. Overrides are also +checked again by `apply`, so editing the managed plan cannot bypass them. + ### `[sources.NAME]` | Field | Meaning | @@ -248,8 +269,14 @@ With a current lock, planning only inventories possible routes. It indexes each external cache once, inspects only matching-Python environments and only the locked package names, checks configured repositories, and measures endpoint RTT. The analysis API does not create or overwrite `.pload_plan.toml`. -Interactive Save/Apply and `--no-ui` persist choices; quitting or `--json` does -not. +Interactive Save/Apply, `--no-ui`, and `[plan].mode = "auto"` persist choices; +quitting or `--json` does not. + +`pload apply` first prints the complete persisted plan table. It then prints a +heading for each exact package, the selected source URL/path/repository, real +download byte progress for network artifacts, environment creation and +verification stages. This makes the saved decision and its execution visible in +one command without allowing apply to recalculate the plan. An unpinned configuration with a missing or stale lock has one additional job: it must determine exact versions and the transitive graph before routes have a diff --git a/src/pload/cli.py b/src/pload/cli.py index 3f0bb24..95a89a6 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -485,7 +485,8 @@ def print_declarative_plan(plan): python = plan["python"] ui.heading( f"Plan · {plan['name']}", - f"Python {python['method']} · {python['status']}", + f"Python {python['method']} · {python['status']} · " + f"{plan.get('planning_mode', 'interactive')} planning", output=console, ) console.print(f"[dim]{python['location']}[/]", overflow="ellipsis", no_wrap=True) @@ -530,6 +531,32 @@ def print_declarative_plan(plan): console.print("[dim]Acquisition choices were not saved.[/]") +def print_apply_progress(message, output=None): + """Render durable, package-oriented apply output instead of one opaque spinner.""" + console = output or ui.console() + kind, _, detail = str(message).partition(" ") + if kind == "PACKAGE": + counter, _, package = detail.partition(" ") + console.print() + line = Text("▶ ", style="bold cyan") + line.append(package, style="bold white") + line.append(f" {counter}", style="dim") + console.print(line) + elif kind == "SOURCE": + line = Text(" ↳ ", style="green") + line.append(detail, style="white") + console.print(line) + elif kind == "DOWNLOAD": + console.print(Text(f" downloaded {detail}", style="cyan")) + elif kind == "STAGE": + console.print() + console.print(Text(f"▶ {detail}", style="bold cyan")) + elif kind == "DONE": + ui.success(detail, output=console) + else: + console.print(message) + + def choose_declarative_routes(plan): """Choose direct-package routes with familiar cursor-key menus.""" from pload.resource_plan import PlanSelection @@ -941,25 +968,29 @@ def run(argv=None): ) action = "save" chooser_console = ui.console() - if chooser_console.is_terminal and not args.no_ui: + if (chooser_console.is_terminal and not args.no_ui + and plan.get("planning_mode") != "auto"): action = choose_declarative_routes(plan) if action != "quit": plan["plan_path"] = str(manager.save_plan(args.file, plan)) plan["saved"] = True print_declarative_plan(plan) if action == "apply": - with ui.status("Applying the saved acquisition plan…") as update_status: - restored = manager.apply( - args.file, offline=args.offline, - progress=update_status, - ) + restored = manager.apply( + args.file, offline=args.offline, + progress=lambda message: print_apply_progress( + message, chooser_console + ), + ) chooser_console.print(restored) else: - with ui.status("Applying the saved acquisition plan…") as update_status: - restored = manager.apply( - args.file, args.name, args.offline, - progress=update_status, - ) + apply_console = ui.console() + saved_plan = manager.saved_plan(args.file) + print_declarative_plan(saved_plan) + restored = manager.apply( + args.file, args.name, args.offline, + progress=lambda message: print_apply_progress(message, apply_console), + ) print(restored) return 0 diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 646048a..972afc5 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -160,6 +160,14 @@ def dump_manifest(data): repositories = policy.get("repositories", []) if repositories: lines.append(f"repositories = {_array(repositories)}") + planning = data.get("plan") + if planning: + lines += ["", "[plan]", f"mode = {_quoted(planning.get('mode', 'interactive'))}"] + for package_name, route in sorted(planning.get("packages", {}).items()): + lines += ["", f"[plan.packages.{_quoted(package_name)}]"] + lines.append(f"method = {_quoted(route['method'])}") + if route.get("location"): + lines.append(f"location = {_quoted(route['location'])}") capabilities = data.get("capabilities", {}) if capabilities: lines += ["", "[capabilities]"] @@ -307,6 +315,30 @@ def validate_manifest(data): raise PloadError("policy.source_build must be fallback or forbid") if not isinstance(policy.get("publish_missing_artifacts", False), bool): raise PloadError("policy.publish_missing_artifacts must be true or false") + planning = data.get("plan", {}) + if not isinstance(planning, dict): + raise PloadError("plan must be a table") + if planning.get("mode", "interactive") not in {"interactive", "auto"}: + raise PloadError("plan.mode must be interactive or auto") + package_routes = planning.get("packages", {}) + if not isinstance(package_routes, dict): + raise PloadError("plan.packages must be a table") + route_methods = { + "cache", "configuration-artifact", "external-cache", "environment-copy", + "repository", "index-exact", "index-resolve", + } + normalized_routes = set() + for package_name, route in package_routes.items(): + if not isinstance(package_name, str) or not package_name: + raise PloadError("plan.packages keys must be package names") + normalized = normalized_name(package_name) + if normalized in normalized_routes: + raise PloadError(f"duplicate plan override for package {package_name}") + normalized_routes.add(normalized) + if not isinstance(route, dict) or route.get("method") not in route_methods: + raise PloadError(f"plan.packages.{package_name}.method is invalid") + if route.get("location") is not None and not isinstance(route["location"], str): + raise PloadError(f"plan.packages.{package_name}.location must be a string") policy_repositories = policy.get("repositories", []) if (not isinstance(policy_repositories, list) or any(not isinstance(name, str) for name in policy_repositories)): @@ -1105,6 +1137,12 @@ def lock(self, manifest_path, offline=False, progress=None): def plan(self, manifest_path, offline=False, progress=None): path, data, current, legacy = self._load_state(manifest_path) + planning = data.get("plan", {}) + planning_mode = planning.get("mode", "interactive") + route_overrides = { + normalized_name(name): route + for name, route in planning.get("packages", {}).items() + } dependencies = data["environment"].get("dependencies", []) direct_names = { normalized_name(_parse_dependency(value).name) for value in dependencies @@ -1175,6 +1213,7 @@ def plan(self, manifest_path, offline=False, progress=None): result = { "path": str(path), "name": data["name"], "python": python_action, "packages": pending, "ready": False, "lock": lock, + "planning_mode": planning_mode, } result["plan_path"] = str(plan_path(path)) result["saved"] = False @@ -1346,6 +1385,25 @@ def network_cost(url): measurement=network_measurement(data["sources"][source_name]["url"]), )) candidates.sort(key=lambda item: item["cost"]) + override = route_overrides.get(normalized_name(package["name"])) + if override: + matching = [ + candidate for candidate in candidates + if candidate["method"] == override["method"] + and (not override.get("location") + or candidate["location"] == override["location"]) + ] + if matching: + candidates = matching + else: + requested = override["method"] + if override.get("location"): + requested += ":" + override["location"] + rejections.insert(0, { + "artifact": package["name"], + "reason": f"configured plan route is unavailable: {requested}", + }) + candidates = [] plans.append({ "name": package["name"], "version": package["version"], "direct": normalized_name(package["name"]) in direct_names, @@ -1356,7 +1414,8 @@ def network_cost(url): ready = (not lock_required and python_action["status"] != "unavailable" and all(item["selected"] for item in plans)) result = {"path": str(path), "name": data["name"], "python": python_action, - "packages": plans, "ready": ready, "lock": lock} + "packages": plans, "ready": ready, "lock": lock, + "planning_mode": planning_mode} result["plan_path"] = str(plan_path(path)) result["saved"] = False return result @@ -1386,7 +1445,12 @@ def _locked_packages(data): "sources": ["default"], "artifact": []}) return result - def apply(self, manifest_path, name=None, offline=False, progress=None): + def saved_plan(self, manifest_path): + """Return the exact persisted plan in the same shape used by the plan UI.""" + _, _, plan = self._load_saved_execution_plan(manifest_path) + return plan + + def _load_saved_execution_plan(self, manifest_path): path, data, current, _ = self._load_state(manifest_path) if data["environment"].get("dependencies") and not current: raise PloadError( @@ -1401,6 +1465,9 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): plan = { "path": str(path), "name": saved["name"], "ready": saved["ready"], "python": saved["python"], "packages": [], + "lock": {"status": "current", "required": False, "path": str(lock_file)}, + "plan_path": str(plan_path(path)), "saved": True, + "planning_mode": data.get("plan", {}).get("mode", "interactive"), } for package in saved.get("package", []): selected = None @@ -1408,14 +1475,16 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): selected = { "method": package["method"], "location": package["location"], "status": package["status"], - "estimated_seconds": package.get("estimated_seconds", 0.0), } + if package.get("measurement"): + selected["measurement"] = package["measurement"] if package.get("fingerprint"): selected["fingerprint"] = package["fingerprint"] if package.get("artifact"): selected["artifact"] = package["artifact"] plan["packages"].append({ "name": package["name"], "version": package["version"], + "direct": package.get("direct", True), "selected": selected, "alternatives": [], "rejections": ([{"reason": package["reason"]}] if package.get("reason") else []), @@ -1429,6 +1498,10 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): label += f" ({item['rejections'][0]['reason']})" details.append(label) raise PloadError("no valid reproduction route for: " + ", ".join(details)) + return path, data, plan + + def apply(self, manifest_path, name=None, offline=False, progress=None): + path, data, plan = self._load_saved_execution_plan(manifest_path) python_method = plan["python"].get("method") if offline and python_method == "install-python": raise PloadError("offline apply forbids the selected Python network route") @@ -1465,12 +1538,12 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): with tempfile.TemporaryDirectory(prefix="pload-apply-") as temporary: stage = Path(temporary) for index_number, package_plan in enumerate(plan["packages"], 1): + selected = package_plan["selected"] if progress: progress( - f"[{index_number}/{len(plan['packages'])}] Preparing " + f"PACKAGE {index_number}/{len(plan['packages'])} " f"{package_plan['name']}=={package_plan['version']}" ) - selected = package_plan["selected"] if offline and selected["method"] in {"index-exact", "index-resolve"}: raise PloadError( f"offline apply forbids the selected network route for " @@ -1484,10 +1557,7 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): if method == "cache": source = Path(selected["location"]) if progress: - progress( - f"[{index_number}/{len(plan['packages'])}] " - f"Using selected cache for {label}" - ) + progress(f"SOURCE using cache {source}") if not source.is_file() or digest(source) != artifact["sha256"]: raise PloadError( f"planned route failed for {label} (cache): " @@ -1496,18 +1566,24 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): ) install_artifact = source else: - action = { - "index-exact": "Downloading", - "repository": "Fetching", - "configuration-artifact": "Copying", - "external-cache": "Copying", - }.get(method, "Acquiring") size = artifact.get("size") size_note = f" ({_human_size(size)})" if size else "" + if method == "index-exact": + source_description = artifact.get("url") or data[ + "sources" + ][selected["location"]]["url"] + action = "downloading from" + elif method == "repository": + source_description = data["repositories"][ + selected["location"] + ]["location"] + action = "fetching from repository" + else: + source_description = selected["location"] + action = "copying from" if progress: progress( - f"[{index_number}/{len(plan['packages'])}] {action} " - f"{label} via {method}:{selected['location']}{size_note}" + f"SOURCE {action} {source_description}{size_note}" ) acquired_dir = stage / f"artifact-{index_number}" acquired_dir.mkdir() @@ -1535,12 +1611,23 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): ) else: if selected["method"] == "environment-copy": + if progress: + progress( + f"SOURCE copying installed package from " + f"{selected['location']}" + ) environment_copies.append(package_plan) else: + source = data.get("sources", {}).get(selected["location"], {}) + if progress: + progress( + f"SOURCE downloading from " + f"{source.get('url', selected['location'])}" + ) compatible.append(package_plan) venvs = VenvManager(self.config) if progress: - progress(f"Creating environment {target_name}") + progress(f"STAGE creating environment {target_name}") env = venvs.create_venv( version=str(interpreter), name=target_name, description=f"Applied from {path.name}", @@ -1548,6 +1635,8 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): try: pip = self.config.get_pip_command(env) if artifacts: + if progress: + progress("STAGE installing verified wheel artifacts") execute(pip + ["install", "--no-index", "--no-deps"] + artifacts, capture=False) for package_plan in environment_copies: @@ -1574,7 +1663,7 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): str(requirements)]) execute(pip + ["check"], capture=False) if progress: - progress("Verified exact requirements and dependency consistency") + progress("DONE verified exact requirements and dependency consistency") (env / ".pload-manifest.sha256").write_text( expected_state + "\n", encoding="utf-8" ) @@ -1651,6 +1740,19 @@ def _validate_saved_plan(saved, data): f"route status does not match method {method} for " f"{package.get('name')}=={package.get('version')}" ) + configured_route = next(( + route for name, route in data.get("plan", {}).get("packages", {}).items() + if normalized_name(name) == key[0] + ), None) + if configured_route and ( + method != configured_route["method"] + or (configured_route.get("location") + and package.get("location") != configured_route["location"]) + ): + raise PloadError( + f"saved route does not match the configured plan override for " + f"{package.get('name')}=={package.get('version')}" + ) artifact = package.get("artifact") if method in artifact_methods: allowed_artifacts = { @@ -1777,19 +1879,28 @@ def _acquire( ) with urlopen(request, timeout=60) as response, temporary.open("wb") as output: received = 0 - expected = artifact.get("size") + response_headers = getattr(response, "headers", {}) + expected = artifact.get("size") or response_headers.get( + "Content-Length" + ) + expected = int(expected) if expected else None + last_reported = 0 while True: chunk = response.read(1024 * 1024) if not chunk: break output.write(chunk) received += len(chunk) - if progress: + if progress and ( + received - last_reported >= 8 * 1024 * 1024 + or (expected and received >= expected) + ): total = f" / {_human_size(expected)}" if expected else "" - progress( - f"Downloading {package['name']}=={package['version']}: " - f"{_human_size(received)}{total}" - ) + progress(f"DOWNLOAD {_human_size(received)}{total}") + last_reported = received + if progress and received != last_reported: + total = f" / {_human_size(expected)}" if expected else "" + progress(f"DOWNLOAD {_human_size(received)}{total}") except OSError as exc: raise PloadError(f"cannot download planned artifact: {exc}") from exc if digest(temporary) != artifact["sha256"]: diff --git a/src/pload/help_content.py b/src/pload/help_content.py index 993d96a..0105242 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -60,8 +60,8 @@ `pload.toml` describes user intent. `plan` resolves from independent metadata when its managed lock is stale, then inventories resources without downloading -wheel bodies. Analysis does not overwrite saved choices; Save, Apply, or -`--no-ui` writes the plan. `apply` executes only that saved plan and never +wheel bodies. Analysis does not overwrite saved choices; Save, Apply, +`--no-ui`, or `[plan].mode = "auto"` writes the plan. `apply` executes only that saved plan and never changes route or uploads. """, ("new",): r""" @@ -390,13 +390,25 @@ wheel body. A current lock skips metadata resolution. Cache roots are indexed once and only matching-Python environments are inspected. The chooser reports measured local read throughput and network/repository RTT instead of fixed time -guesses. Analysis does not overwrite `.pload_plan.toml`; Save, Apply, and -`--no-ui` persist choices, while quit and `--json` do not. +guesses. Analysis does not overwrite `.pload_plan.toml`; Save, Apply, +`--no-ui`, and `[plan].mode = "auto"` persist choices, while quit and `--json` +do not. The chooser shows only requirements written in `pload.toml` and automatically assigns routes to their transitive dependencies. It supports previous/next, undo, reset, save, and save-and-apply; `--no-ui` selects the lowest-cost measured routes for automation. + +For configuration-driven automation, add: + +```toml +[plan] +mode = "auto" +``` + +This saves recommended routes without opening the chooser. Strict exceptions +can be declared under `[plan.packages.NAME]` with `method` and optional +`location`; an unavailable override fails planning instead of falling back. """ GUIDES[("lock",)] = r""" # Resolve the user declaration into an exact managed lock @@ -421,8 +433,11 @@ ``` Apply requires a current `.pload_lock.toml` and `.pload_plan.toml`, then installs -Python when needed, executes each selected route, verifies hashes, creates the -environment, installs packages and runs `pip check`. It never resolves, tries an +Python when needed. Before changing anything it prints the complete saved plan. +Execution then prints each `NAME==VERSION` as a heading, its selected source +URL/path/repository, real network byte progress, and the later install and +verification stages. It executes each selected route, verifies hashes, creates +the environment, installs packages and runs `pip check`. It never resolves, tries an alternative route, or uploads package bytes. A selected `index-exact` route downloads from the locked URL even if the same wheel is already cached; the verified wheel is then installed locally with package networking disabled. diff --git a/tests/test_cli.py b/tests/test_cli.py index d107b26..7510f08 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,7 @@ import pytest +import pload.cli as cli_module +import pload.declarative as declarative_module from pload.cli import build_parser, main, shell_script @@ -232,6 +234,39 @@ def test_no_arguments_show_welcome_and_simple_usage(capsys): assert output.index("Simple usage") < output.index("Global options") +def test_apply_prints_saved_plan_before_package_progress(tmp_path, monkeypatch, capsys): + events = [] + + class FakeManager: + def __init__(self, config): + pass + + def saved_plan(self, path): + events.append("loaded") + return {"name": "demo"} + + def apply(self, path, name, offline, progress): + events.append("apply") + progress("PACKAGE 1/1 numpy==2.0.2") + progress("SOURCE downloading from https://files.invalid/numpy.whl") + progress("DOWNLOAD 8.0 MiB / 16.0 MiB") + return tmp_path / "demo" + + monkeypatch.setattr(declarative_module, "DeclarativeEnvironmentManager", FakeManager) + monkeypatch.setattr( + cli_module, "print_declarative_plan", + lambda plan: events.append("printed-plan"), + ) + + assert main(["apply", str(tmp_path / "pload.toml")]) == 0 + + assert events == ["loaded", "printed-plan", "apply"] + output = capsys.readouterr().out + assert "numpy==2.0.2" in output + assert "downloading from https://files.invalid/numpy.whl" in output + assert "downloaded 8.0 MiB / 16.0 MiB" in output + + def test_new_without_options_uses_guided_creation(tmp_path, monkeypatch): captured = {} diff --git a/tests/test_declarative.py b/tests/test_declarative.py index f522ce1..8dfc154 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -98,6 +98,33 @@ def test_manifest_roundtrip(tmp_path): assert locked["package"] == data["package"] +def test_manifest_roundtrip_with_automatic_plan_and_package_override(tmp_path): + data = simple_manifest() + data["plan"] = { + "mode": "auto", + "packages": { + "numpy": {"method": "index-exact", "location": "default"}, + }, + } + path = tmp_path / "pload.toml" + write_configuration(path, data) + + _, loaded = load_manifest(path) + + assert loaded["plan"] == data["plan"] + + +def test_manifest_rejects_invalid_planning_mode_and_route(): + data = simple_manifest() + data["plan"] = {"mode": "always-guess"} + with pytest.raises(PloadError, match="plan.mode"): + validate_manifest(data) + + data["plan"] = {"mode": "auto", "packages": {"demo": {"method": "magic"}}} + with pytest.raises(PloadError, match="plan.packages.demo.method"): + validate_manifest(data) + + def test_lock_migrates_legacy_embedded_lock_without_resolving(tmp_path, monkeypatch): data = simple_manifest("exact") path = tmp_path / "pload.toml" @@ -205,6 +232,56 @@ def test_plan_uses_measurements_instead_of_fixed_fake_times(tmp_path, monkeypatc assert "estimated_seconds" not in route +def test_automatic_plan_honors_package_route_override(tmp_path): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + wheel = tiny_wheel(tmp_path / "fixture") + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + }) + data["package"][0]["artifact"] = [{ + "filename": wheel.name, "sha256": digest(wheel), + "tags": ["py3-none-any"], "repositories": [], + "url": "https://files.invalid/" + wheel.name, + }] + data["plan"] = { + "mode": "auto", + "packages": {"demo": {"method": "index-exact", "location": "default"}}, + } + manifest = tmp_path / "pload.toml" + write_configuration(manifest, data) + manager = DeclarativeEnvironmentManager(config) + manager.cache.mkdir(parents=True) + shutil.copyfile(wheel, manager.cache / wheel.name) + + result = manager.plan(manifest) + + assert result["planning_mode"] == "auto" + assert result["packages"][0]["selected"]["method"] == "index-exact" + assert result["packages"][0]["alternatives"] == [] + + +def test_configured_route_must_exist_in_automatic_plan(tmp_path): + data = simple_manifest() + data["plan"] = { + "mode": "auto", + "packages": {"demo": {"method": "repository", "location": "missing"}}, + } + path = tmp_path / "pload.toml" + write_configuration(path, data) + + package = DeclarativeEnvironmentManager( + ConfigManager(home=tmp_path / "home") + ).plan(path)["packages"][0] + + assert package["selected"] is None + assert package["rejections"][0]["reason"].endswith("repository:missing") + + def test_changed_python_rejects_incompatible_cached_wheel_before_apply(tmp_path, monkeypatch): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) @@ -610,8 +687,8 @@ def record_fetch(cls, *args, **kwargs): assert output == "42" assert target.apply(manifest) == restored assert digest(target.cache / artifact["filename"]) == artifact["sha256"] - assert any("Preparing pload-demo==1.0" in item for item in apply_progress) - assert any("Verified exact requirements" in item for item in apply_progress) + assert any("PACKAGE 1/1 pload-demo==1.0" in item for item in apply_progress) + assert any("DONE verified exact requirements" in item for item in apply_progress) def test_apply_rolls_back_a_new_environment_after_install_failure(tmp_path): @@ -776,7 +853,11 @@ def fake_urlopen(request, timeout): progress = [] restored = manager.apply(manifest, progress=progress.append) assert requested == [artifact_url] - assert any("Downloading pload-demo==1.0" in item for item in progress) + assert any(item == "PACKAGE 1/1 pload-demo==1.0" for item in progress) + assert any( + item.startswith(f"SOURCE downloading from {artifact_url}") for item in progress + ) + assert any(item.startswith("DOWNLOAD ") for item in progress) assert execute([ config.get_pip_command(restored)[0], "-c", "import pload_demo; print(pload_demo.answer)", From df7272db0e6d40d08583d42c4159ef9a1b21c370 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Mon, 28 Sep 2026 14:29:45 +0800 Subject: [PATCH 31/41] Render apply with compact package progress --- README.md | 5 +- docs/declarative-environments.md | 12 +-- src/pload/cli.py | 143 +++++++++++++++++++++++-------- src/pload/declarative.py | 43 +++++++--- src/pload/help_content.py | 6 +- tests/test_cli.py | 39 +++++++-- tests/test_declarative.py | 15 +++- 7 files changed, 197 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 1fb0ca4..da95949 100644 --- a/README.md +++ b/README.md @@ -100,8 +100,9 @@ real package sources. `apply` executes exactly the saved plan: it never re-resolves, changes route, falls back, or uploads packages. Set `[plan].mode = "auto"` in `pload.toml` to save recommended routes without the chooser; strict per-package exceptions live under `[plan.packages.NAME]`. -`apply` prints the saved plan first, then package headings, exact source -URLs/paths and real download byte progress. +`apply` prints the saved plan first, then compact Docker-style progress rows. +Each package is prefixed by its route, such as `[cache]`, `[index-exact]`, or +`[repository]`, alongside its source and real download byte progress. Back up a special or slow wheel only when you explicitly request it: diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index 1c5660b..3de1528 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -272,11 +272,13 @@ RTT. The analysis API does not create or overwrite `.pload_plan.toml`. Interactive Save/Apply, `--no-ui`, and `[plan].mode = "auto"` persist choices; quitting or `--json` does not. -`pload apply` first prints the complete persisted plan table. It then prints a -heading for each exact package, the selected source URL/path/repository, real -download byte progress for network artifacts, environment creation and -verification stages. This makes the saved decision and its execution visible in -one command without allowing apply to recalculate the plan. +`pload apply` first prints the complete persisted plan table. It then uses +compact Docker-style progress rows: every exact package starts with its selected +method in brackets (`[cache]`, `[index-exact]`, `[repository]`, and so on), +followed by the package, source, progress bar and real byte counts. Redirected +output uses stable plain-text rows without terminal control characters. This +makes the saved decision and its execution visible in one command without +allowing apply to recalculate the plan. An unpinned configuration with a missing or stale lock has one additional job: it must determine exact versions and the transitive graph before routes have a diff --git a/src/pload/cli.py b/src/pload/cli.py index 95a89a6..e6f20c2 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -7,6 +7,7 @@ from questionary import Choice from rich import box from rich.panel import Panel +from rich.progress import BarColumn, Progress, TextColumn from rich.table import Table from rich.text import Text @@ -531,30 +532,104 @@ def print_declarative_plan(plan): console.print("[dim]Acquisition choices were not saved.[/]") -def print_apply_progress(message, output=None): - """Render durable, package-oriented apply output instead of one opaque spinner.""" - console = output or ui.console() - kind, _, detail = str(message).partition(" ") - if kind == "PACKAGE": - counter, _, package = detail.partition(" ") - console.print() - line = Text("▶ ", style="bold cyan") - line.append(package, style="bold white") - line.append(f" {counter}", style="dim") - console.print(line) - elif kind == "SOURCE": - line = Text(" ↳ ", style="green") - line.append(detail, style="white") - console.print(line) - elif kind == "DOWNLOAD": - console.print(Text(f" downloaded {detail}", style="cyan")) - elif kind == "STAGE": - console.print() - console.print(Text(f"▶ {detail}", style="bold cyan")) - elif kind == "DONE": - ui.success(detail, output=console) - else: - console.print(message) +def _progress_size(value): + size = float(value or 0) + for unit in ("B", "KiB", "MiB", "GiB"): + if size < 1024 or unit == "GiB": + return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + + +class ApplyProgress: + """Docker-like persistent package rows with a plain-text CI fallback.""" + + def __init__(self, output=None): + self.console = output or ui.console() + self.interactive = self.console.is_terminal + self.tasks = {} + self.details = {} + self.progress = Progress( + TextColumn("{task.fields[tag]}", style="bold cyan", markup=False), + TextColumn("{task.fields[package]}", style="bold white", markup=False), + BarColumn(bar_width=18, complete_style="green", finished_style="green"), + TextColumn("{task.fields[amount]}", style="cyan", markup=False), + TextColumn("{task.fields[status]}", style="dim", markup=False), + console=self.console, + transient=False, + ) + + def __enter__(self): + if self.interactive: + self.progress.start() + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self.interactive: + self.progress.stop() + + def __call__(self, message): + parts = str(message).split("\t") + kind = parts[0].split(" ", 1)[0] + if kind == "PACKAGE": + _, counter, method, package, size = parts + total = int(size) or 1 + detail = { + "tag": f"[{method}]", + "package": package, + "counter": counter, + "method": method, + "total": total, + } + self.details[package] = detail + if self.interactive: + self.tasks[package] = self.progress.add_task( + package, total=total, tag=detail["tag"], package=package, + amount="", status="waiting", + ) + elif kind == "SOURCE": + _, package, status = parts + self.details[package]["status"] = status + if self.interactive: + self.progress.update(self.tasks[package], status=status) + else: + detail = self.details[package] + self.console.print( + Text(f"{detail['tag']:<24} {package} {status}") + ) + elif kind == "DOWNLOAD": + _, package, received, expected = parts + received, expected = int(received), int(expected) + detail = self.details[package] + total = expected or detail["total"] + amount = _progress_size(received) + if total > 1: + amount += f" / {_progress_size(total)}" + if self.interactive: + self.progress.update( + self.tasks[package], completed=received, total=total, + amount=amount, status="downloading", + ) + else: + self.console.print(Text(f"{detail['tag']:<24} {package} {amount}")) + elif kind == "PACKAGE_DONE": + _, package, status = parts + detail = self.details[package] + if self.interactive: + task = self.progress.tasks[self.tasks[package]] + self.progress.update( + self.tasks[package], completed=task.total, + amount=task.fields.get("amount", ""), status=status, + ) + elif "status" not in detail: + self.console.print(Text(f"{detail['tag']:<24} {package} {status}")) + elif kind == "STAGE": + _, _, detail = str(message).partition(" ") + self.console.print(Text(f"▶ {detail}", style="bold cyan")) + elif kind == "DONE": + _, _, detail = str(message).partition(" ") + ui.success(detail, output=self.console) + else: + self.console.print(message) def choose_declarative_routes(plan): @@ -976,21 +1051,21 @@ def run(argv=None): plan["saved"] = True print_declarative_plan(plan) if action == "apply": - restored = manager.apply( - args.file, offline=args.offline, - progress=lambda message: print_apply_progress( - message, chooser_console - ), - ) + with ApplyProgress(chooser_console) as apply_progress: + restored = manager.apply( + args.file, offline=args.offline, + progress=apply_progress, + ) chooser_console.print(restored) else: apply_console = ui.console() saved_plan = manager.saved_plan(args.file) print_declarative_plan(saved_plan) - restored = manager.apply( - args.file, args.name, args.offline, - progress=lambda message: print_apply_progress(message, apply_console), - ) + with ApplyProgress(apply_console) as apply_progress: + restored = manager.apply( + args.file, args.name, args.offline, + progress=apply_progress, + ) print(restored) return 0 diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 972afc5..5ef4907 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -1539,25 +1539,26 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): stage = Path(temporary) for index_number, package_plan in enumerate(plan["packages"], 1): selected = package_plan["selected"] + label = f"{package_plan['name']}=={package_plan['version']}" + artifact = selected.get("artifact") + size = artifact.get("size", 0) if artifact else 0 if progress: progress( - f"PACKAGE {index_number}/{len(plan['packages'])} " - f"{package_plan['name']}=={package_plan['version']}" + f"PACKAGE\t{index_number}/{len(plan['packages'])}\t" + f"{selected['method']}\t{label}\t{size}" ) if offline and selected["method"] in {"index-exact", "index-resolve"}: raise PloadError( f"offline apply forbids the selected network route for " f"{package_plan['name']}=={package_plan['version']}" ) - artifact = selected.get("artifact") if artifact: cached = self.cache / artifact["filename"] - label = f"{package_plan['name']}=={package_plan['version']}" method = selected["method"] if method == "cache": source = Path(selected["location"]) if progress: - progress(f"SOURCE using cache {source}") + progress(f"SOURCE\t{label}\tusing cache {source}") if not source.is_file() or digest(source) != artifact["sha256"]: raise PloadError( f"planned route failed for {label} (cache): " @@ -1566,7 +1567,6 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): ) install_artifact = source else: - size = artifact.get("size") size_note = f" ({_human_size(size)})" if size else "" if method == "index-exact": source_description = artifact.get("url") or data[ @@ -1583,7 +1583,8 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): action = "copying from" if progress: progress( - f"SOURCE {action} {source_description}{size_note}" + f"SOURCE\t{label}\t{action} " + f"{source_description}{size_note}" ) acquired_dir = stage / f"artifact-{index_number}" acquired_dir.mkdir() @@ -1609,11 +1610,13 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): install_artifact.resolve().as_uri() + "#sha256=" + artifact["sha256"] ) + if progress: + progress(f"PACKAGE_DONE\t{label}\tverified") else: if selected["method"] == "environment-copy": if progress: progress( - f"SOURCE copying installed package from " + f"SOURCE\t{label}\tcopying installed package from " f"{selected['location']}" ) environment_copies.append(package_plan) @@ -1621,7 +1624,7 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): source = data.get("sources", {}).get(selected["location"], {}) if progress: progress( - f"SOURCE downloading from " + f"SOURCE\t{label}\tdownloading from " f"{source.get('url', selected['location'])}" ) compatible.append(package_plan) @@ -1641,6 +1644,11 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): capture=False) for package_plan in environment_copies: self._copy_distribution(package_plan, env, stage) + if progress: + progress( + f"PACKAGE_DONE\t{package_plan['name']}==" + f"{package_plan['version']}\tcopied" + ) for package_plan in compatible: source_name = package_plan["selected"]["location"] source = data.get("sources", {}).get(source_name, {}) @@ -1652,6 +1660,11 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): execute(command + [ f"{package_plan['name']}=={package_plan['version']}" ], capture=False) + if progress: + progress( + f"PACKAGE_DONE\t{package_plan['name']}==" + f"{package_plan['version']}\tinstalled" + ) requirements = stage / "requirements.txt" requirements.write_text( "".join( @@ -1895,12 +1908,16 @@ def _acquire( received - last_reported >= 8 * 1024 * 1024 or (expected and received >= expected) ): - total = f" / {_human_size(expected)}" if expected else "" - progress(f"DOWNLOAD {_human_size(received)}{total}") + progress( + f"DOWNLOAD\t{package['name']}=={package['version']}\t" + f"{received}\t{expected or 0}" + ) last_reported = received if progress and received != last_reported: - total = f" / {_human_size(expected)}" if expected else "" - progress(f"DOWNLOAD {_human_size(received)}{total}") + progress( + f"DOWNLOAD\t{package['name']}=={package['version']}\t" + f"{received}\t{expected or 0}" + ) except OSError as exc: raise PloadError(f"cannot download planned artifact: {exc}") from exc if digest(temporary) != artifact["sha256"]: diff --git a/src/pload/help_content.py b/src/pload/help_content.py index 0105242..c6f18f5 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -434,9 +434,9 @@ Apply requires a current `.pload_lock.toml` and `.pload_plan.toml`, then installs Python when needed. Before changing anything it prints the complete saved plan. -Execution then prints each `NAME==VERSION` as a heading, its selected source -URL/path/repository, real network byte progress, and the later install and -verification stages. It executes each selected route, verifies hashes, creates +Execution then uses compact Docker-style progress rows. Each package starts with +its selected method in brackets, followed by `NAME==VERSION`, source, progress +bar and real network byte counts. It executes each selected route, verifies hashes, creates the environment, installs packages and runs `pip check`. It never resolves, tries an alternative route, or uploads package bytes. A selected `index-exact` route downloads from the locked URL even if the same wheel is already cached; the diff --git a/tests/test_cli.py b/tests/test_cli.py index 7510f08..650a52c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,7 @@ +import io + import pytest +from rich.console import Console import pload.cli as cli_module import pload.declarative as declarative_module @@ -247,9 +250,13 @@ def saved_plan(self, path): def apply(self, path, name, offline, progress): events.append("apply") - progress("PACKAGE 1/1 numpy==2.0.2") - progress("SOURCE downloading from https://files.invalid/numpy.whl") - progress("DOWNLOAD 8.0 MiB / 16.0 MiB") + progress("PACKAGE\t1/1\tindex-exact\tnumpy==2.0.2\t16777216") + progress( + "SOURCE\tnumpy==2.0.2\t" + "downloading from https://files.invalid/numpy.whl" + ) + progress("DOWNLOAD\tnumpy==2.0.2\t8388608\t16777216") + progress("PACKAGE_DONE\tnumpy==2.0.2\tverified") return tmp_path / "demo" monkeypatch.setattr(declarative_module, "DeclarativeEnvironmentManager", FakeManager) @@ -262,9 +269,31 @@ def apply(self, path, name, offline, progress): assert events == ["loaded", "printed-plan", "apply"] output = capsys.readouterr().out + normalized = " ".join(output.split()) + assert "[index-exact]" in output assert "numpy==2.0.2" in output - assert "downloading from https://files.invalid/numpy.whl" in output - assert "downloaded 8.0 MiB / 16.0 MiB" in output + assert "downloading from https://files.invalid/numpy.whl" in normalized + assert "8.0 MiB / 16.0 MiB" in output + + +def test_apply_progress_uses_one_docker_style_row_per_package(): + stream = io.StringIO() + console = Console(file=stream, force_terminal=True, width=160) + + with cli_module.ApplyProgress(console) as progress: + progress("PACKAGE\t1/1\tindex-exact\tnumpy==2.0.2\t16777216") + progress( + "SOURCE\tnumpy==2.0.2\t" + "downloading from https://files.invalid/numpy.whl" + ) + progress("DOWNLOAD\tnumpy==2.0.2\t8388608\t16777216") + progress("PACKAGE_DONE\tnumpy==2.0.2\tverified") + + rendered = stream.getvalue() + assert "[index-exact]" in rendered + assert "numpy==2.0.2" in rendered + assert "8.0 MiB / 16.0 MiB" in rendered + assert "verified" in rendered def test_new_without_options_uses_guided_creation(tmp_path, monkeypatch): diff --git a/tests/test_declarative.py b/tests/test_declarative.py index 8dfc154..4912d48 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -687,7 +687,10 @@ def record_fetch(cls, *args, **kwargs): assert output == "42" assert target.apply(manifest) == restored assert digest(target.cache / artifact["filename"]) == artifact["sha256"] - assert any("PACKAGE 1/1 pload-demo==1.0" in item for item in apply_progress) + assert any( + item.startswith("PACKAGE\t1/1\trepository\tpload-demo==1.0") + for item in apply_progress + ) assert any("DONE verified exact requirements" in item for item in apply_progress) @@ -853,11 +856,15 @@ def fake_urlopen(request, timeout): progress = [] restored = manager.apply(manifest, progress=progress.append) assert requested == [artifact_url] - assert any(item == "PACKAGE 1/1 pload-demo==1.0" for item in progress) assert any( - item.startswith(f"SOURCE downloading from {artifact_url}") for item in progress + item.startswith("PACKAGE\t1/1\tindex-exact\tpload-demo==1.0") + for item in progress + ) + assert any( + item.startswith(f"SOURCE\tpload-demo==1.0\tdownloading from {artifact_url}") + for item in progress ) - assert any(item.startswith("DOWNLOAD ") for item in progress) + assert any(item.startswith("DOWNLOAD\tpload-demo==1.0\t") for item in progress) assert execute([ config.get_pip_command(restored)[0], "-c", "import pload_demo; print(pload_demo.answer)", From e41c8a8bed863386024f0af219053b52ec909d74 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Mon, 28 Sep 2026 14:44:55 +0800 Subject: [PATCH 32/41] Use bounded block bars for apply progress --- src/pload/cli.py | 21 +++++++++++++++++++-- tests/test_cli.py | 6 +++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/pload/cli.py b/src/pload/cli.py index e6f20c2..49a1b1b 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -7,7 +7,7 @@ from questionary import Choice from rich import box from rich.panel import Panel -from rich.progress import BarColumn, Progress, TextColumn +from rich.progress import Progress, ProgressColumn, TextColumn from rich.table import Table from rich.text import Text @@ -540,6 +540,23 @@ def _progress_size(value): size /= 1024 +class DockerBarColumn(ProgressColumn): + """Fixed-width block bar with visible start and end boundaries.""" + + def __init__(self, width=18): + super().__init__() + self.width = width + + def render(self, task): + ratio = 0 if not task.total else min(max(task.completed / task.total, 0), 1) + complete = int(ratio * self.width) + bar = Text("[", style="dim") + bar.append("█" * complete, style="green") + bar.append("░" * (self.width - complete), style="dim") + bar.append("]", style="dim") + return bar + + class ApplyProgress: """Docker-like persistent package rows with a plain-text CI fallback.""" @@ -551,7 +568,7 @@ def __init__(self, output=None): self.progress = Progress( TextColumn("{task.fields[tag]}", style="bold cyan", markup=False), TextColumn("{task.fields[package]}", style="bold white", markup=False), - BarColumn(bar_width=18, complete_style="green", finished_style="green"), + DockerBarColumn(width=18), TextColumn("{task.fields[amount]}", style="cyan", markup=False), TextColumn("{task.fields[status]}", style="dim", markup=False), console=self.console, diff --git a/tests/test_cli.py b/tests/test_cli.py index 650a52c..44ceb02 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -278,7 +278,7 @@ def apply(self, path, name, offline, progress): def test_apply_progress_uses_one_docker_style_row_per_package(): stream = io.StringIO() - console = Console(file=stream, force_terminal=True, width=160) + console = Console(file=stream, force_terminal=True, color_system=None, width=160) with cli_module.ApplyProgress(console) as progress: progress("PACKAGE\t1/1\tindex-exact\tnumpy==2.0.2\t16777216") @@ -287,13 +287,13 @@ def test_apply_progress_uses_one_docker_style_row_per_package(): "downloading from https://files.invalid/numpy.whl" ) progress("DOWNLOAD\tnumpy==2.0.2\t8388608\t16777216") - progress("PACKAGE_DONE\tnumpy==2.0.2\tverified") rendered = stream.getvalue() assert "[index-exact]" in rendered assert "numpy==2.0.2" in rendered + assert "[█████████░░░░░░░░░]" in rendered assert "8.0 MiB / 16.0 MiB" in rendered - assert "verified" in rendered + assert "downloading" in rendered def test_new_without_options_uses_guided_creation(tmp_path, monkeypatch): From e8cff636b7f50e1c9b88942416878c1f54b16ac5 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Mon, 28 Sep 2026 14:55:40 +0800 Subject: [PATCH 33/41] Add project-local declarative apply --- CHANGELOG.md | 4 ++++ README.md | 8 +++++++- docs/declarative-environments.md | 17 +++++++++++++---- src/pload/cli.py | 29 +++++++++++++++++++++-------- src/pload/declarative.py | 12 ++++++++---- src/pload/help_content.py | 7 +++++++ tests/test_cli.py | 31 +++++++++++++++++++++++++++++-- tests/test_declarative.py | 26 ++++++++++++++++++++++++++ 8 files changed, 115 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4e5592..c1aa0d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,10 @@ Pre-release: `1.2.0a1` instead of appearing frozen while network or SSH work is captured. - Refuse `apply` when the lock is absent or stale instead of resolving and downloading implicitly. +- Add `pload apply --local` / `-l` to reproduce a saved plan into `.venv` + beside its configuration file, matching `pload init` project layout. +- Render apply transfers as bounded, spaced solid-block progress bars with + blank unfinished cells and the selected acquisition method in brackets. ### Removed diff --git a/README.md b/README.md index da95949..dca6f3b 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,7 @@ pload manages `.pload_lock.toml` (exact versions and artifacts) and pload describe v2 -o pload.toml pload plan pload.toml pload apply pload.toml +pload apply --local # reproduce into .venv beside pload.toml ``` `plan` fetches only index pages and independent package metadata—never wheel @@ -102,7 +103,12 @@ it never re-resolves, changes route, falls back, or uploads packages. Set chooser; strict per-package exceptions live under `[plan.packages.NAME]`. `apply` prints the saved plan first, then compact Docker-style progress rows. Each package is prefixed by its route, such as `[cache]`, `[index-exact]`, or -`[repository]`, alongside its source and real download byte progress. +`[repository]`, alongside its source and real download byte progress. The bar +uses bounded, spaced solid blocks; unfinished cells remain blank. + +Use `pload apply --local` (or `-l`) for a project-local environment. It creates +`.venv` beside the selected `pload.toml`, matching the layout produced by +`pload init` while executing the saved declarative plan exactly. Back up a special or slow wheel only when you explicitly request it: diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index 3de1528..da7cb9a 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -23,6 +23,7 @@ The normal workflow is: ```console pload plan pload.toml # resolve metadata, inventory and choose routes pload apply pload.toml # execute only the saved plan +pload apply --local # execute into .venv beside pload.toml ``` By default, `plan` opens a route chooser for only the direct requirements the @@ -275,10 +276,18 @@ quitting or `--json` does not. `pload apply` first prints the complete persisted plan table. It then uses compact Docker-style progress rows: every exact package starts with its selected method in brackets (`[cache]`, `[index-exact]`, `[repository]`, and so on), -followed by the package, source, progress bar and real byte counts. Redirected -output uses stable plain-text rows without terminal control characters. This -makes the saved decision and its execution visible in one command without -allowing apply to recalculate the plan. +followed by the package, source, a bounded spaced-block progress bar and real +byte counts. Completed cells are solid blocks separated by narrow terminal-cell +gaps; unfinished cells remain blank. Redirected output uses stable plain-text +rows without terminal control characters. This makes the saved decision and +its execution visible in one command without allowing apply to recalculate the +plan. + +By default apply creates a named managed environment. `pload apply --local` +instead creates `.venv` beside the configuration file, matching `pload init`'s +project-local layout while retaining the saved declarative plan. Placement is a +CLI concern so the portable environment description remains independent of a +particular checkout path. An unpinned configuration with a missing or stale lock has one additional job: it must determine exact versions and the transitive graph before routes have a diff --git a/src/pload/cli.py b/src/pload/cli.py index 49a1b1b..a61c0a9 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -8,7 +8,7 @@ from rich import box from rich.panel import Panel from rich.progress import Progress, ProgressColumn, TextColumn -from rich.table import Table +from rich.table import Column, Table from rich.text import Text from pload import __version__, ui @@ -327,7 +327,14 @@ def build_parser(): ) apply.add_argument("file", nargs="?", default="pload.toml", help="environment configuration (default: pload.toml)") - apply.add_argument("--name", "-n", help="override the materialized environment name") + apply_target = apply.add_mutually_exclusive_group() + apply_target.add_argument( + "--name", "-n", help="override the managed environment name" + ) + apply_target.add_argument( + "--local", "-l", action="store_true", + help="create .venv beside the configuration file", + ) apply.add_argument("--offline", "-o", action="store_true", help="forbid internet access; configured local/SSH repositories remain usable") repo = subparsers.add_parser("repo", help="manage resource providers", @@ -541,18 +548,20 @@ def _progress_size(value): class DockerBarColumn(ProgressColumn): - """Fixed-width block bar with visible start and end boundaries.""" + """Fixed-width, spaced block bar with visible start and end boundaries.""" - def __init__(self, width=18): - super().__init__() + def __init__(self, width=10): + super().__init__(table_column=Column(width=width * 2 + 1, no_wrap=True)) self.width = width def render(self, task): ratio = 0 if not task.total else min(max(task.completed / task.total, 0), 1) complete = int(ratio * self.width) bar = Text("[", style="dim") - bar.append("█" * complete, style="green") - bar.append("░" * (self.width - complete), style="dim") + for cell in range(self.width): + if cell: + bar.append(" ") + bar.append("█" if cell < complete else " ", style="green") bar.append("]", style="dim") return bar @@ -568,7 +577,7 @@ def __init__(self, output=None): self.progress = Progress( TextColumn("{task.fields[tag]}", style="bold cyan", markup=False), TextColumn("{task.fields[package]}", style="bold white", markup=False), - DockerBarColumn(width=18), + DockerBarColumn(width=10), TextColumn("{task.fields[amount]}", style="cyan", markup=False), TextColumn("{task.fields[status]}", style="dim", markup=False), console=self.console, @@ -1078,9 +1087,13 @@ def run(argv=None): apply_console = ui.console() saved_plan = manager.saved_plan(args.file) print_declarative_plan(saved_plan) + target = None + if args.local: + target = Path(args.file).expanduser().resolve().parent / ".venv" with ApplyProgress(apply_console) as apply_progress: restored = manager.apply( args.file, args.name, args.offline, + target=target, progress=apply_progress, ) print(restored) diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 5ef4907..5e4b566 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -1500,7 +1500,9 @@ def _load_saved_execution_plan(self, manifest_path): raise PloadError("no valid reproduction route for: " + ", ".join(details)) return path, data, plan - def apply(self, manifest_path, name=None, offline=False, progress=None): + def apply( + self, manifest_path, name=None, offline=False, progress=None, target=None, + ): path, data, plan = self._load_saved_execution_plan(manifest_path) python_method = plan["python"].get("method") if offline and python_method == "install-python": @@ -1521,9 +1523,11 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): raise PloadError(f"unsupported Python acquisition method: {python_method}") self._check_runtime(interpreter, data["environment"]) target_name = name or data["name"] + target_path = Path(target).expanduser().resolve() if target else None + target_reference = str(target_path) if target_path else target_name expected_state = manifest_digest(data) try: - existing = VenvManager(self.config).resolve_existing(target_name) + existing = VenvManager(self.config).resolve_existing(target_reference) except PloadError: existing = None if existing: @@ -1632,7 +1636,7 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): if progress: progress(f"STAGE creating environment {target_name}") env = venvs.create_venv( - version=str(interpreter), name=target_name, + version=str(interpreter), name=target_name, target=target_path, description=f"Applied from {path.name}", ) try: @@ -1682,7 +1686,7 @@ def apply(self, manifest_path, name=None, offline=False, progress=None): ) return env except (OSError, PloadError): - venvs.remove_venv(target_name) + venvs.remove_venv(str(env)) raise def save_plan(self, manifest_path, plan): diff --git a/src/pload/help_content.py b/src/pload/help_content.py index c6f18f5..8e56d82 100644 --- a/src/pload/help_content.py +++ b/src/pload/help_content.py @@ -429,6 +429,7 @@ ```console pload apply pload apply project.pload.toml -n project-copy +pload apply --local pload apply --offline ``` @@ -443,6 +444,12 @@ verified wheel is then installed locally with package networking disabled. Reapplying the same configuration is idempotent; an existing environment with different state is never silently overwritten. + +`--local` (or `-l`) creates `.venv` beside the selected configuration file, +giving declarative projects the same local-environment layout as `pload init`. +The location follows the configuration file rather than the caller's current +working directory, so `pload apply path/to/pload.toml --local` reliably targets +`path/to/.venv`. """ GUIDES[("repo",)] = r""" # Configure reusable artifact providers diff --git a/tests/test_cli.py b/tests/test_cli.py index 44ceb02..ecf55e8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -248,7 +248,7 @@ def saved_plan(self, path): events.append("loaded") return {"name": "demo"} - def apply(self, path, name, offline, progress): + def apply(self, path, name, offline, progress, target=None): events.append("apply") progress("PACKAGE\t1/1\tindex-exact\tnumpy==2.0.2\t16777216") progress( @@ -291,11 +291,38 @@ def test_apply_progress_uses_one_docker_style_row_per_package(): rendered = stream.getvalue() assert "[index-exact]" in rendered assert "numpy==2.0.2" in rendered - assert "[█████████░░░░░░░░░]" in rendered + assert "[█ █ █ █ █" in rendered + assert "░" not in rendered + progress_line = next(line for line in rendered.splitlines() if "8.0 MiB" in line) + assert "]" in progress_line assert "8.0 MiB / 16.0 MiB" in rendered assert "downloading" in rendered +def test_apply_local_targets_venv_beside_manifest(tmp_path, monkeypatch): + captured = {} + manifest = tmp_path / "project" / "pload.toml" + + class FakeManager: + def __init__(self, config): + pass + + def saved_plan(self, path): + return {"packages": []} + + def apply(self, path, name, offline, progress, target=None): + captured["target"] = target + return target + + monkeypatch.setattr( + declarative_module, "DeclarativeEnvironmentManager", FakeManager + ) + monkeypatch.setattr(cli_module, "print_declarative_plan", lambda plan: None) + + assert main(["apply", str(manifest), "--local"]) == 0 + assert captured["target"] == manifest.parent / ".venv" + + def test_new_without_options_uses_guided_creation(tmp_path, monkeypatch): captured = {} diff --git a/tests/test_declarative.py b/tests/test_declarative.py index 4912d48..df00e52 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -379,6 +379,32 @@ def reject_package_network(command, *args, **kwargs): assert output == "42" +def test_apply_can_materialize_project_local_environment(tmp_path): + config = ConfigManager(home=tmp_path / "home") + config.get_python_path = lambda version=None: Path(sys.executable) + data = simple_manifest("exact") + data["environment"].update({ + "python": platform.python_version(), + "implementation": sys.implementation.name, + "system": platform.system(), + "machine": platform.machine(), + "dependencies": [], + }) + data["package"] = [] + project = tmp_path / "project" + project.mkdir() + manifest = project / "pload.toml" + write_configuration(manifest, data, with_lock=False) + manager = DeclarativeEnvironmentManager(config) + plan_and_save(manager, manifest) + + restored = manager.apply(manifest, target=project / ".venv") + + assert restored == project / ".venv" + assert (restored / "pyvenv.cfg").is_file() + assert manager.apply(manifest, target=project / ".venv") == restored + + def test_plan_discovers_and_apply_uses_exact_external_cache(tmp_path, monkeypatch): external_cache = tmp_path / "pip-cache" wheel = tiny_wheel(external_cache) From 2f39c374eb38fccdb9234b440355a11809a9eb04 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Mon, 28 Sep 2026 15:08:01 +0800 Subject: [PATCH 34/41] Tighten apply progress block spacing --- CHANGELOG.md | 4 ++-- README.md | 3 ++- docs/declarative-environments.md | 6 +++--- src/pload/cli.py | 12 ++++++------ tests/test_cli.py | 2 +- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1aa0d6..c695a5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,8 +59,8 @@ Pre-release: `1.2.0a1` downloading implicitly. - Add `pload apply --local` / `-l` to reproduce a saved plan into `.venv` beside its configuration file, matching `pload init` project layout. -- Render apply transfers as bounded, spaced solid-block progress bars with - blank unfinished cells and the selected acquisition method in brackets. +- Render apply transfers as bounded square-cell progress bars with subtle glyph + separation, blank unfinished cells and the acquisition method in brackets. ### Removed diff --git a/README.md b/README.md index dca6f3b..8df585e 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,8 @@ chooser; strict per-package exceptions live under `[plan.packages.NAME]`. `apply` prints the saved plan first, then compact Docker-style progress rows. Each package is prefixed by its route, such as `[cache]`, `[index-exact]`, or `[repository]`, alongside its source and real download byte progress. The bar -uses bounded, spaced solid blocks; unfinished cells remain blank. +uses bounded square cells with subtle visual separation; unfinished cells remain +blank. Use `pload apply --local` (or `-l`) for a project-local environment. It creates `.venv` beside the selected `pload.toml`, matching the layout produced by diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index da7cb9a..f79d636 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -276,9 +276,9 @@ quitting or `--json` does not. `pload apply` first prints the complete persisted plan table. It then uses compact Docker-style progress rows: every exact package starts with its selected method in brackets (`[cache]`, `[index-exact]`, `[repository]`, and so on), -followed by the package, source, a bounded spaced-block progress bar and real -byte counts. Completed cells are solid blocks separated by narrow terminal-cell -gaps; unfinished cells remain blank. Redirected output uses stable plain-text +followed by the package, source, a bounded square-cell progress bar and real +byte counts. Completed cells use a square glyph with subtle built-in separation; +unfinished cells remain blank. Redirected output uses stable plain-text rows without terminal control characters. This makes the saved decision and its execution visible in one command without allowing apply to recalculate the plan. diff --git a/src/pload/cli.py b/src/pload/cli.py index a61c0a9..f04f2c9 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -548,20 +548,20 @@ def _progress_size(value): class DockerBarColumn(ProgressColumn): - """Fixed-width, spaced block bar with visible start and end boundaries.""" + """Fixed-width square bar with visible start and end boundaries.""" def __init__(self, width=10): - super().__init__(table_column=Column(width=width * 2 + 1, no_wrap=True)) + super().__init__(table_column=Column(width=width + 2, no_wrap=True)) self.width = width def render(self, task): ratio = 0 if not task.total else min(max(task.completed / task.total, 0), 1) complete = int(ratio * self.width) bar = Text("[", style="dim") - for cell in range(self.width): - if cell: - bar.append(" ") - bar.append("█" if cell < complete else " ", style="green") + # U+25A0 has a small amount of built-in side bearing, so adjacent + # completed cells remain distinct without inserting full terminal cells. + bar.append("■" * complete, style="green") + bar.append(" " * (self.width - complete)) bar.append("]", style="dim") return bar diff --git a/tests/test_cli.py b/tests/test_cli.py index ecf55e8..91d460a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -291,7 +291,7 @@ def test_apply_progress_uses_one_docker_style_row_per_package(): rendered = stream.getvalue() assert "[index-exact]" in rendered assert "numpy==2.0.2" in rendered - assert "[█ █ █ █ █" in rendered + assert "[■■■■■ ]" in rendered assert "░" not in rendered progress_line = next(line for line in rendered.splitlines() if "8.0 MiB" in line) assert "]" in progress_line From 5048339f0a35ecc4236b5471e6a6b18a763a9897 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Mon, 28 Sep 2026 15:15:02 +0800 Subject: [PATCH 35/41] Use adaptive Rich apply progress bars --- CHANGELOG.md | 4 ++-- README.md | 4 ++-- docs/declarative-environments.md | 7 ++++--- src/pload/cli.py | 35 +++++++++++--------------------- tests/test_cli.py | 8 ++++---- 5 files changed, 24 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c695a5b..47734fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,8 +59,8 @@ Pre-release: `1.2.0a1` downloading implicitly. - Add `pload apply --local` / `-l` to reproduce a saved plan into `.venv` beside its configuration file, matching `pload init` project layout. -- Render apply transfers as bounded square-cell progress bars with subtle glyph - separation, blank unfinished cells and the acquisition method in brackets. +- Render apply transfers with Rich's adaptive-width standard progress bar, + explicit boundaries and the acquisition method in brackets. ### Removed diff --git a/README.md b/README.md index 8df585e..4930413 100644 --- a/README.md +++ b/README.md @@ -104,8 +104,8 @@ chooser; strict per-package exceptions live under `[plan.packages.NAME]`. `apply` prints the saved plan first, then compact Docker-style progress rows. Each package is prefixed by its route, such as `[cache]`, `[index-exact]`, or `[repository]`, alongside its source and real download byte progress. The bar -uses bounded square cells with subtle visual separation; unfinished cells remain -blank. +uses Rich's standard adaptive-width bar with explicit start and end boundaries, +so it grows naturally on wider terminals without another progress dependency. Use `pload apply --local` (or `-l`) for a project-local environment. It creates `.venv` beside the selected `pload.toml`, matching the layout produced by diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index f79d636..e7b55c1 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -276,9 +276,10 @@ quitting or `--json` does not. `pload apply` first prints the complete persisted plan table. It then uses compact Docker-style progress rows: every exact package starts with its selected method in brackets (`[cache]`, `[index-exact]`, `[repository]`, and so on), -followed by the package, source, a bounded square-cell progress bar and real -byte counts. Completed cells use a square glyph with subtle built-in separation; -unfinished cells remain blank. Redirected output uses stable plain-text +followed by the package, source, Rich's bounded adaptive-width progress bar and +real byte counts. The bar consumes the terminal width still available after the +useful text columns, so wide terminals get a substantially longer bar without +breaking narrow ones. Redirected output uses stable plain-text rows without terminal control characters. This makes the saved decision and its execution visible in one command without allowing apply to recalculate the plan. diff --git a/src/pload/cli.py b/src/pload/cli.py index f04f2c9..ee31bdf 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -7,8 +7,8 @@ from questionary import Choice from rich import box from rich.panel import Panel -from rich.progress import Progress, ProgressColumn, TextColumn -from rich.table import Column, Table +from rich.progress import BarColumn, Progress, TextColumn +from rich.table import Table from rich.text import Text from pload import __version__, ui @@ -547,27 +547,8 @@ def _progress_size(value): size /= 1024 -class DockerBarColumn(ProgressColumn): - """Fixed-width square bar with visible start and end boundaries.""" - - def __init__(self, width=10): - super().__init__(table_column=Column(width=width + 2, no_wrap=True)) - self.width = width - - def render(self, task): - ratio = 0 if not task.total else min(max(task.completed / task.total, 0), 1) - complete = int(ratio * self.width) - bar = Text("[", style="dim") - # U+25A0 has a small amount of built-in side bearing, so adjacent - # completed cells remain distinct without inserting full terminal cells. - bar.append("■" * complete, style="green") - bar.append(" " * (self.width - complete)) - bar.append("]", style="dim") - return bar - - class ApplyProgress: - """Docker-like persistent package rows with a plain-text CI fallback.""" + """Persistent Rich progress rows with a plain-text CI fallback.""" def __init__(self, output=None): self.console = output or ui.console() @@ -577,11 +558,19 @@ def __init__(self, output=None): self.progress = Progress( TextColumn("{task.fields[tag]}", style="bold cyan", markup=False), TextColumn("{task.fields[package]}", style="bold white", markup=False), - DockerBarColumn(width=10), + TextColumn("[", style="dim", markup=False), + BarColumn( + bar_width=None, + style="grey23", + complete_style="green", + finished_style="green", + ), + TextColumn("]", style="dim", markup=False), TextColumn("{task.fields[amount]}", style="cyan", markup=False), TextColumn("{task.fields[status]}", style="dim", markup=False), console=self.console, transient=False, + expand=True, ) def __enter__(self): diff --git a/tests/test_cli.py b/tests/test_cli.py index 91d460a..225e996 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -276,7 +276,7 @@ def apply(self, path, name, offline, progress, target=None): assert "8.0 MiB / 16.0 MiB" in output -def test_apply_progress_uses_one_docker_style_row_per_package(): +def test_apply_progress_uses_one_adaptive_rich_row_per_package(): stream = io.StringIO() console = Console(file=stream, force_terminal=True, color_system=None, width=160) @@ -291,10 +291,10 @@ def test_apply_progress_uses_one_docker_style_row_per_package(): rendered = stream.getvalue() assert "[index-exact]" in rendered assert "numpy==2.0.2" in rendered - assert "[■■■■■ ]" in rendered - assert "░" not in rendered progress_line = next(line for line in rendered.splitlines() if "8.0 MiB" in line) - assert "]" in progress_line + assert "[" in progress_line and "]" in progress_line + assert "━" in progress_line + assert "░" not in rendered assert "8.0 MiB / 16.0 MiB" in rendered assert "downloading" in rendered From 764187ad443410488625341501dac9619b48c72b Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Mon, 28 Sep 2026 15:18:36 +0800 Subject: [PATCH 36/41] Stream apply download progress smoothly --- CHANGELOG.md | 2 ++ README.md | 2 ++ docs/declarative-environments.md | 5 +++- src/pload/cli.py | 10 +++++++- src/pload/declarative.py | 13 ++++++---- tests/test_declarative.py | 44 ++++++++++++++++++++++++++++++++ 6 files changed, 69 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47734fe..b30a72f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,8 @@ Pre-release: `1.2.0a1` beside its configuration file, matching `pload init` project layout. - Render apply transfers with Rich's adaptive-width standard progress bar, explicit boundaries and the acquisition method in brackets. +- Stream direct downloads in 16 KiB chunks and report measured byte progress at + up to 30 updates per second while coalescing non-interactive logs. ### Removed diff --git a/README.md b/README.md index 4930413..78a4ecf 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,8 @@ Each package is prefixed by its route, such as `[cache]`, `[index-exact]`, or `[repository]`, alongside its source and real download byte progress. The bar uses Rich's standard adaptive-width bar with explicit start and end boundaries, so it grows naturally on wider terminals without another progress dependency. +Network transfers report real downloaded bytes from 16 KiB reads at up to 30 +updates per second; redirected logs remain coalesced instead of flooding CI. Use `pload apply --local` (or `-l`) for a project-local environment. It creates `.venv` beside the selected `pload.toml`, matching the layout produced by diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index e7b55c1..65dc5c9 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -279,7 +279,10 @@ method in brackets (`[cache]`, `[index-exact]`, `[repository]`, and so on), followed by the package, source, Rich's bounded adaptive-width progress bar and real byte counts. The bar consumes the terminal width still available after the useful text columns, so wide terminals get a substantially longer bar without -breaking narrow ones. Redirected output uses stable plain-text +breaking narrow ones. Direct artifact downloads read in 16 KiB chunks and +publish real byte counts at up to 30 updates per second; this keeps interactive +motion fluid without inventing intermediate progress. Redirected output +coalesces those updates into stable plain-text rows without terminal control characters. This makes the saved decision and its execution visible in one command without allowing apply to recalculate the plan. diff --git a/src/pload/cli.py b/src/pload/cli.py index ee31bdf..8d138e4 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -571,6 +571,7 @@ def __init__(self, output=None): console=self.console, transient=False, expand=True, + refresh_per_second=30, ) def __enter__(self): @@ -594,6 +595,7 @@ def __call__(self, message): "counter": counter, "method": method, "total": total, + "last_plain_received": 0, } self.details[package] = detail if self.interactive: @@ -625,7 +627,13 @@ def __call__(self, message): amount=amount, status="downloading", ) else: - self.console.print(Text(f"{detail['tag']:<24} {package} {amount}")) + final = expected and received >= expected + if (final or received - detail["last_plain_received"] + >= 8 * 1024 * 1024): + self.console.print(Text( + f"{detail['tag']:<24} {package} {amount}" + )) + detail["last_plain_received"] = received elif kind == "PACKAGE_DONE": _, package, status = parts detail = self.details[package] diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 5e4b566..9a7dbc9 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -46,6 +46,8 @@ PIN = re.compile(r"([A-Za-z0-9][A-Za-z0-9_.-]*)==([A-Za-z0-9][A-Za-z0-9_.+!-]*)") WHEEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+!-]*\.whl") SHA256 = re.compile(r"[0-9a-f]{64}") +DOWNLOAD_CHUNK_SIZE = 16 * 1024 +DOWNLOAD_PROGRESS_INTERVAL = 1 / 30 def _human_size(value): @@ -1902,21 +1904,22 @@ def _acquire( ) expected = int(expected) if expected else None last_reported = 0 + last_reported_at = time.monotonic() while True: - chunk = response.read(1024 * 1024) + chunk = response.read(DOWNLOAD_CHUNK_SIZE) if not chunk: break output.write(chunk) received += len(chunk) - if progress and ( - received - last_reported >= 8 * 1024 * 1024 - or (expected and received >= expected) - ): + now = time.monotonic() + if progress and (now - last_reported_at + >= DOWNLOAD_PROGRESS_INTERVAL): progress( f"DOWNLOAD\t{package['name']}=={package['version']}\t" f"{received}\t{expected or 0}" ) last_reported = received + last_reported_at = now if progress and received != last_reported: progress( f"DOWNLOAD\t{package['name']}=={package['version']}\t" diff --git a/tests/test_declarative.py b/tests/test_declarative.py index df00e52..b4c7e18 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -897,6 +897,50 @@ def fake_urlopen(request, timeout): ]) == "42" +def test_exact_download_reports_frequent_real_byte_progress(tmp_path, monkeypatch): + payload = b"x" * (declarative_module.DOWNLOAD_CHUNK_SIZE * 4) + source = tmp_path / "source.whl" + source.write_bytes(payload) + response = io.BytesIO(payload) + response.headers = {"Content-Length": str(len(payload))} + monkeypatch.setattr( + declarative_module, "urlopen", lambda request, timeout: response + ) + ticks = iter((0, 0.04, 0.08, 0.12, 0.16)) + monkeypatch.setattr(declarative_module.time, "monotonic", lambda: next(ticks)) + config = ConfigManager(home=tmp_path / "home") + manager = DeclarativeEnvironmentManager(config) + stage = tmp_path / "stage" + stage.mkdir() + destination = tmp_path / "output" / "demo.whl" + destination.parent.mkdir() + artifact = { + "filename": "demo.whl", + "sha256": digest(source), + "size": len(payload), + "url": "https://files.example.invalid/demo.whl", + } + package = {"name": "demo", "version": "1.0"} + selected = {"method": "index-exact", "location": "default", "artifact": artifact} + events = [] + + manager._acquire( + tmp_path / "pload.toml", + {"sources": {"default": {"url": "https://example.invalid/simple"}}}, + package, + selected, + destination, + stage, + Path(sys.executable), + progress=events.append, + ) + + downloads = [event for event in events if event.startswith("DOWNLOAD\t")] + assert len(downloads) >= 3 + assert downloads[-1].split("\t")[2:] == [str(len(payload)), str(len(payload))] + assert destination.read_bytes() == payload + + def test_offline_apply_rejects_saved_network_route(tmp_path, monkeypatch): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) From 2234264f972794f5d42600a4015c597066fcf770 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Mon, 28 Sep 2026 15:27:12 +0800 Subject: [PATCH 37/41] Stream repository transfer progress --- CHANGELOG.md | 2 + README.md | 5 +- docs/declarative-environments.md | 6 +- src/pload/declarative.py | 156 +++++++++++++++++++++++++++++-- tests/test_declarative.py | 80 ++++++++++++++++ 5 files changed, 236 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b30a72f..b6feeda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,8 @@ Pre-release: `1.2.0a1` explicit boundaries and the acquisition method in brackets. - Stream direct downloads in 16 KiB chunks and report measured byte progress at up to 30 updates per second while coalescing non-interactive logs. +- Stream local repository copies and SSH object responses so repository routes + report real byte progress instead of completing at once. ### Removed diff --git a/README.md b/README.md index 78a4ecf..94b07a8 100644 --- a/README.md +++ b/README.md @@ -106,8 +106,9 @@ Each package is prefixed by its route, such as `[cache]`, `[index-exact]`, or `[repository]`, alongside its source and real download byte progress. The bar uses Rich's standard adaptive-width bar with explicit start and end boundaries, so it grows naturally on wider terminals without another progress dependency. -Network transfers report real downloaded bytes from 16 KiB reads at up to 30 -updates per second; redirected logs remain coalesced instead of flooding CI. +Index and repository transfers report real received bytes at up to 30 updates +per second. SSH repository downloads stream the remote object directly; +redirected logs remain coalesced instead of flooding CI. Use `pload apply --local` (or `-l`) for a project-local environment. It creates `.venv` beside the selected `pload.toml`, matching the layout produced by diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index 65dc5c9..aac2785 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -281,8 +281,10 @@ real byte counts. The bar consumes the terminal width still available after the useful text columns, so wide terminals get a substantially longer bar without breaking narrow ones. Direct artifact downloads read in 16 KiB chunks and publish real byte counts at up to 30 updates per second; this keeps interactive -motion fluid without inventing intermediate progress. Redirected output -coalesces those updates into stable plain-text +motion fluid without inventing intermediate progress. Local repositories stream +their copied bytes, while SSH repositories stream the remote object into the +local destination at the same refresh cadence. Redirected output coalesces those +updates into stable plain-text rows without terminal control characters. This makes the saved decision and its execution visible in one command without allowing apply to recalculate the plan. diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 9a7dbc9..9a102fb 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -8,6 +8,7 @@ import shlex import shutil import socket +import subprocess import sys import tempfile import time @@ -596,13 +597,94 @@ def publish_package_record(cls, spec, package, version, artifact, base): ], timeout=75) @classmethod - def fetch(cls, spec, checksum, destination, base): - cls.fetch_many(spec, [(checksum, destination)], base) + def fetch( + cls, spec, checksum, destination, base, progress=None, expected_size=0, + ): + if spec["kind"] == "ssh": + cls._fetch_ssh_stream( + spec, checksum, Path(destination), progress, expected_size, + ) + return + callback = None + if progress: + callback = lambda _checksum, received, total: progress(received, total) + cls.fetch_many( + spec, [(checksum, destination)], base, + progress=callback, sizes={checksum: expected_size}, + ) @classmethod - def fetch_many(cls, spec, artifacts, base): + def _fetch_ssh_stream( + cls, spec, checksum, destination, progress=None, expected_size=0, + ): + """Stream one SSH object so received bytes are visible immediately.""" + host, root = RepositoryManager.ssh_location(spec["location"]) + remote = root + "/objects/" + checksum + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(destination.name + "." + uuid.uuid4().hex) + command = [ + "ssh", *cls.SSH_OPTIONS, host, + f"cat -- {shlex.quote(remote)}", + ] + try: + process = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + except OSError as exc: + raise PloadError(f"cannot run ssh: {exc}") from exc + received = 0 + started = time.monotonic() + last_reported_at = started + try: + with temporary.open("wb") as output: + while True: + reader = ( + process.stdout.read1 + if hasattr(process.stdout, "read1") + else process.stdout.read + ) + chunk = reader(DOWNLOAD_CHUNK_SIZE) + if not chunk: + break + output.write(chunk) + received += len(chunk) + now = time.monotonic() + if now - started > 300: + process.kill() + raise PloadError("ssh artifact download timed out after 300 seconds") + if progress and (now - last_reported_at + >= DOWNLOAD_PROGRESS_INTERVAL): + progress(received, expected_size) + last_reported_at = now + remaining = max(0.1, 300 - (time.monotonic() - started)) + try: + returncode = process.wait(timeout=remaining) + except subprocess.TimeoutExpired as exc: + process.kill() + process.wait() + raise PloadError( + "ssh artifact download timed out after 300 seconds" + ) from exc + stderr = process.stderr.read().decode(errors="replace") if process.stderr else "" + if returncode: + raise PloadError(f"ssh failed: {stderr.strip() or returncode}") + if progress: + progress(received, expected_size or received) + if digest(temporary) != checksum: + raise PloadError("retrieved artifact checksum mismatch") + temporary.replace(destination) + except Exception: + temporary.unlink(missing_ok=True) + if process.poll() is None: + process.kill() + process.wait() + raise + + @classmethod + def fetch_many(cls, spec, artifacts, base, progress=None, sizes=None): """Fetch multiple content-addressed objects through one SFTP session.""" artifacts = [(checksum, Path(destination)) for checksum, destination in artifacts] + sizes = sizes or {} if not artifacts: return for _, destination in artifacts: @@ -610,10 +692,27 @@ def fetch_many(cls, spec, artifacts, base): if spec["kind"] == "local": root = cls._local_root(spec, base) / "objects" for checksum, destination in artifacts: + source = root / checksum temporary = destination.with_name( destination.name + "." + uuid.uuid4().hex ) - shutil.copyfile(root / checksum, temporary) + expected = sizes.get(checksum) or source.stat().st_size + received = 0 + last_reported_at = time.monotonic() + with source.open("rb") as input_stream, temporary.open("wb") as output: + while True: + chunk = input_stream.read(DOWNLOAD_CHUNK_SIZE) + if not chunk: + break + output.write(chunk) + received += len(chunk) + now = time.monotonic() + if progress and (now - last_reported_at + >= DOWNLOAD_PROGRESS_INTERVAL): + progress(checksum, received, expected) + last_reported_at = now + if progress: + progress(checksum, received, expected) if digest(temporary) != checksum: temporary.unlink(missing_ok=True) raise PloadError("retrieved artifact checksum mismatch") @@ -622,19 +721,50 @@ def fetch_many(cls, spec, artifacts, base): host, root = RepositoryManager.ssh_location(spec["location"]) with tempfile.TemporaryDirectory(prefix="pload-fetch-") as temporary: stage = Path(temporary) + batch = stage / "commands.sftp" commands = [] for checksum, _ in artifacts: commands.append( f"get {root}/objects/{checksum} {stage / checksum}" ) - execute( - ["sftp", "-q", "-b", "-", *cls.SSH_OPTIONS, host], - timeout=300, input_text="\n".join(commands) + "\n", - ) + batch.write_text("\n".join(commands) + "\n", encoding="utf-8") + command = ["sftp", "-q", "-b", str(batch), *cls.SSH_OPTIONS, host] + try: + process = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + ) + except OSError as exc: + raise PloadError(f"cannot run sftp: {exc}") from exc + started = time.monotonic() + last_sizes = {checksum: -1 for checksum, _ in artifacts} + while process.poll() is None: + now = time.monotonic() + if now - started > 300: + process.kill() + process.wait() + raise PloadError("sftp timed out after 300 seconds") + if progress: + for checksum, _ in artifacts: + downloaded = stage / checksum + received = downloaded.stat().st_size if downloaded.is_file() else 0 + if received != last_sizes[checksum]: + progress(checksum, received, sizes.get(checksum, 0)) + last_sizes[checksum] = received + time.sleep(DOWNLOAD_PROGRESS_INTERVAL) + stdout = process.stdout.read() if process.stdout else "" + stderr = process.stderr.read() if process.stderr else "" + if process.returncode: + detail = (stderr or stdout).strip() + raise PloadError(f"sftp failed: {detail or process.returncode}") for checksum, destination in artifacts: downloaded = stage / checksum if not downloaded.is_file() or digest(downloaded) != checksum: raise PloadError("retrieved artifact checksum mismatch") + if progress: + progress( + checksum, downloaded.stat().st_size, + sizes.get(checksum) or downloaded.stat().st_size, + ) temporary_destination = destination.with_name( destination.name + "." + uuid.uuid4().hex ) @@ -1885,9 +2015,17 @@ def _acquire( raise PloadError("selected external cache artifact changed or disappeared") shutil.copyfile(source, destination) elif selected["method"] == "repository": + transfer_progress = None + if progress: + label = f"{package['name']}=={package['version']}" + + def transfer_progress(received, total): + progress(f"DOWNLOAD\t{label}\t{received}\t{total}") + ArtifactRepository.fetch( data["repositories"][selected["location"]], artifact["sha256"], - destination, path.parent, + destination, path.parent, progress=transfer_progress, + expected_size=artifact.get("size", 0), ) elif selected["method"] == "index-exact": if artifact.get("url"): diff --git a/tests/test_declarative.py b/tests/test_declarative.py index b4c7e18..d6d9e05 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -1,3 +1,4 @@ +import hashlib import io import platform import shutil @@ -11,6 +12,7 @@ import pload.declarative as declarative_module from pload.cli import build_parser, main, shell_script from pload.declarative import ( + ArtifactRepository, DeclarativeEnvironmentManager, dump_lock, dump_manifest, @@ -941,6 +943,84 @@ def test_exact_download_reports_frequent_real_byte_progress(tmp_path, monkeypatc assert destination.read_bytes() == payload +def test_local_repository_fetch_streams_real_byte_progress(tmp_path, monkeypatch): + payload = b"r" * (declarative_module.DOWNLOAD_CHUNK_SIZE * 4) + checksum = hashlib.sha256(payload).hexdigest() + repository = tmp_path / "repository" + source = repository / "objects" / checksum + source.parent.mkdir(parents=True) + source.write_bytes(payload) + ticks = iter((0, 0.04, 0.08, 0.12, 0.16)) + monkeypatch.setattr(declarative_module.time, "monotonic", lambda: next(ticks)) + destination = tmp_path / "downloaded.whl" + events = [] + + ArtifactRepository.fetch( + {"kind": "local", "location": str(repository)}, + checksum, + destination, + tmp_path, + progress=lambda received, total: events.append((received, total)), + expected_size=len(payload), + ) + + assert len(events) >= 3 + assert events[-1] == (len(payload), len(payload)) + assert destination.read_bytes() == payload + + +def test_ssh_repository_fetch_streams_real_bytes(tmp_path, monkeypatch): + chunk = b"s" * declarative_module.DOWNLOAD_CHUNK_SIZE + payload = chunk * 4 + checksum = hashlib.sha256(payload).hexdigest() + + class StreamingOutput: + def __init__(self): + self.parts = 0 + + def read1(self, size): + if self.parts < 4: + self.parts += 1 + return chunk + return b"" + + class StreamingSsh: + def __init__(self, command, stdout, stderr): + self.stdout = StreamingOutput() + self.stderr = io.BytesIO() + self.returncode = 0 + + def poll(self): + return 0 + + def kill(self): + self.returncode = -1 + + def wait(self, timeout=None): + return self.returncode + + monkeypatch.setattr(declarative_module.subprocess, "Popen", StreamingSsh) + ticks = iter((0, 0.04, 0.08, 0.12, 0.16, 0.20)) + monkeypatch.setattr(declarative_module.time, "monotonic", lambda: next(ticks)) + events = [] + destination = tmp_path / "ssh-download.whl" + + ArtifactRepository.fetch( + {"kind": "ssh", "location": "example:/repository"}, + checksum, + destination, + tmp_path, + progress=lambda received, total: events.append((received, total)), + expected_size=len(payload), + ) + + assert [received for received, _ in events[:4]] == [ + len(chunk), len(chunk) * 2, len(chunk) * 3, len(chunk) * 4, + ] + assert events[-1] == (len(payload), len(payload)) + assert destination.read_bytes() == payload + + def test_offline_apply_rejects_saved_network_route(tmp_path, monkeypatch): config = ConfigManager(home=tmp_path / "home") config.get_python_path = lambda version=None: Path(sys.executable) From 85b22e9d3e2fba2c65f87cdc650af24fa49bc78b Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Mon, 28 Sep 2026 15:40:45 +0800 Subject: [PATCH 38/41] Make every artifact route report real progress --- CHANGELOG.md | 2 + README.md | 7 ++- docs/declarative-environments.md | 16 +++--- src/pload/cli.py | 6 --- src/pload/declarative.py | 86 ++++++++++++++++++++++++++++---- tests/test_cli.py | 5 +- tests/test_declarative.py | 13 +++-- 7 files changed, 105 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6feeda..1a0d0d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,8 @@ Pre-release: `1.2.0a1` up to 30 updates per second while coalescing non-interactive logs. - Stream local repository copies and SSH object responses so repository routes report real byte progress instead of completing at once. +- Remove grey source/status suffixes from progress rows. Cache verification and + local artifact copying now report actual processed bytes before completion. ### Removed diff --git a/README.md b/README.md index 94b07a8..1849a2b 100644 --- a/README.md +++ b/README.md @@ -103,12 +103,15 @@ it never re-resolves, changes route, falls back, or uploads packages. Set chooser; strict per-package exceptions live under `[plan.packages.NAME]`. `apply` prints the saved plan first, then compact Docker-style progress rows. Each package is prefixed by its route, such as `[cache]`, `[index-exact]`, or -`[repository]`, alongside its source and real download byte progress. The bar +`[repository]`, followed only by its byte progress. Verbose grey source/status +suffixes are intentionally omitted because the saved plan above already names +the selected resource. The bar uses Rich's standard adaptive-width bar with explicit start and end boundaries, so it grows naturally on wider terminals without another progress dependency. Index and repository transfers report real received bytes at up to 30 updates per second. SSH repository downloads stream the remote object directly; -redirected logs remain coalesced instead of flooding CI. +cache routes report bytes while verifying SHA-256 instead of turning green +before visible work. Redirected logs remain coalesced instead of flooding CI. Use `pload apply --local` (or `-l`) for a project-local environment. It creates `.venv` beside the selected `pload.toml`, matching the layout produced by diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index aac2785..f65f943 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -276,16 +276,20 @@ quitting or `--json` does not. `pload apply` first prints the complete persisted plan table. It then uses compact Docker-style progress rows: every exact package starts with its selected method in brackets (`[cache]`, `[index-exact]`, `[repository]`, and so on), -followed by the package, source, Rich's bounded adaptive-width progress bar and -real byte counts. The bar consumes the terminal width still available after the -useful text columns, so wide terminals get a substantially longer bar without -breaking narrow ones. Direct artifact downloads read in 16 KiB chunks and +followed by the package, Rich's bounded adaptive-width progress bar and real +byte counts. Source descriptions and status prose are omitted from the row +because the saved plan table immediately above already identifies the resource. +The bar consumes the terminal width still available after the useful text +columns, so wide terminals get a substantially longer bar without breaking +narrow ones. Direct artifact downloads read in 16 KiB chunks and publish real byte counts at up to 30 updates per second; this keeps interactive motion fluid without inventing intermediate progress. Local repositories stream their copied bytes, while SSH repositories stream the remote object into the local destination at the same refresh cadence. Redirected output coalesces those -updates into stable plain-text -rows without terminal control characters. This makes the saved decision and +updates into stable plain-text rows without terminal control characters. Cache +routes report bytes as their SHA-256 content is actually read; configuration +artifacts and external caches report bytes while copying and hashing. This makes +the saved decision and its execution visible in one command without allowing apply to recalculate the plan. diff --git a/src/pload/cli.py b/src/pload/cli.py index 8d138e4..58565f7 100644 --- a/src/pload/cli.py +++ b/src/pload/cli.py @@ -567,7 +567,6 @@ def __init__(self, output=None): ), TextColumn("]", style="dim", markup=False), TextColumn("{task.fields[amount]}", style="cyan", markup=False), - TextColumn("{task.fields[status]}", style="dim", markup=False), console=self.console, transient=False, expand=True, @@ -608,11 +607,6 @@ def __call__(self, message): self.details[package]["status"] = status if self.interactive: self.progress.update(self.tasks[package], status=status) - else: - detail = self.details[package] - self.console.print( - Text(f"{detail['tag']:<24} {package} {status}") - ) elif kind == "DOWNLOAD": _, package, received, expected = parts received, expected = int(received), int(expected) diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 9a102fb..c5098b7 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -64,6 +64,56 @@ def _duration(value): return f"{value * 1000:.0f} ms" if value < 1 else f"{value:.2f} s" +def _report_transfer(progress, label, received, total): + if progress: + progress(f"DOWNLOAD\t{label}\t{received}\t{total}") + + +def _digest_with_progress(path, label, progress=None): + """Hash a local artifact while reporting bytes actually verified.""" + path = Path(path) + total = path.stat().st_size + checksum = hashlib.sha256() + received = 0 + last_reported_at = time.monotonic() + with path.open("rb") as stream: + while True: + chunk = stream.read(1024 * 1024) + if not chunk: + break + checksum.update(chunk) + received += len(chunk) + now = time.monotonic() + if now - last_reported_at >= DOWNLOAD_PROGRESS_INTERVAL: + _report_transfer(progress, label, received, total) + last_reported_at = now + _report_transfer(progress, label, received, total) + return checksum.hexdigest() + + +def _copy_with_progress(source, destination, label, progress=None): + """Copy and hash a local artifact while reporting real processed bytes.""" + source = Path(source) + total = source.stat().st_size + checksum = hashlib.sha256() + received = 0 + last_reported_at = time.monotonic() + with source.open("rb") as input_stream, Path(destination).open("wb") as output: + while True: + chunk = input_stream.read(1024 * 1024) + if not chunk: + break + output.write(chunk) + checksum.update(chunk) + received += len(chunk) + now = time.monotonic() + if now - last_reported_at >= DOWNLOAD_PROGRESS_INTERVAL: + _report_transfer(progress, label, received, total) + last_reported_at = now + _report_transfer(progress, label, received, total) + return checksum.hexdigest() + + def normalized_name(value): return re.sub(r"[-_.]+", "-", value).lower() @@ -624,7 +674,10 @@ def _fetch_ssh_stream( temporary = destination.with_name(destination.name + "." + uuid.uuid4().hex) command = [ "ssh", *cls.SSH_OPTIONS, host, - f"cat -- {shlex.quote(remote)}", + ( + f"size=$(wc -c < {shlex.quote(remote)}) && " + f"printf '%s\\n' \"$size\" && cat -- {shlex.quote(remote)}" + ), ] try: process = subprocess.Popen( @@ -636,6 +689,12 @@ def _fetch_ssh_stream( started = time.monotonic() last_reported_at = started try: + size_line = process.stdout.readline(64) + try: + announced_size = int(size_line.strip()) + except (TypeError, ValueError) as exc: + raise PloadError("ssh repository returned an invalid object size") from exc + total = announced_size or expected_size with temporary.open("wb") as output: while True: reader = ( @@ -654,7 +713,7 @@ def _fetch_ssh_stream( raise PloadError("ssh artifact download timed out after 300 seconds") if progress and (now - last_reported_at >= DOWNLOAD_PROGRESS_INTERVAL): - progress(received, expected_size) + progress(received, total) last_reported_at = now remaining = max(0.1, 300 - (time.monotonic() - started)) try: @@ -669,7 +728,7 @@ def _fetch_ssh_stream( if returncode: raise PloadError(f"ssh failed: {stderr.strip() or returncode}") if progress: - progress(received, expected_size or received) + progress(received, total or received) if digest(temporary) != checksum: raise PloadError("retrieved artifact checksum mismatch") temporary.replace(destination) @@ -1695,7 +1754,9 @@ def apply( source = Path(selected["location"]) if progress: progress(f"SOURCE\t{label}\tusing cache {source}") - if not source.is_file() or digest(source) != artifact["sha256"]: + if not source.is_file() or _digest_with_progress( + source, label, progress, + ) != artifact["sha256"]: raise PloadError( f"planned route failed for {label} (cache): " "selected cache artifact changed or disappeared; " @@ -2007,20 +2068,25 @@ def _acquire( progress=None, ): artifact = selected["artifact"] + label = f"{package['name']}=={package['version']}" if selected["method"] == "configuration-artifact": - shutil.copyfile(selected["location"], destination) + checksum = _copy_with_progress( + selected["location"], destination, label, progress, + ) + if checksum != artifact["sha256"]: + raise PloadError("selected configuration artifact changed or disappeared") elif selected["method"] == "external-cache": source = Path(selected["location"]) - if not source.is_file() or digest(source) != artifact["sha256"]: + if not source.is_file(): + raise PloadError("selected external cache artifact changed or disappeared") + checksum = _copy_with_progress(source, destination, label, progress) + if checksum != artifact["sha256"]: raise PloadError("selected external cache artifact changed or disappeared") - shutil.copyfile(source, destination) elif selected["method"] == "repository": transfer_progress = None if progress: - label = f"{package['name']}=={package['version']}" - def transfer_progress(received, total): - progress(f"DOWNLOAD\t{label}\t{received}\t{total}") + _report_transfer(progress, label, received, total) ArtifactRepository.fetch( data["repositories"][selected["location"]], artifact["sha256"], diff --git a/tests/test_cli.py b/tests/test_cli.py index 225e996..b23c2a9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -269,10 +269,9 @@ def apply(self, path, name, offline, progress, target=None): assert events == ["loaded", "printed-plan", "apply"] output = capsys.readouterr().out - normalized = " ".join(output.split()) assert "[index-exact]" in output assert "numpy==2.0.2" in output - assert "downloading from https://files.invalid/numpy.whl" in normalized + assert "downloading from https://files.invalid/numpy.whl" not in output assert "8.0 MiB / 16.0 MiB" in output @@ -296,7 +295,7 @@ def test_apply_progress_uses_one_adaptive_rich_row_per_package(): assert "━" in progress_line assert "░" not in rendered assert "8.0 MiB / 16.0 MiB" in rendered - assert "downloading" in rendered + assert "downloading from" not in rendered def test_apply_local_targets_venv_beside_manifest(tmp_path, monkeypatch): diff --git a/tests/test_declarative.py b/tests/test_declarative.py index d6d9e05..de72669 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -374,11 +374,15 @@ def reject_package_network(command, *args, **kwargs): return original_execute(command, *args, **kwargs) monkeypatch.setattr(declarative_module, "execute", reject_package_network) - restored = manager.apply(manifest) + progress = [] + restored = manager.apply(manifest, progress=progress.append) output = original_execute([ config.get_pip_command(restored)[0], "-c", "import pload_demo; print(pload_demo.answer)", ]) assert output == "42" + assert any( + event.startswith("DOWNLOAD\tpload-demo==1.0\t") for event in progress + ) def test_apply_can_materialize_project_local_environment(tmp_path): @@ -961,7 +965,7 @@ def test_local_repository_fetch_streams_real_byte_progress(tmp_path, monkeypatch destination, tmp_path, progress=lambda received, total: events.append((received, total)), - expected_size=len(payload), + expected_size=0, ) assert len(events) >= 3 @@ -978,6 +982,9 @@ class StreamingOutput: def __init__(self): self.parts = 0 + def readline(self, size): + return f"{len(payload)}\n".encode() + def read1(self, size): if self.parts < 4: self.parts += 1 @@ -1011,7 +1018,7 @@ def wait(self, timeout=None): destination, tmp_path, progress=lambda received, total: events.append((received, total)), - expected_size=len(payload), + expected_size=0, ) assert [received for received, _ in events[:4]] == [ From c22b919ac6afb1a88e74c459ffbdc459f088254a Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Mon, 28 Sep 2026 15:44:36 +0800 Subject: [PATCH 39/41] Stream environment copy progress --- CHANGELOG.md | 2 ++ README.md | 4 ++- docs/declarative-environments.md | 6 ++-- src/pload/declarative.py | 57 +++++++++++++++++++++++--------- tests/test_cli.py | 16 +++++++++ tests/test_declarative.py | 10 +++++- 6 files changed, 76 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a0d0d2..8e80d1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,8 @@ Pre-release: `1.2.0a1` report real byte progress instead of completing at once. - Remove grey source/status suffixes from progress rows. Cache verification and local artifact copying now report actual processed bytes before completion. +- Replace environment-copy's opaque zip/extract step with a validated file-list + stream that reports total and copied bytes and finishes with a green bar. ### Removed diff --git a/README.md b/README.md index 1849a2b..f039926 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,9 @@ so it grows naturally on wider terminals without another progress dependency. Index and repository transfers report real received bytes at up to 30 updates per second. SSH repository downloads stream the remote object directly; cache routes report bytes while verifying SHA-256 instead of turning green -before visible work. Redirected logs remain coalesced instead of flooding CI. +before visible work. Environment-copy routes validate the source `RECORD`, then +report each byte copied from the compatible environment and finish green. +Redirected logs remain coalesced instead of flooding CI. Use `pload apply --local` (or `-l`) for a project-local environment. It creates `.venv` beside the selected `pload.toml`, matching the layout produced by diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index f65f943..a398f29 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -288,8 +288,10 @@ their copied bytes, while SSH repositories stream the remote object into the local destination at the same refresh cadence. Redirected output coalesces those updates into stable plain-text rows without terminal control characters. Cache routes report bytes as their SHA-256 content is actually read; configuration -artifacts and external caches report bytes while copying and hashing. This makes -the saved decision and +artifacts and external caches report bytes while copying and hashing. +Environment-copy routes first validate the planned version and `RECORD` +fingerprint, then copy the validated file list with a real total and processed +byte count. This makes the saved decision and its execution visible in one command without allowing apply to recalculate the plan. diff --git a/src/pload/declarative.py b/src/pload/declarative.py index c5098b7..56754d3 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -13,7 +13,6 @@ import tempfile import time import uuid -import zipfile from concurrent.futures import ThreadPoolExecutor from pathlib import Path from urllib.parse import urlsplit @@ -1840,7 +1839,9 @@ def apply( execute(pip + ["install", "--no-index", "--no-deps"] + artifacts, capture=False) for package_plan in environment_copies: - self._copy_distribution(package_plan, env, stage) + self._copy_distribution( + package_plan, env, stage, progress=progress, + ) if progress: progress( f"PACKAGE_DONE\t{package_plan['name']}==" @@ -2012,15 +2013,15 @@ def _validate_saved_plan(saved, data): and data["policy"].get("reproducibility") != "compatible"): raise PloadError("index-resolve is forbidden by exact reproducibility policy") - def _copy_distribution(self, package, target_environment, stage): + def _copy_distribution(self, package, target_environment, stage, progress=None): selected = package["selected"] source_environment = Path(selected["location"]) source_python = source_environment / ( "Scripts/python.exe" if os.name == "nt" else "bin/python" ) - archive = stage / (normalized_name(package["name"]) + ".zip") + file_manifest = stage / (normalized_name(package["name"]) + ".files.json") script = r''' -import hashlib, json, os, sys, sysconfig, zipfile +import hashlib, json, os, sys, sysconfig from importlib import metadata name, version, fingerprint, output = sys.argv[1:] dist = metadata.distribution(name) @@ -2041,13 +2042,12 @@ def _copy_distribution(self, package, target_environment, stage): raise SystemExit('unsafe distribution path') if os.path.isfile(source): files.append((source, relative)) -with zipfile.ZipFile(output, 'w', compression=zipfile.ZIP_DEFLATED) as bundle: - for source, relative in files: - bundle.write(source, relative) +with open(output, 'w', encoding='utf-8') as stream: + json.dump(files, stream) ''' execute([ str(source_python), "-c", script, package["name"], package["version"], - selected["fingerprint"], str(archive), + selected["fingerprint"], str(file_manifest), ]) target_python = Path(target_environment) / ( "Scripts/python.exe" if os.name == "nt" else "bin/python" @@ -2056,12 +2056,39 @@ def _copy_distribution(self, package, target_environment, stage): str(target_python), "-c", "import sysconfig; print(sysconfig.get_path('purelib'))", ])) - with zipfile.ZipFile(archive) as bundle: - for member in bundle.infolist(): - destination = (site_packages / member.filename).resolve() - if site_packages.resolve() not in destination.parents: - raise PloadError("environment package contains an unsafe path") - bundle.extractall(site_packages) + try: + files = json.loads(file_manifest.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise PloadError("cannot read environment package file list") from exc + checked = [] + total = 0 + site_root = site_packages.resolve() + for source_value, relative in files: + source = Path(source_value) + destination = (site_root / relative).resolve() + if site_root not in destination.parents or not source.is_file(): + raise PloadError("environment package contains an unsafe path") + size = source.stat().st_size + total += size + checked.append((source, destination)) + label = f"{package['name']}=={package['version']}" + received = 0 + last_reported_at = time.monotonic() + _report_transfer(progress, label, 0, total) + for source, destination in checked: + destination.parent.mkdir(parents=True, exist_ok=True) + with source.open("rb") as input_stream, destination.open("wb") as output: + while True: + chunk = input_stream.read(1024 * 1024) + if not chunk: + break + output.write(chunk) + received += len(chunk) + now = time.monotonic() + if now - last_reported_at >= DOWNLOAD_PROGRESS_INTERVAL: + _report_transfer(progress, label, received, total) + last_reported_at = now + _report_transfer(progress, label, received, total) def _acquire( self, path, data, package, selected, destination, stage, interpreter, diff --git a/tests/test_cli.py b/tests/test_cli.py index b23c2a9..fafb533 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -298,6 +298,22 @@ def test_apply_progress_uses_one_adaptive_rich_row_per_package(): assert "downloading from" not in rendered +def test_environment_copy_progress_finishes_green(): + stream = io.StringIO() + console = Console( + file=stream, force_terminal=True, color_system="standard", no_color=False, + width=120, height=5, _environ={}, + ) + + with cli_module.ApplyProgress(console) as progress: + progress("PACKAGE\t1/1\tenvironment-copy\tdemo==1.0\t0") + progress("DOWNLOAD\tdemo==1.0\t0\t100") + progress("DOWNLOAD\tdemo==1.0\t100\t100") + progress("PACKAGE_DONE\tdemo==1.0\tcopied") + + assert "\x1b[32m" in stream.getvalue() + + def test_apply_local_targets_venv_beside_manifest(tmp_path, monkeypatch): captured = {} manifest = tmp_path / "project" / "pload.toml" diff --git a/tests/test_declarative.py b/tests/test_declarative.py index de72669..d2faf76 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -507,11 +507,19 @@ def reject_package_network(command, *args, **kwargs): return original_execute(command, *args, **kwargs) monkeypatch.setattr(declarative_module, "execute", reject_package_network) - restored = manager.apply(manifest) + progress = [] + restored = manager.apply(manifest, progress=progress.append) assert original_execute([ config.get_pip_command(restored)[0], "-c", "import pload_demo; print(pload_demo.answer)", ]) == "42" + copy_events = [ + event for event in progress + if event.startswith("DOWNLOAD\tpload-demo==1.0\t") + ] + assert copy_events + _, _, received, total = copy_events[-1].split("\t") + assert int(received) == int(total) > 0 def test_plan_resolves_metadata_once_without_downloading_wheels(tmp_path, monkeypatch): From 65f495f99a504d262db3ac691c2118b849be1b8d Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Mon, 28 Sep 2026 15:55:38 +0800 Subject: [PATCH 40/41] Stage environment copies in plan order --- CHANGELOG.md | 3 +- README.md | 4 +- docs/declarative-environments.md | 6 ++- src/pload/declarative.py | 70 +++++++++++++++++++++++--------- tests/test_declarative.py | 9 ++++ 5 files changed, 68 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e80d1a..96f23ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,7 +68,8 @@ Pre-release: `1.2.0a1` - Remove grey source/status suffixes from progress rows. Cache verification and local artifact copying now report actual processed bytes before completion. - Replace environment-copy's opaque zip/extract step with a validated file-list - stream that reports total and copied bytes and finishes with a green bar. + staging stream that reports total and copied bytes and finishes green before + the next package begins. ### Removed diff --git a/README.md b/README.md index f039926..d2c8f2b 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,9 @@ Index and repository transfers report real received bytes at up to 30 updates per second. SSH repository downloads stream the remote object directly; cache routes report bytes while verifying SHA-256 instead of turning green before visible work. Environment-copy routes validate the source `RECORD`, then -report each byte copied from the compatible environment and finish green. +immediately stage each byte from the compatible environment and finish green +before pload advances to the next package. The staged files are installed after +the target environment exists. Redirected logs remain coalesced instead of flooding CI. Use `pload apply --local` (or `-l`) for a project-local environment. It creates diff --git a/docs/declarative-environments.md b/docs/declarative-environments.md index a398f29..30ecd4d 100644 --- a/docs/declarative-environments.md +++ b/docs/declarative-environments.md @@ -290,8 +290,10 @@ updates into stable plain-text rows without terminal control characters. Cache routes report bytes as their SHA-256 content is actually read; configuration artifacts and external caches report bytes while copying and hashing. Environment-copy routes first validate the planned version and `RECORD` -fingerprint, then copy the validated file list with a real total and processed -byte count. This makes the saved decision and +fingerprint, then immediately stage the validated file list with a real total +and processed byte count. The row finishes before pload advances to the next +package; staged files move into site-packages after the target environment is +created. This makes the saved decision and its execution visible in one command without allowing apply to recalculate the plan. diff --git a/src/pload/declarative.py b/src/pload/declarative.py index 56754d3..0556afe 100644 --- a/src/pload/declarative.py +++ b/src/pload/declarative.py @@ -1815,7 +1815,23 @@ def apply( f"SOURCE\t{label}\tcopying installed package from " f"{selected['location']}" ) - environment_copies.append(package_plan) + staged_distribution = stage / f"environment-{index_number}" + try: + self._stage_distribution( + package_plan, staged_distribution, stage, + progress=progress, + ) + except (OSError, PloadError) as exc: + raise PloadError( + f"planned route failed for {label} " + f"(environment-copy): {exc}; " + "run pload plan to choose another route" + ) from exc + environment_copies.append( + (package_plan, staged_distribution) + ) + if progress: + progress(f"PACKAGE_DONE\t{label}\tcopied") else: source = data.get("sources", {}).get(selected["location"], {}) if progress: @@ -1838,15 +1854,10 @@ def apply( progress("STAGE installing verified wheel artifacts") execute(pip + ["install", "--no-index", "--no-deps"] + artifacts, capture=False) - for package_plan in environment_copies: - self._copy_distribution( - package_plan, env, stage, progress=progress, + for package_plan, staged_distribution in environment_copies: + self._install_staged_distribution( + staged_distribution, env, ) - if progress: - progress( - f"PACKAGE_DONE\t{package_plan['name']}==" - f"{package_plan['version']}\tcopied" - ) for package_plan in compatible: source_name = package_plan["selected"]["location"] source = data.get("sources", {}).get(source_name, {}) @@ -2013,7 +2024,9 @@ def _validate_saved_plan(saved, data): and data["policy"].get("reproducibility") != "compatible"): raise PloadError("index-resolve is forbidden by exact reproducibility policy") - def _copy_distribution(self, package, target_environment, stage, progress=None): + def _stage_distribution( + self, package, staged_distribution, stage, progress=None, + ): selected = package["selected"] source_environment = Path(selected["location"]) source_python = source_environment / ( @@ -2049,24 +2062,18 @@ def _copy_distribution(self, package, target_environment, stage, progress=None): str(source_python), "-c", script, package["name"], package["version"], selected["fingerprint"], str(file_manifest), ]) - target_python = Path(target_environment) / ( - "Scripts/python.exe" if os.name == "nt" else "bin/python" - ) - site_packages = Path(execute([ - str(target_python), "-c", - "import sysconfig; print(sysconfig.get_path('purelib'))", - ])) try: files = json.loads(file_manifest.read_text(encoding="utf-8")) except (OSError, ValueError) as exc: raise PloadError("cannot read environment package file list") from exc checked = [] total = 0 - site_root = site_packages.resolve() + staged_root = Path(staged_distribution).resolve() + staged_root.mkdir(parents=True) for source_value, relative in files: source = Path(source_value) - destination = (site_root / relative).resolve() - if site_root not in destination.parents or not source.is_file(): + destination = (staged_root / relative).resolve() + if staged_root not in destination.parents or not source.is_file(): raise PloadError("environment package contains an unsafe path") size = source.stat().st_size total += size @@ -2090,6 +2097,29 @@ def _copy_distribution(self, package, target_environment, stage, progress=None): last_reported_at = now _report_transfer(progress, label, received, total) + @staticmethod + def _install_staged_distribution(staged_distribution, target_environment): + target_python = Path(target_environment) / ( + "Scripts/python.exe" if os.name == "nt" else "bin/python" + ) + site_packages = Path(execute([ + str(target_python), "-c", + "import sysconfig; print(sysconfig.get_path('purelib'))", + ])).resolve() + for source in Path(staged_distribution).rglob("*"): + if not source.is_file(): + continue + relative = source.relative_to(staged_distribution) + destination = (site_packages / relative).resolve() + if site_packages not in destination.parents: + raise PloadError("staged environment package contains an unsafe path") + destination.parent.mkdir(parents=True, exist_ok=True) + try: + source.replace(destination) + except OSError: + shutil.copy2(source, destination) + source.unlink() + def _acquire( self, path, data, package, selected, destination, stage, interpreter, progress=None, diff --git a/tests/test_declarative.py b/tests/test_declarative.py index d2faf76..c9d7923 100644 --- a/tests/test_declarative.py +++ b/tests/test_declarative.py @@ -520,6 +520,15 @@ def reject_package_network(command, *args, **kwargs): assert copy_events _, _, received, total = copy_events[-1].split("\t") assert int(received) == int(total) > 0 + completed = next( + index for index, event in enumerate(progress) + if event.startswith("PACKAGE_DONE\tpload-demo==1.0\t") + ) + creates_environment = next( + index for index, event in enumerate(progress) + if event.startswith("STAGE creating environment") + ) + assert completed < creates_environment def test_plan_resolves_metadata_once_without_downloading_wheels(tmp_path, monkeypatch): From 29c4fbe51a92497da3e485e95cd7ee5c2ed91826 Mon Sep 17 00:00:00 2001 From: Yunming HU Date: Mon, 28 Sep 2026 16:08:35 +0800 Subject: [PATCH 41/41] Prepare 1.2.0b1 prerelease --- CHANGELOG.md | 6 ++++-- README.md | 2 +- pyproject.toml | 2 +- src/pload/__init__.py | 2 +- tests/test_cli.py | 4 +++- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96f23ea..5540e38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Changelog -## Unreleased +## 1.2.0b1 - 2026-09-28 -Pre-release: `1.2.0a1` +Feature-complete beta for the declarative environment workflow. ### Architecture reset @@ -78,6 +78,8 @@ Pre-release: `1.2.0a1` ### Fixed +- Accept Rich's documented ASCII progress-bar fallback on Windows consoles in + the cross-platform release test suite. - Keep an external environment's interpreter path intact when probing packages, instead of following its Python symlink out to the base installation. - Batch SSH repository inventory, publication and retrieval so environments with diff --git a/README.md b/README.md index d2c8f2b..7456cde 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ pload-install --yes --package-spec "pload==1.0.0" To opt into the declarative-environment pre-release for testing: ```console -pload-install --yes --package-spec "pload==1.2.0a1" +pload-install --yes --package-spec "pload==1.2.0b1" ``` To always follow the newest published release instead of pinning a version, diff --git a/pyproject.toml b/pyproject.toml index 8f7fd95..3934127 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pload" -version = "1.2.0a1" +version = "1.2.0b1" description = "A small, relocatable Python virtual environment manager" readme = "README.md" requires-python = ">=3.8" diff --git a/src/pload/__init__.py b/src/pload/__init__.py index 7bfbef1..4834673 100644 --- a/src/pload/__init__.py +++ b/src/pload/__init__.py @@ -1 +1 @@ -__version__ = "1.2.0a1" +__version__ = "1.2.0b1" diff --git a/tests/test_cli.py b/tests/test_cli.py index fafb533..cee7cfa 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -292,7 +292,9 @@ def test_apply_progress_uses_one_adaptive_rich_row_per_package(): assert "numpy==2.0.2" in rendered progress_line = next(line for line in rendered.splitlines() if "8.0 MiB" in line) assert "[" in progress_line and "]" in progress_line - assert "━" in progress_line + # Rich falls back to ASCII on legacy Windows consoles even when a terminal + # is forced, while Unix terminals use the Unicode heavy bar. + assert "━" in progress_line or "-" in progress_line assert "░" not in rendered assert "8.0 MiB / 16.0 MiB" in rendered assert "downloading from" not in rendered