From 1e85b8f491a5a22879ca608ca43c5e8e59a54469 Mon Sep 17 00:00:00 2001 From: "shipmill-romamo[bot]" <338557544+shipmill-romamo[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:43:17 +0200 Subject: [PATCH 1/5] Run Component build steps in artifact build artifact build packaged the git tree only, so a Component could not ship a built frontend such as a gitignored web/dist. It now checks every spec.buildTools entry on PATH at its minimum version before cloning, runs each spec.build step in the clean checkout through env -i with only PATH, HOME, LANG, TMPDIR and CI=true, streams step output to stderr so stdout stays one JSON envelope, and packages the tracked files plus every spec.artifact.include path. A failed or timed-out step or a missing include writes no archive. The Artifact metadata gains an optional spec.build with the steps and tool versions. A Component without build is packaged exactly as before. Spec 002 is now built: every criterion has its test and its verification. Fixes #115 Co-Authored-By: Claude Opus 5.5 --- changelog.d/115-component-build-steps.md | 3 + config/schemas/v1/artifact.schema.json | 44 ++ docs/specs/002-component-build-steps.md | 18 +- engine/README.md | 13 + engine/src/cloudfall_engine/artifact.py | 339 ++++++++++- engine/src/cloudfall_engine/cli.py | 42 +- engine/tests/test_artifact_build_steps.py | 663 ++++++++++++++++++++++ engine/tests/test_artifact_builder.py | 13 + sdk/src/cloudfall/app.py | 5 +- 9 files changed, 1112 insertions(+), 28 deletions(-) create mode 100644 changelog.d/115-component-build-steps.md create mode 100644 engine/tests/test_artifact_build_steps.py diff --git a/changelog.d/115-component-build-steps.md b/changelog.d/115-component-build-steps.md new file mode 100644 index 0000000..fd8d639 --- /dev/null +++ b/changelog.d/115-component-build-steps.md @@ -0,0 +1,3 @@ +### Added + +- `cloudfall-engine artifact build` runs a Component's declared `spec.build` steps on the controller before packaging: it checks every `spec.buildTools` entry is on `PATH` at its minimum version before cloning (`artifact_build_tool_missing`, `artifact_build_tool_unsupported`), runs each step in the clean checkout with only `PATH`, `HOME`, `LANG`, `TMPDIR` and `CI=true` in its environment and its output on stderr, fails on a non-zero exit or a timeout (`artifact_build_step_failed`) or a missing include path (`artifact_include_missing`) without writing an archive, packages the tracked files plus every `spec.artifact.include` path (a gitignored `web/dist`, say), and records the steps and tool versions as the new optional `spec.build` of the Artifact metadata. A Component without `build` is packaged as before (#115) diff --git a/config/schemas/v1/artifact.schema.json b/config/schemas/v1/artifact.schema.json index d366530..d0841ce 100644 --- a/config/schemas/v1/artifact.schema.json +++ b/config/schemas/v1/artifact.schema.json @@ -46,6 +46,50 @@ "sizeBytes": { "type": "integer", "minimum": 1 + }, + "build": { + "type": "object", + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "properties": { + "workingDirectory": { + "type": "string", + "minLength": 1 + }, + "command": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "workingDirectory", + "command" + ], + "additionalProperties": false + } + }, + "tools": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string", + "pattern": "^[0-9]{1,9}(?:\\.[0-9]{1,9}){0,7}$" + } + } + }, + "required": [ + "steps", + "tools" + ], + "additionalProperties": false } }, "required": [ diff --git a/docs/specs/002-component-build-steps.md b/docs/specs/002-component-build-steps.md index b631100..ffdc6a8 100644 --- a/docs/specs/002-component-build-steps.md +++ b/docs/specs/002-component-build-steps.md @@ -1,6 +1,6 @@ # S-002: Component build steps -status: approved +status: built ## Problem @@ -139,3 +139,19 @@ validate`, and the `build_artifact` MCP tool. - cloudfall-dev/cloudfall#115: S-002-2, S-002-3, S-002-4, S-002-5, S-002-6, S-002-7, S-002-8, S-002-9, S-002-10, S-002-12, S-002-13 ## Verification + +- S-002-1: `test_s002_1_component_with_build_steps_validates`, `test_s002_1_existing_component_without_build_is_unchanged` and `test_s002_1_build_step_timeout_accepts_its_bounds` (`sdk/tests/test_component_build_validation.py`) passed, and `cloudfall config validate --project config/examples` still exits 0 +- S-002-2: `test_s002_2_the_tarball_holds_the_gitignored_build_output` runs `cloudfall-engine artifact build --ref ` as a child process with stand-in `node` and `pnpm` on a temporary `PATH`; the tarball held `./web/dist/index.html` although the fixture's `.gitignore` lists `web/dist/` +- S-002-3: `test_s002_3_the_tarball_holds_every_tracked_file_and_no_node_modules` compared the tarball against `git ls-tree -r` at the commit: every tracked file, no `node_modules` entry, no `.git` +- S-002-4: `test_s002_4_the_metadata_records_the_steps_and_tool_versions` read `spec.build.steps` (both steps in order) and `spec.build.tools` `{"node": "22.3.0", "pnpm": "9.1.0"}` from the versions the stand-ins printed (`v22.3.0`, `9.1.0`), and validated the file against `artifact.schema.json` +- S-002-5: `test_s002_5_a_missing_tool_fails_before_any_clone` exited 81 with `artifact_build_tool_missing` and `tool: node`; the logging `git` wrapper on `PATH` recorded no `clone`, and no file was written +- S-002-6: `test_s002_6_an_old_tool_names_the_tool_and_both_versions` (`node` printing `v20.11.1` against `22.0.0`) and `test_s002_6_a_tool_that_prints_no_version_is_unsupported` exited 81 with `artifact_build_tool_unsupported`, naming the tool and both versions +- S-002-7: `test_s002_7_a_failing_step_names_itself_and_writes_nothing` exited 81 with `artifact_build_step_failed`, `step: 1`, `argv: ["pnpm", "fail"]`, `exitCode: 3` and the step's stderr tail; `test_s002_7_a_step_past_its_timeout_fails_the_build` covered a 1-second `timeoutSeconds`; neither left a `.tar.gz` or `.json` in the output directory +- S-002-8: `test_s002_8_a_missing_include_path_writes_no_artifact` exited 81 with `artifact_include_missing` naming `web/dist` and wrote nothing +- S-002-9: `test_s002_9_step_stdout_stays_out_of_the_json_envelope` and `test_s002_9_stdout_is_exactly_one_envelope` parsed the command's whole stdout as one JSON envelope, on success and on a failed step; the steps' stdout lines were on stderr +- S-002-10: `test_s002_10_a_step_sees_ci_and_none_of_the_controller_secrets` set `HCLOUD_TOKEN` and `SOPS_AGE_KEY` for the command; the environment the build step dumped into the archive had `CI=true`, the controller's `PATH`, `HOME` and `LANG`, and neither secret +- S-002-11: the `test_s002_11_*` tests in `sdk/tests/test_component_build_validation.py` passed (undeclared tool, absolute and `..` paths, `buildTools` or `artifact` without `build`, malformed steps) +- S-002-12: `test_s002_12_deploy_and_rollback_carry_the_built_files` builds two releases, runs the `cloudfall_deploy` role's own path, digest, unpack and switch tasks and then `rollback.yml`'s switch tasks under `ansible-playbook` against a directory on the controller (ownership dropped, no systemd); `releases//web/dist/index.html` held each build's page and `current/web/dist/` followed the deploy and the rollback. It needs GNU tar for `unarchive`, so it runs on CI's Linux runner; on macOS the same flow passed with the unpack done by `tar -xzf`. Not run against a live Debian host +- S-002-13: `test_s002_13_a_component_without_build_packages_the_clean_tree` found the archive equal to the clean checkout's tracked files and directories, no `git ls-files` run and no `spec.build` key; `test_s002_13_the_built_archive_lists_entries_like_the_plain_one` checked a build adds only its include paths +- S-002-14: `test_s002_14_reserved_runtimes_are_refused_as_config_invalid` (`sdk/tests/test_validation.py`) passed +- S-002-15: `test_s002_15_node_web_service_is_a_gap_not_a_component` (`sdk/tests/test_importer.py`) passed diff --git a/engine/README.md b/engine/README.md index 0ad6577..b0abf98 100644 --- a/engine/README.md +++ b/engine/README.md @@ -169,6 +169,19 @@ and writes schema-valid artifact metadata beside it: task artifact:build COMPONENT=crm-backend REF=main ``` +A component that declares `spec.build` (with `spec.buildTools` and +`spec.artifact.include`, see `docs/specs/002-component-build-steps.md`) is +built on the controller before it is packaged. Every `buildTools` entry must +be on `PATH` at its minimum version before anything is cloned; then each step +runs in order in the clean checkout, with only `PATH`, `HOME`, `LANG` and +`TMPDIR` from the controller's environment plus `CI=true`, so cloud tokens and +SOPS keys stay out of `package.json` scripts. Step output goes to stderr. The +archive holds the files git tracks plus the `artifact.include` paths, such as +a gitignored `web/dist`, and the metadata records the steps and the tool +versions under `spec.build`. A failed step or a missing include path writes +no archive. The command's time limit (1800 seconds) covers the clone and every +step; pass `--timeout` for longer builds. + `ansible/playbooks/deploy.yml` deploys one built release to the component's declared servers, one host at a time: diff --git a/engine/src/cloudfall_engine/artifact.py b/engine/src/cloudfall_engine/artifact.py index 71954bf..c3ac18f 100644 --- a/engine/src/cloudfall_engine/artifact.py +++ b/engine/src/cloudfall_engine/artifact.py @@ -4,14 +4,16 @@ import hashlib import json +import os import re +import shlex import shutil import tarfile import tempfile -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol from cloudfall.validation import ( ConfigValidationError, @@ -21,13 +23,33 @@ from jsonschema.exceptions import ValidationError if TYPE_CHECKING: - from collections.abc import Callable, Sequence + from collections.abc import Callable, Mapping, Sequence - from cloudfall.inventory import ComponentInventory, PlatformInventory + from cloudfall.domain import BuildTimeoutSeconds + from cloudfall.inventory import ( + ComponentBuild, + ComponentInventory, + PlatformInventory, + ) from treaty import Completed Runner = Callable[[Sequence[str]], Completed] - """Runs one git argv to completion without raising on a non-zero exit""" + """Runs one argv (git, a tool's ``--version``) without raising on non-zero""" + + +class StepRunner(Protocol): + """Runs one build step's argv in a directory, its output kept off stdout. + + It answers the completed run without raising on a non-zero exit, and + raises ``BuildStepTimeoutError`` when the step outruns ``timeout``. + """ + + def __call__( + self, argv: Sequence[str], *, cwd: Path, timeout: BuildTimeoutSeconds + ) -> Completed: + """Run ``argv`` in ``cwd`` for at most ``timeout``.""" + ... + _ARTIFACT_SCHEMA = "artifact.schema.json" GIT_TIMEOUT_SECONDS = 600 @@ -35,20 +57,36 @@ """The first git with ``--end-of-options``, which ref resolution passes""" _SHORT_COMMIT_LENGTH = 7 GIT_REF_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/@^~-]{0,199}") +_TOOL_VERSION_PATTERN = re.compile(r"[0-9]{1,9}(?:\.[0-9]{1,9}){0,7}") +BUILD_STEP_VARIABLES = ("PATH", "HOME", "LANG", "TMPDIR") +"""The only controller variables a build step sees, besides ``CI=true``""" +_ENV_TOOL = "env" +"""Starts each build step with ``env -i``, so it inherits nothing else""" +_STDERR_TAIL_CHARACTERS = 500 _ERROR_COMPONENT_MISSING = "artifact_component_missing" _ERROR_SUBDIRECTORY_MISSING = "artifact_subdirectory_missing" _ERROR_REF_INVALID = "artifact_ref_invalid" _ERROR_GIT_MISSING = "artifact_git_missing" _ERROR_GIT_FAILED = "artifact_git_failed" +_ERROR_TOOL_MISSING = "artifact_build_tool_missing" +_ERROR_TOOL_UNSUPPORTED = "artifact_build_tool_unsupported" +_ERROR_STEP_FAILED = "artifact_build_step_failed" +_ERROR_INCLUDE_MISSING = "artifact_include_missing" class ArtifactBuildError(RuntimeError): """Fail-fast artifact build error with a stable machine-readable code.""" - def __init__(self, code: str, message: str) -> None: - """Record the failure code and human-readable detail.""" + def __init__( + self, + code: str, + message: str, + context: Mapping[str, object] | None = None, + ) -> None: + """Record the failure code, human-readable detail, and its facts.""" self.code = code self.detail = message + self.context: dict[str, object] = dict(context or {}) super().__init__(f"{code}: {message}") def as_dict(self) -> dict[str, object]: @@ -59,6 +97,34 @@ def as_dict(self) -> dict[str, object]: } +class BuildStepTimeoutError(RuntimeError): + """A build step ran past its own ``timeoutSeconds`` and was stopped.""" + + +@dataclass(frozen=True, slots=True) +class _ResolvedTool: + """A build tool found on the controller's ``PATH`` and the version it read.""" + + name: str + path: str + version: str + + +@dataclass(frozen=True, slots=True) +class _BuildHost: + """What build steps run with: the tools, ``env``, and the step variables.""" + + tools: Mapping[str, _ResolvedTool] + env_path: str + variables: tuple[tuple[str, str], ...] = field(default=()) + + def argv(self, command: Sequence[str]) -> tuple[str, ...]: + """Start the step's tool through ``env -i`` with the step variables.""" + assignments = tuple(f"{name}={value}" for name, value in self.variables) + tool = self.tools[command[0]].path + return (self.env_path, "-i", *assignments, tool, *command[1:]) + + @dataclass(frozen=True, slots=True) class BuiltArtifact: """One built and hashed release artifact.""" @@ -89,7 +155,7 @@ def as_dict(self) -> dict[str, object]: } -def build_artifact( # noqa: PLR0913 - the build inputs plus its git runner. +def build_artifact( # noqa: PLR0913 - the build inputs plus its runners. inventory: PlatformInventory, component_id: str, git_ref: str, @@ -97,14 +163,22 @@ def build_artifact( # noqa: PLR0913 - the build inputs plus its git runner. schema_directory: Path, *, run: Runner, + run_step: StepRunner, + environment: Mapping[str, str], ) -> BuiltArtifact: - """Clone the component repository at a ref and package a release. + """Clone the component repository at a ref, build it, and package it. - ``run`` starts git: the CLI passes ``ctx.run``, so treaty owns its time - limit, locale, and cancellation. + ``run`` starts git and each build tool's ``--version``; ``run_step`` + starts the build steps. The CLI passes ``ctx.run`` for both, so treaty + owns their time limits and cancellation, and step output goes to stderr. + ``environment`` is the controller's: build tools are looked up on its + ``PATH``, and a step sees only its ``BUILD_STEP_VARIABLES`` and + ``CI=true``. A component without ``build`` runs none of this. """ component = _component(inventory, component_id) _validate_ref(git_ref) + build = component.build + host = None if build is None else _build_host(build, environment, run) built_at = datetime.now(tz=UTC) built_at_text = built_at.strftime("%Y-%m-%dT%H:%M:%SZ") with tempfile.TemporaryDirectory(prefix="cloudfall-artifact-") as workdir: @@ -138,6 +212,12 @@ def build_artifact( # noqa: PLR0913 - the build inputs plus its git runner. raise ArtifactBuildError( _ERROR_SUBDIRECTORY_MISSING, detail ) + tracked: tuple[str, ...] = () + if build is not None and host is not None: + # The tracked files are read before any step can touch the index. + tracked = _tracked_files(run, source_root) + _run_steps(build, host, source_root, run_step) + _require_includes(build, source_root) release = ( f"{built_at.strftime('%Y%m%dT%H%M%SZ')}" @@ -146,10 +226,25 @@ def build_artifact( # noqa: PLR0913 - the build inputs plus its git runner. component_directory = output_directory / component_id component_directory.mkdir(parents=True, exist_ok=True) archive_path = component_directory / f"{release}.tar.gz" - _write_archive(source_root, archive_path) + if build is None: + _write_archive(source_root, archive_path) + else: + _write_built_archive(source_root, archive_path, tracked, build) digest = hashlib.sha256(archive_path.read_bytes()).hexdigest() size_bytes = archive_path.stat().st_size + spec: dict[str, object] = { + "component": component_id, + "release": release, + "gitRef": git_ref, + "gitCommit": commit, + "builtAt": built_at_text, + "archive": archive_path.name, + "archiveSha256": digest, + "sizeBytes": size_bytes, + } + if build is not None and host is not None: + spec["build"] = _build_record(build, host) metadata = { "apiVersion": "cloudfall/v1", "kind": "Artifact", @@ -157,16 +252,7 @@ def build_artifact( # noqa: PLR0913 - the build inputs plus its git runner. "id": component_id, "description": f"Release artifact for {component_id}", }, - "spec": { - "component": component_id, - "release": release, - "gitRef": git_ref, - "gitCommit": commit, - "builtAt": built_at_text, - "archive": archive_path.name, - "archiveSha256": digest, - "sizeBytes": size_bytes, - }, + "spec": spec, } catalog = SchemaCatalog(schema_directory) try: @@ -265,3 +351,212 @@ def _skip_git(info: tarfile.TarInfo) -> tarfile.TarInfo | None: with tarfile.open(archive_path, "w:gz") as archive: archive.add(source_root, arcname=".", filter=_skip_git) + + +def _build_host( + build: ComponentBuild, environment: Mapping[str, str], run: Runner +) -> _BuildHost: + """Check every build tool before anything is cloned (S-002 step 1).""" + search_path = environment.get("PATH", "") + tools: dict[str, _ResolvedTool] = {} + for tool in build.tools: + name = tool.name.value + path = _which(name, search_path) + found = _tool_version(run, name, path) + minimum = tool.minimum_version.value + if _version_key(found) < _version_key(minimum): + detail = ( + f"build tool {name} is {found}, older than the {minimum} " + "spec.buildTools requires" + ) + raise ArtifactBuildError( + _ERROR_TOOL_UNSUPPORTED, + detail, + {"tool": name, "found": found, "minimum": minimum}, + ) + tools[name] = _ResolvedTool(name=name, path=path, version=found) + variables = tuple( + (name, environment[name]) + for name in BUILD_STEP_VARIABLES + if name in environment + ) + return _BuildHost( + tools=tools, + env_path=_which(_ENV_TOOL, search_path), + variables=(*variables, ("CI", "true")), + ) + + +def _which(name: str, search_path: str) -> str: + path = shutil.which(name, path=search_path) + if path is None: + detail = f"build tool {name} is not on the controller's PATH" + raise ArtifactBuildError(_ERROR_TOOL_MISSING, detail, {"tool": name}) + if "=" in path: + # ``env -i`` would read the path as a variable assignment. + detail = f"build tool {name} resolves to a path with '=': {path}" + raise ArtifactBuildError(_ERROR_TOOL_UNSUPPORTED, detail, {"tool": name}) + return path + + +def _tool_version(run: Runner, name: str, path: str) -> str: + completed = run((path, "--version")) + match = ( + _TOOL_VERSION_PATTERN.search(completed.stdout) + or _TOOL_VERSION_PATTERN.search(completed.stderr) + if completed.returncode == 0 + else None + ) + if match is None: + detail = ( + f"build tool {name} printed no version for --version " + f"(exit code {completed.returncode})" + ) + raise ArtifactBuildError( + _ERROR_TOOL_UNSUPPORTED, detail, {"tool": name, "found": None} + ) + return match.group(0) + + +def _version_key(version: str) -> tuple[int, ...]: + parts = [int(part) for part in version.split(".")] + return tuple(parts + [0] * (8 - len(parts))) + + +def _tracked_files(run: Runner, source_root: Path) -> tuple[str, ...]: + listed = _run_git(run, "-C", str(source_root), "ls-files", "-z") + return tuple(path for path in listed.split("\0") if path) + + +def _run_steps( + build: ComponentBuild, + host: _BuildHost, + source_root: Path, + run_step: StepRunner, +) -> None: + for index, step in enumerate(build.steps): + command = step.command.value + context: dict[str, object] = { + "step": index, + "workingDirectory": step.working_directory.value, + "argv": list(command), + } + label = f"build step {index} ({shlex.join(command)})" + directory = source_root / step.working_directory.value + if not directory.is_dir(): + detail = ( + f"{label}: working directory " + f"{step.working_directory.value} does not exist" + ) + raise ArtifactBuildError( + _ERROR_STEP_FAILED, detail, {**context, "exitCode": None} + ) + try: + completed = run_step( + host.argv(command), cwd=directory, timeout=step.timeout_seconds + ) + except BuildStepTimeoutError as error: + detail = ( + f"{label} ran past its {step.timeout_seconds.value}s timeout " + "and was stopped" + ) + raise ArtifactBuildError( + _ERROR_STEP_FAILED, + detail, + { + **context, + "exitCode": None, + "timeoutSeconds": step.timeout_seconds.value, + }, + ) from error + if completed.returncode != 0: + tail = completed.stderr.strip()[-_STDERR_TAIL_CHARACTERS:] + detail = f"{label} exited with code {completed.returncode}" + if tail: + detail = f"{detail}: {tail}" + raise ArtifactBuildError( + _ERROR_STEP_FAILED, + detail, + {**context, "exitCode": completed.returncode}, + ) + + +def _require_includes(build: ComponentBuild, source_root: Path) -> None: + for include in build.include: + if not os.path.lexists(source_root / include.value): + detail = ( + f"artifact.include path {include.value} does not exist " + "after the build steps" + ) + raise ArtifactBuildError( + _ERROR_INCLUDE_MISSING, detail, {"path": include.value} + ) + + +def _write_built_archive( + source_root: Path, + archive_path: Path, + tracked: Sequence[str], + build: ComponentBuild, +) -> None: + """Package the tracked files plus every ``artifact.include`` path. + + Whatever else the steps left behind (``node_modules/``, caches) stays out. + A tracked file a step deleted is not packaged. + """ + entries: set[Path] = set() + for name in tracked: + relative = Path(name) + if os.path.lexists(source_root / relative): + entries.add(relative) + for include in build.include: + entries.update(_include_entries(source_root, Path(include.value))) + for entry in tuple(entries): + entries.update(entry.parents) + entries.discard(Path()) + ordered = sorted( + (entry for entry in entries if ".git" not in entry.parts), + key=lambda entry: entry.parts, + ) + with tarfile.open(archive_path, "w:gz") as archive: + archive.add(source_root, arcname=".", recursive=False, filter=_owned_by_root) + for entry in ordered: + archive.add( + source_root / entry, + arcname=f"./{entry.as_posix()}", + recursive=False, + filter=_owned_by_root, + ) + + +def _include_entries(source_root: Path, include: Path) -> set[Path]: + top = source_root / include + if top.is_symlink() or not top.is_dir(): + return {include} + entries = {include} + for directory, directories, files in os.walk(top): + base = Path(directory).relative_to(source_root) + entries.update(base / name for name in (*directories, *files)) + return entries + + +def _owned_by_root(info: tarfile.TarInfo) -> tarfile.TarInfo: + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + return info + + +def _build_record(build: ComponentBuild, host: _BuildHost) -> dict[str, object]: + """``spec.build`` of the metadata: the steps as run and the tool versions.""" + return { + "steps": [ + { + "workingDirectory": step.working_directory.value, + "command": list(step.command.value), + } + for step in build.steps + ], + "tools": {name: tool.version for name, tool in host.tools.items()}, + } diff --git a/engine/src/cloudfall_engine/cli.py b/engine/src/cloudfall_engine/cli.py index 0e02358..cfb1c0e 100644 --- a/engine/src/cloudfall_engine/cli.py +++ b/engine/src/cloudfall_engine/cli.py @@ -47,8 +47,10 @@ from treaty import ( App, Arg, + CliExit, Ctx, Exit, + ExitCodeName, External, Flag, Out, @@ -63,6 +65,7 @@ GIT_REF_PATTERN, GIT_TIMEOUT_SECONDS, ArtifactBuildError, + BuildStepTimeoutError, build_artifact, ) from cloudfall_engine.playbook import ( @@ -79,6 +82,9 @@ from cloudfall.secrets import OwnedRolePassword + from cloudfall.domain import BuildTimeoutSeconds + from treaty import Completed + app = App( "cloudfall-engine", version=version("cloudfall"), @@ -262,13 +268,17 @@ class BuiltRelease: @artifact.command( "build", - description="Clone, package, and hash one component release", + description=( + "Clone, build (the component's declared steps), package, and hash " + "one component release" + ), danger_level="mutating", exit_codes=["PROJECT_INVALID", "CONFIG_INVALID", "ARTIFACT_BUILD_FAILED"], timeout=3 * GIT_TIMEOUT_SECONDS, subprocess=Subprocess("git"), required_tools={"git": GIT_MINIMUM_VERSION}, supports_raw_payload=True, + child_log=True, examples=[ ( "Package the main branch of one component", @@ -277,7 +287,10 @@ class BuiltRelease: ], ) def build(args: BuildArgs, ctx: Ctx, project: Project) -> BuiltRelease: - """Clone the component at ``--ref`` and package it under the project.""" + """Clone the component at ``--ref``, build it, and package it. + + Build step output streams to stderr, never into the envelope on stdout. + """ try: built = build_artifact( project.inventory, @@ -288,10 +301,12 @@ def build(args: BuildArgs, ctx: Ctx, project: Project) -> BuiltRelease: run=partial( ctx.run, timeout=Timeout(GIT_TIMEOUT_SECONDS), check=False ), + run_step=partial(_run_build_step, ctx), + environment=dict(os.environ), ) except ArtifactBuildError as error: raise Exit.ARTIFACT_BUILD_FAILED( - error.detail, context={"code": error.code} + error.detail, context={"code": error.code, **error.context} ) from error except ConfigValidationError as error: raise _config_invalid(error) from error @@ -309,6 +324,27 @@ def build(args: BuildArgs, ctx: Ctx, project: Project) -> BuiltRelease: ) +def _run_build_step( + ctx: Ctx, argv: Sequence[str], *, cwd: Path, timeout: BuildTimeoutSeconds +) -> Completed: + """Run one build step, its output streamed to stderr as plain text.""" + try: + return ctx.run( + argv, + cwd=cwd, + timeout=Timeout(timeout.value), + check=False, + stream="always", + ) + except CliExit as error: + # Only the step's own limit is a step failure; running out of the + # command's time stays treaty's TIMEOUT. + own_limit = error.context.get("timeout_ms") == timeout.value * 1000 + if error.name == ExitCodeName("TIMEOUT") and own_limit: + raise BuildStepTimeoutError(error.message) from error + raise + + playbook = app.group( "playbook", description="Run Ansible playbooks with the engine configuration" ) diff --git a/engine/tests/test_artifact_build_steps.py b/engine/tests/test_artifact_build_steps.py new file mode 100644 index 0000000..01f8e20 --- /dev/null +++ b/engine/tests/test_artifact_build_steps.py @@ -0,0 +1,663 @@ +"""Component build steps in ``artifact build`` (S-002). + +Every build runs ``cloudfall-engine artifact build`` as a child process with +its own environment: ``PATH`` holds only a temporary directory with a ``git`` +that logs its argv before running the real one, ``env``, and stand-ins for +``node`` and ``pnpm``, so the tests need neither installed. The deploy test +runs the deploy and rollback role's own unpack and switch tasks under +``ansible-playbook`` against a directory on the controller. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +import sys +import tarfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +import pytest +import yaml +from ansible.parsing.dataloader import DataLoader +from cloudfall.validation import SchemaCatalog +from cloudfall_engine.cli import main + +ROOT = Path(__file__).parents[2] +SCHEMAS = ROOT / "config" / "schemas" / "v1" +EXAMPLES = ROOT / "config" / "examples" +ENGINE = ROOT / "engine" +ROLE = ENGINE / "ansible" / "roles" / "cloudfall_deploy" +_GIT = shutil.which("git") or "git" +_ENV = shutil.which("env") or "/usr/bin/env" +ARTIFACT_BUILD_FAILED = 81 +SECRET = "hcloud-secret-token-value" # noqa: S105 - a planted test value. +INSTALL = ["pnpm", "install", "--frozen-lockfile"] +BUILD = ["pnpm", "build"] + +GIT_WRAPPER = """#!{python} +import os, sys +with open({log!r}, "a", encoding="utf-8") as log: + log.write(" ".join(sys.argv[1:]) + "\\n") +os.execv({git!r}, [{git!r}, *sys.argv[1:]]) +""" + +NODE = """#!{python} +import sys +if sys.argv[1:] == ["--version"]: + print({version!r}) + sys.exit(0) +sys.exit(64) +""" + +PNPM = """#!{python} +import json, os, pathlib, sys +arguments = sys.argv[1:] +if arguments == ["--version"]: + print({version!r}) + sys.exit(0) +if arguments[0] == "install": + print("installing packages to stdout") + modules = pathlib.Path("node_modules/left-pad") + modules.mkdir(parents=True) + (modules / "index.js").write_text("module.exports = 1\\n") +elif arguments[0] == "build": + print("building to stdout") + dist = pathlib.Path("dist") + dist.mkdir(exist_ok=True) + (dist / "index.html").write_text(pathlib.Path("src/index.html").read_text()) + (dist / "env.json").write_text(json.dumps(dict(os.environ))) +elif arguments[0] == "fail": + print("compiling to stdout", flush=True) + print("error TS2322: boom", file=sys.stderr) + sys.exit(3) +elif arguments[0] == "sleep": + import time + time.sleep(60) +elif arguments[0] == "noop": + print("nothing to do") +else: + sys.exit(64) +""" + + +def _git(*arguments: str) -> str: + completed = subprocess.run( # noqa: S603 - fixture repository setup. + [ + _GIT, + "-c", + "user.email=test@example.test", + "-c", + "user.name=Cloudfall Test", + *arguments, + ], + capture_output=True, + check=True, + text=True, + ) + return completed.stdout + + +def _executable(path: Path, text: str) -> None: + path.write_text(text, encoding="utf-8") + path.chmod(0o755) + + +@dataclass(frozen=True, slots=True) +class _Fixture: + """A component repository, its project, and the controller ``PATH``.""" + + root: Path + + @property + def repository(self) -> Path: + return self.root / "repository" + + @property + def project(self) -> Path: + return self.root / "config" + + @property + def bin(self) -> Path: + return self.root / "bin" + + @property + def artifacts(self) -> Path: + return self.root / "artifacts" + + @property + def git_log(self) -> Path: + return self.root / "git.log" + + def commit(self, page: str) -> str: + (self.repository / "web" / "src" / "index.html").write_text( + page, encoding="utf-8" + ) + _git("-C", str(self.repository), "add", ".") + _git("-C", str(self.repository), "commit", "--quiet", "--message", page) + return _git("-C", str(self.repository), "rev-parse", "HEAD").strip() + + def tracked(self, commit: str) -> list[str]: + listed = _git( + "-C", str(self.repository), "ls-tree", "-r", "--name-only", commit + ) + return listed.splitlines() + + def build(self, ref: str) -> tuple[int, dict[str, Any], str]: + """Run ``artifact build``; answer its exit code, envelope, and stderr.""" + completed = subprocess.run( # noqa: S603 - the CLI under test. + [ + sys.executable, + "-m", + "cloudfall_engine", + "artifact", + "build", + "crm-backend", + "--ref", + ref, + "--project", + str(self.project), + "--schemas", + str(SCHEMAS), + "--output-dir", + str(self.artifacts), + "--format", + "json", + ], + env={ + "PATH": str(self.bin), + "HOME": str(self.root / "home"), + "LANG": "C.UTF-8", + "HCLOUD_TOKEN": SECRET, + "SOPS_AGE_KEY": SECRET, + }, + capture_output=True, + check=False, + text=True, + ) + envelope = cast("dict[str, Any]", json.loads(completed.stdout)) + return completed.returncode, envelope, completed.stderr + + def built(self, ref: str) -> dict[str, Any]: + exit_code, envelope, _ = self.build(ref) + assert exit_code == 0, envelope["error"] + return cast("dict[str, Any]", envelope["data"]) + + def git_runs(self) -> set[str]: + """Every git subcommand the logging wrapper saw.""" + if not self.git_log.exists(): + return set() + lines = self.git_log.read_text(encoding="utf-8").splitlines() + return {word for line in lines for word in line.split()} + + def written(self) -> list[Path]: + if not self.artifacts.exists(): + return [] + return [ + path + for path in self.artifacts.rglob("*") + if path.name.endswith((".tar.gz", ".json")) + ] + + +def _fixture( # noqa: PLR0913 - one knob per criterion. + tmp_path: Path, + *, + steps: list[list[str]] | None = None, + include: list[str] | None = None, + build: bool = True, + node: str | None = "v22.3.0", + pnpm: str = "9.1.0", +) -> _Fixture: + fixture = _Fixture(tmp_path) + repository = fixture.repository + (repository / "web" / "src").mkdir(parents=True) + _git("-C", str(repository), "-c", "init.defaultBranch=main", "init", "--quiet") + (repository / "app.py").write_text("print('crm')\n", encoding="utf-8") + (repository / "pyproject.toml").write_text( + '[project]\nname = "crm-backend"\nversion = "1.0.0"\n', encoding="utf-8" + ) + (repository / ".gitignore").write_text( + "web/dist/\nweb/node_modules/\n", encoding="utf-8" + ) + (repository / "web" / "package.json").write_text("{}\n", encoding="utf-8") + fixture.commit("

v1

\n") + + shutil.copytree(EXAMPLES, fixture.project) + path = fixture.project / "components" / "crm-backend.yaml" + document = yaml.safe_load(path.read_text(encoding="utf-8")) + spec = document["spec"] + spec["repository"]["url"] = f"file://{repository}" + if build: + spec["buildTools"] = {"node": "22.0.0", "pnpm": "9.0.0"} + spec["build"] = [ + {"workingDirectory": "web", "command": command} + for command in (steps if steps is not None else [INSTALL, BUILD]) + ] + for step in spec["build"]: + if step["command"] == ["pnpm", "sleep"]: + step["timeoutSeconds"] = 1 + spec["artifact"] = {"include": include or ["web/dist"]} + path.write_text(yaml.safe_dump(document), encoding="utf-8") + + fixture.bin.mkdir() + (tmp_path / "home").mkdir() + python = sys.executable + _executable( + fixture.bin / "git", + GIT_WRAPPER.format(python=python, log=str(fixture.git_log), git=_GIT), + ) + (fixture.bin / "env").symlink_to(_ENV) + if node is not None: + _executable(fixture.bin / "node", NODE.format(python=python, version=node)) + _executable(fixture.bin / "pnpm", PNPM.format(python=python, version=pnpm)) + return fixture + + +def _members(archive: Path) -> dict[str, tarfile.TarInfo]: + with tarfile.open(archive) as opened: + return {member.name: member for member in opened.getmembers()} + + +def _read(archive: Path, name: str) -> str: + with tarfile.open(archive) as opened: + extracted = opened.extractfile(name) + assert extracted is not None + return extracted.read().decode("utf-8") + + +def _failure(envelope: dict[str, Any]) -> dict[str, Any]: + error = cast("dict[str, Any]", envelope["error"]) + return cast("dict[str, Any]", error["context"]) + + +def test_s002_2_the_tarball_holds_the_gitignored_build_output(tmp_path: Path) -> None: + fixture = _fixture(tmp_path) + commit = _git("-C", str(fixture.repository), "rev-parse", "HEAD").strip() + + built = fixture.built(commit) + + archive = Path(built["archive"]) + assert built["git_commit"] == commit + assert "./web/dist/index.html" in _members(archive) + assert _read(archive, "./web/dist/index.html") == "

v1

\n" + + +def test_s002_3_the_tarball_holds_every_tracked_file_and_no_node_modules( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + commit = _git("-C", str(fixture.repository), "rev-parse", "HEAD").strip() + + names = set(_members(Path(fixture.built(commit)["archive"]))) + + assert {f"./{path}" for path in fixture.tracked(commit)} <= names + assert not [name for name in names if "node_modules" in Path(name).parts] + assert all(".git" not in Path(name).parts for name in names) + + +def test_s002_4_the_metadata_records_the_steps_and_tool_versions( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + + metadata_path = Path(fixture.built("main")["metadata"]) + + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + assert metadata["spec"]["build"] == { + "steps": [ + {"workingDirectory": "web", "command": INSTALL}, + {"workingDirectory": "web", "command": BUILD}, + ], + "tools": {"node": "22.3.0", "pnpm": "9.1.0"}, + } + SchemaCatalog(SCHEMAS).validate_named("artifact.schema.json", metadata) + + +def test_s002_5_a_missing_tool_fails_before_any_clone(tmp_path: Path) -> None: + fixture = _fixture(tmp_path, node=None) + + exit_code, envelope, _ = fixture.build("main") + + assert exit_code == ARTIFACT_BUILD_FAILED + context = _failure(envelope) + assert context["code"] == "artifact_build_tool_missing" + assert context["tool"] == "node" + assert "node" in envelope["error"]["message"] + assert "clone" not in fixture.git_runs() + assert fixture.written() == [] + + +def test_s002_6_an_old_tool_names_the_tool_and_both_versions(tmp_path: Path) -> None: + fixture = _fixture(tmp_path, node="v20.11.1") + + exit_code, envelope, _ = fixture.build("main") + + assert exit_code == ARTIFACT_BUILD_FAILED + context = _failure(envelope) + assert context["code"] == "artifact_build_tool_unsupported" + assert context["tool"] == "node" + assert context["found"] == "20.11.1" + assert context["minimum"] == "22.0.0" + message = envelope["error"]["message"] + assert "node" in message + assert "20.11.1" in message + assert "22.0.0" in message + + +def test_s002_6_a_tool_that_prints_no_version_is_unsupported(tmp_path: Path) -> None: + fixture = _fixture(tmp_path, pnpm="unknown") + + exit_code, envelope, _ = fixture.build("main") + + assert exit_code == ARTIFACT_BUILD_FAILED + assert _failure(envelope)["code"] == "artifact_build_tool_unsupported" + assert _failure(envelope)["tool"] == "pnpm" + + +def test_s002_7_a_failing_step_names_itself_and_writes_nothing(tmp_path: Path) -> None: + fixture = _fixture(tmp_path, steps=[INSTALL, ["pnpm", "fail"], BUILD]) + + exit_code, envelope, stderr = fixture.build("main") + + assert exit_code == ARTIFACT_BUILD_FAILED + context = _failure(envelope) + assert context["code"] == "artifact_build_step_failed" + assert context["step"] == 1 + assert context["argv"] == ["pnpm", "fail"] + assert context["exitCode"] == 3 + message = envelope["error"]["message"] + assert "step 1 (pnpm fail) exited with code 3" in message + assert "error TS2322: boom" in message + assert "error TS2322: boom" in stderr + assert fixture.written() == [] + + +def test_s002_7_a_step_past_its_timeout_fails_the_build(tmp_path: Path) -> None: + fixture = _fixture(tmp_path, steps=[["pnpm", "sleep"], BUILD]) + + exit_code, envelope, _ = fixture.build("main") + + assert exit_code == ARTIFACT_BUILD_FAILED + context = _failure(envelope) + assert context["code"] == "artifact_build_step_failed" + assert context["step"] == 0 + assert context["timeoutSeconds"] == 1 + assert "1s timeout" in envelope["error"]["message"] + assert fixture.written() == [] + + +def test_s002_8_a_missing_include_path_writes_no_artifact(tmp_path: Path) -> None: + fixture = _fixture(tmp_path, steps=[INSTALL, ["pnpm", "noop"]]) + + exit_code, envelope, _ = fixture.build("main") + + assert exit_code == ARTIFACT_BUILD_FAILED + context = _failure(envelope) + assert context["code"] == "artifact_include_missing" + assert context["path"] == "web/dist" + assert fixture.written() == [] + + +def test_s002_9_step_stdout_stays_out_of_the_json_envelope(tmp_path: Path) -> None: + fixture = _fixture(tmp_path) + + exit_code, envelope, stderr = fixture.build("main") + + assert exit_code == 0, envelope["error"] + assert envelope["ok"] is True + assert "installing packages to stdout" in stderr + assert "building to stdout" in stderr + + +def test_s002_9_stdout_is_exactly_one_envelope(tmp_path: Path) -> None: + """``json.loads`` of the whole stdout fails on any stray line.""" + fixture = _fixture(tmp_path, steps=[INSTALL, ["pnpm", "fail"]]) + + exit_code, envelope, _ = fixture.build("main") + + assert exit_code == ARTIFACT_BUILD_FAILED + assert envelope["ok"] is False + + +def test_s002_10_a_step_sees_ci_and_none_of_the_controller_secrets( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + + archive = Path(fixture.built("main")["archive"]) + + environment = json.loads(_read(archive, "./web/dist/env.json")) + assert environment["CI"] == "true" + # macOS sets __CF_USER_TEXT_ENCODING in every new process itself. + allowed = {"PATH", "HOME", "LANG", "TMPDIR", "CI", "__CF_USER_TEXT_ENCODING"} + assert set(environment) <= allowed + assert "HCLOUD_TOKEN" not in environment + assert "SOPS_AGE_KEY" not in environment + assert SECRET not in json.dumps(environment) + assert environment["PATH"] == str(fixture.bin) + assert environment["HOME"] == str(tmp_path / "home") + assert environment["LANG"] == "C.UTF-8" + + +def test_s002_13_a_component_without_build_packages_the_clean_tree( + tmp_path: Path, +) -> None: + """Without ``build`` the archive is the clean checkout as before. + + Every tracked file and directory, nothing else, and no ``spec.build``. + """ + fixture = _fixture(tmp_path, build=False) + commit = _git("-C", str(fixture.repository), "rev-parse", "HEAD").strip() + + built = fixture.built("main") + + tracked = [Path(path) for path in fixture.tracked(commit)] + expected = {"."} + for path in tracked: + expected.add(f"./{path.as_posix()}") + expected.update( + f"./{parent.as_posix()}" for parent in path.parents if parent != Path() + ) + assert set(_members(Path(built["archive"]))) == expected + metadata = json.loads(Path(built["metadata"]).read_text(encoding="utf-8")) + assert "build" not in metadata["spec"] + SchemaCatalog(SCHEMAS).validate_named("artifact.schema.json", metadata) + runs = fixture.git_runs() + assert "clone" in runs + assert "ls-files" not in runs + + +def test_s002_13_the_built_archive_lists_entries_like_the_plain_one( + tmp_path: Path, +) -> None: + """A build adds its include paths; every other entry is the plain tree's.""" + fixture = _fixture(tmp_path) + commit = _git("-C", str(fixture.repository), "rev-parse", "HEAD").strip() + + members = _members(Path(fixture.built(commit)["archive"])) + + tracked = {f"./{path}" for path in fixture.tracked(commit)} + built_output = { + "./web/dist", + "./web/dist/index.html", + "./web/dist/env.json", + } + directories = {".", "./web", "./web/src"} + assert set(members) == tracked | built_output | directories + assert {member.uname for member in members.values()} == {"root"} + + +# Deploy and rollback of a built release (S-002-12). + +DEPLOY_TASKS = ( + "Compute release paths", + "Create release and shared directories", + "Verify the transferred artifact digest", + "Refuse a release whose digest does not match its metadata", + "Unpack the verified release", + "Inspect the currently active release", + "Record the rollback candidate", + "Switch the current release symlink", +) +ROLLBACK_TASKS = ( + "Compute rollback paths", + "Inspect the rollback target release", + "Require the rollback target on the server", + "Switch the current release symlink to the rollback target", +) + + +def _gnu_tar() -> bool: + tar = shutil.which("gtar") or shutil.which("tar") + if tar is None: + return False + completed = subprocess.run( # noqa: S603 - probing the controller's tar. + [tar, "--version"], capture_output=True, check=False, text=True + ) + return "GNU tar" in completed.stdout + + +def _role_tasks(file: str, names: tuple[str, ...]) -> list[dict[str, object]]: + """Load the role's own tasks by name, with owner and group dropped. + + The role chowns to the application user as root; the test runs as the + controller's user, so the tasks keep their paths, digest check, unpack, + and symlink switch, and lose only the ownership. + """ + loaded = DataLoader().load_from_file(str(ROLE / "tasks" / file)) + assert isinstance(loaded, list) + flat: list[dict[str, object]] = [] + for task in loaded: + flat.append(task) + block = task.get("block") + if isinstance(block, list): + flat.extend(block) + tasks: list[dict[str, object]] = [] + for name in names: + # A JSON round trip drops Ansible's tagged types, which YAML can't dump. + task = json.loads(json.dumps(next(t for t in flat if t["name"] == name))) + for module in task.values(): + if isinstance(module, dict): + module.pop("owner", None) + module.pop("group", None) + tasks.append(task) + return tasks + + +def _play( + tmp_path: Path, + name: str, + variables: dict[str, object], + tasks: list[dict[str, object]], +) -> int: + inventory = tmp_path / "inventory.json" + inventory.write_text( + json.dumps( + { + "all": { + "hosts": { + "localhost": { + "ansible_connection": "local", + "ansible_python_interpreter": sys.executable, + } + } + } + } + ), + encoding="utf-8", + ) + playbook = tmp_path / f"{name}.yml" + playbook.write_text( + yaml.safe_dump( + [ + { + "name": name, + "hosts": "localhost", + "gather_facts": False, + "vars": variables, + "tasks": tasks, + } + ] + ), + encoding="utf-8", + ) + return main( + [ + "playbook", + "run", + str(playbook), + "--engine", + str(ENGINE), + "--inventory", + str(inventory), + ] + ) + + +def _deploy(tmp_path: Path, install_root: Path, built: dict[str, Any]) -> int: + archive = Path(built["archive"]) + assert hashlib.sha256(archive.read_bytes()).hexdigest() == built["archive_sha256"] + staged: dict[str, object] = { + "name": "Stage the artifact where the transfer would", + "ansible.builtin.set_fact": {"cloudfall_deploy_staging_archive": str(archive)}, + } + tasks = _role_tasks("main.yml", DEPLOY_TASKS) + return _play( + tmp_path, + f"deploy-{built['release']}", + { + "cloudfall_deploy_component": { + "id": "crm-backend", + "install_root": str(install_root), + "application_user": "crm", + }, + "cloudfall_deploy_release": built["release"], + "cloudfall_deploy_artifact_sha256": built["archive_sha256"], + }, + [tasks[0], staged, *tasks[1:]], + ) + + +def _rollback(tmp_path: Path, install_root: Path, release: str) -> int: + return _play( + tmp_path, + f"rollback-{release}", + { + "cloudfall_deploy_component": { + "id": "crm-backend", + "install_root": str(install_root), + }, + "cloudfall_deploy_release": release, + }, + _role_tasks("rollback.yml", ROLLBACK_TASKS), + ) + + +@pytest.mark.skipif(not _gnu_tar(), reason="Ansible's unarchive needs GNU tar") +def test_s002_12_deploy_and_rollback_carry_the_built_files(tmp_path: Path) -> None: + fixture = _fixture(tmp_path) + first = fixture.built(fixture.commit("

first

\n")) + second = fixture.built(fixture.commit("

second

\n")) + install_root = tmp_path / "srv" / "crm-backend" + current = install_root / "current" / "web" / "dist" / "index.html" + + assert _deploy(tmp_path, install_root, first) == 0 + assert _deploy(tmp_path, install_root, second) == 0 + + for built, page in ((first, "first"), (second, "second")): + index = install_root / "releases" / built["release"] / "web" / "dist" + assert (index / "index.html").read_text(encoding="utf-8") == ( + f"

{page}

\n" + ) + assert current.read_text(encoding="utf-8") == "

second

\n" + + assert _rollback(tmp_path, install_root, first["release"]) == 0 + + assert (install_root / "current").readlink() == ( + install_root / "releases" / first["release"] + ) + assert current.read_text(encoding="utf-8") == "

first

\n" diff --git a/engine/tests/test_artifact_builder.py b/engine/tests/test_artifact_builder.py index 842dce2..bfa68a0 100644 --- a/engine/tests/test_artifact_builder.py +++ b/engine/tests/test_artifact_builder.py @@ -19,6 +19,8 @@ if TYPE_CHECKING: from collections.abc import Sequence + from cloudfall.domain import BuildTimeoutSeconds + ROOT = Path(__file__).parents[2] SCHEMAS = ROOT / "config" / "schemas" / "v1" EXAMPLES = ROOT / "config" / "examples" @@ -94,6 +96,13 @@ def _git_never_runs(argv: Sequence[str]) -> NoReturn: raise AssertionError(message) +def _step_never_runs( + argv: Sequence[str], *, cwd: Path, timeout: BuildTimeoutSeconds +) -> NoReturn: + message = f"no build step runs for refused input: {argv} in {cwd} ({timeout})" + raise AssertionError(message) + + def test_builder_packages_a_hashed_release_with_metadata( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: @@ -155,6 +164,8 @@ def test_builder_rejects_an_unknown_component(tmp_path: Path) -> None: tmp_path / "artifacts", SCHEMAS, run=_git_never_runs, + run_step=_step_never_runs, + environment={}, ) assert error.value.code == "artifact_component_missing" @@ -171,6 +182,8 @@ def test_builder_rejects_an_unsafe_git_ref(tmp_path: Path) -> None: tmp_path / "artifacts", SCHEMAS, run=_git_never_runs, + run_step=_step_never_runs, + environment={}, ) assert error.value.code == "artifact_ref_invalid" diff --git a/sdk/src/cloudfall/app.py b/sdk/src/cloudfall/app.py index 29edc7c..1329a83 100644 --- a/sdk/src/cloudfall/app.py +++ b/sdk/src/cloudfall/app.py @@ -4334,8 +4334,9 @@ def converge_domain_routes( McpTool( name="build_artifact", description=( - "Clone, package, and hash one component release under the " - "artifacts directory, for deploy to ship" + "Clone, build (the component's declared build steps, on this " + "controller), package, and hash one component release under " + "the artifacts directory, for deploy to ship" ), input_schema={ "type": "object", From e0c7189b4ba52f69b2b1751feb7a9d09e70ead40 Mon Sep 17 00:00:00 2001 From: "shipmill-romamo[bot]" <338557544+shipmill-romamo[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:00:31 +0200 Subject: [PATCH 2/5] Bound artifact build by its declared step timeouts; refuse deleted tracked files A fixed 1800 s command limit cut long builds short with treaty's TIMEOUT, and the SDK/MCP path capped the engine call at 3600 s, so a step never reached its own timeoutSeconds. artifact build now declares no fixed limit and applies its own: 1800 s for the clone plus every declared step timeout (ComponentInventory.artifact_timeout_seconds), shared by every child, so a step's own limit is the one that fires (artifact_build_step_failed). An explicit --timeout still replaces it. The SDK's artifact step carries no cap of its own and leaves the limit to the engine. A step that deletes a tracked file broke S-002-3 by leaving it out of the archive. The build now refuses it with artifact_tracked_file_missing, naming the path, before any archive is written. Co-Authored-By: Claude Opus 5.5 --- changelog.d/115-component-build-steps.md | 2 +- docs/specs/002-component-build-steps.md | 4 +- engine/README.md | 7 +- engine/src/cloudfall_engine/artifact.py | 23 +++++-- engine/src/cloudfall_engine/cli.py | 78 +++++++++++++++++++-- engine/tests/test_artifact_build_steps.py | 84 ++++++++++++++++++++--- sdk/src/cloudfall/app.py | 3 +- sdk/src/cloudfall/domain.py | 2 + sdk/src/cloudfall/inventory.py | 9 +++ sdk/src/cloudfall/lifecycle.py | 17 +++-- sdk/tests/test_lifecycle.py | 47 +++++++++++++ 11 files changed, 239 insertions(+), 37 deletions(-) diff --git a/changelog.d/115-component-build-steps.md b/changelog.d/115-component-build-steps.md index fd8d639..9a39d08 100644 --- a/changelog.d/115-component-build-steps.md +++ b/changelog.d/115-component-build-steps.md @@ -1,3 +1,3 @@ ### Added -- `cloudfall-engine artifact build` runs a Component's declared `spec.build` steps on the controller before packaging: it checks every `spec.buildTools` entry is on `PATH` at its minimum version before cloning (`artifact_build_tool_missing`, `artifact_build_tool_unsupported`), runs each step in the clean checkout with only `PATH`, `HOME`, `LANG`, `TMPDIR` and `CI=true` in its environment and its output on stderr, fails on a non-zero exit or a timeout (`artifact_build_step_failed`) or a missing include path (`artifact_include_missing`) without writing an archive, packages the tracked files plus every `spec.artifact.include` path (a gitignored `web/dist`, say), and records the steps and tool versions as the new optional `spec.build` of the Artifact metadata. A Component without `build` is packaged as before (#115) +- `cloudfall-engine artifact build` runs a Component's declared `spec.build` steps on the controller before packaging: it checks every `spec.buildTools` entry is on `PATH` at its minimum version before cloning (`artifact_build_tool_missing`, `artifact_build_tool_unsupported`), runs each step in the clean checkout with only `PATH`, `HOME`, `LANG`, `TMPDIR` and `CI=true` in its environment and its output on stderr, fails without writing an archive on a non-zero exit or a step outrunning its own `timeoutSeconds` (`artifact_build_step_failed`), a missing include path (`artifact_include_missing`) or a tracked file a step deleted (`artifact_tracked_file_missing`), packages the tracked files plus every `spec.artifact.include` path (a gitignored `web/dist`, say), and records the steps and tool versions as the new optional `spec.build` of the Artifact metadata. The command's time limit is now the clone time (1800 seconds) plus every declared step timeout, on the CLI and the SDK/MCP path alike; an explicit `--timeout` replaces it. A Component without `build` is packaged as before (#115) diff --git a/docs/specs/002-component-build-steps.md b/docs/specs/002-component-build-steps.md index ffdc6a8..75b2f91 100644 --- a/docs/specs/002-component-build-steps.md +++ b/docs/specs/002-component-build-steps.md @@ -142,11 +142,11 @@ validate`, and the `build_artifact` MCP tool. - S-002-1: `test_s002_1_component_with_build_steps_validates`, `test_s002_1_existing_component_without_build_is_unchanged` and `test_s002_1_build_step_timeout_accepts_its_bounds` (`sdk/tests/test_component_build_validation.py`) passed, and `cloudfall config validate --project config/examples` still exits 0 - S-002-2: `test_s002_2_the_tarball_holds_the_gitignored_build_output` runs `cloudfall-engine artifact build --ref ` as a child process with stand-in `node` and `pnpm` on a temporary `PATH`; the tarball held `./web/dist/index.html` although the fixture's `.gitignore` lists `web/dist/` -- S-002-3: `test_s002_3_the_tarball_holds_every_tracked_file_and_no_node_modules` compared the tarball against `git ls-tree -r` at the commit: every tracked file, no `node_modules` entry, no `.git` +- S-002-3: `test_s002_3_the_tarball_holds_every_tracked_file_and_no_node_modules` compared the tarball against `git ls-tree -r` at the commit: every tracked file, no `node_modules` entry, no `.git`. A step that deletes a tracked file would break this, so the build refuses it: `test_s002_3_a_step_that_deletes_a_tracked_file_writes_no_artifact` exited 81 with the code `artifact_tracked_file_missing` naming `web/package.json`, and wrote nothing - S-002-4: `test_s002_4_the_metadata_records_the_steps_and_tool_versions` read `spec.build.steps` (both steps in order) and `spec.build.tools` `{"node": "22.3.0", "pnpm": "9.1.0"}` from the versions the stand-ins printed (`v22.3.0`, `9.1.0`), and validated the file against `artifact.schema.json` - S-002-5: `test_s002_5_a_missing_tool_fails_before_any_clone` exited 81 with `artifact_build_tool_missing` and `tool: node`; the logging `git` wrapper on `PATH` recorded no `clone`, and no file was written - S-002-6: `test_s002_6_an_old_tool_names_the_tool_and_both_versions` (`node` printing `v20.11.1` against `22.0.0`) and `test_s002_6_a_tool_that_prints_no_version_is_unsupported` exited 81 with `artifact_build_tool_unsupported`, naming the tool and both versions -- S-002-7: `test_s002_7_a_failing_step_names_itself_and_writes_nothing` exited 81 with `artifact_build_step_failed`, `step: 1`, `argv: ["pnpm", "fail"]`, `exitCode: 3` and the step's stderr tail; `test_s002_7_a_step_past_its_timeout_fails_the_build` covered a 1-second `timeoutSeconds`; neither left a `.tar.gz` or `.json` in the output directory +- S-002-7: `test_s002_7_a_failing_step_names_itself_and_writes_nothing` exited 81 with `artifact_build_step_failed`, `step: 1`, `argv: ["pnpm", "fail"]`, `exitCode: 3` and the step's stderr tail; `test_s002_7_a_step_past_its_timeout_fails_the_build` showed a step's own 2-second `timeoutSeconds` firing as `artifact_build_step_failed`; neither left a `.tar.gz` or `.json` in the output directory. The command's limit is the clone time (1800 s) plus every declared step timeout, on the CLI and on the SDK/MCP path, whose engine call carries no 3600 s cap of its own: `test_s002_7_the_run_limit_covers_every_declared_step` (three 3600 s steps), `test_s002_7_the_sdk_leaves_the_build_limit_to_the_engine`, and `test_s002_7_an_explicit_timeout_still_wins` (`--timeout 3` ends in treaty's `TIMEOUT`) - S-002-8: `test_s002_8_a_missing_include_path_writes_no_artifact` exited 81 with `artifact_include_missing` naming `web/dist` and wrote nothing - S-002-9: `test_s002_9_step_stdout_stays_out_of_the_json_envelope` and `test_s002_9_stdout_is_exactly_one_envelope` parsed the command's whole stdout as one JSON envelope, on success and on a failed step; the steps' stdout lines were on stderr - S-002-10: `test_s002_10_a_step_sees_ci_and_none_of_the_controller_secrets` set `HCLOUD_TOKEN` and `SOPS_AGE_KEY` for the command; the environment the build step dumped into the archive had `CI=true`, the controller's `PATH`, `HOME` and `LANG`, and neither secret diff --git a/engine/README.md b/engine/README.md index b0abf98..56ea3eb 100644 --- a/engine/README.md +++ b/engine/README.md @@ -178,9 +178,10 @@ runs in order in the clean checkout, with only `PATH`, `HOME`, `LANG` and SOPS keys stay out of `package.json` scripts. Step output goes to stderr. The archive holds the files git tracks plus the `artifact.include` paths, such as a gitignored `web/dist`, and the metadata records the steps and the tool -versions under `spec.build`. A failed step or a missing include path writes -no archive. The command's time limit (1800 seconds) covers the clone and every -step; pass `--timeout` for longer builds. +versions under `spec.build`. A failed or timed-out step, a missing include +path, or a tracked file a step deleted writes no archive. The command may take +1800 seconds for the clone plus every step's `timeoutSeconds`; an explicit +`--timeout` replaces that limit. `ansible/playbooks/deploy.yml` deploys one built release to the component's declared servers, one host at a time: diff --git a/engine/src/cloudfall_engine/artifact.py b/engine/src/cloudfall_engine/artifact.py index c3ac18f..58963f0 100644 --- a/engine/src/cloudfall_engine/artifact.py +++ b/engine/src/cloudfall_engine/artifact.py @@ -72,6 +72,7 @@ def __call__( _ERROR_TOOL_UNSUPPORTED = "artifact_build_tool_unsupported" _ERROR_STEP_FAILED = "artifact_build_step_failed" _ERROR_INCLUDE_MISSING = "artifact_include_missing" +_ERROR_TRACKED_FILE_MISSING = "artifact_tracked_file_missing" class ArtifactBuildError(RuntimeError): @@ -218,6 +219,7 @@ def build_artifact( # noqa: PLR0913 - the build inputs plus its runners. tracked = _tracked_files(run, source_root) _run_steps(build, host, source_root, run_step) _require_includes(build, source_root) + _require_tracked(tracked, source_root) release = ( f"{built_at.strftime('%Y%m%dT%H%M%SZ')}" @@ -493,6 +495,19 @@ def _require_includes(build: ComponentBuild, source_root: Path) -> None: ) +def _require_tracked(tracked: Sequence[str], source_root: Path) -> None: + """Refuse a build whose steps deleted a file git tracks at the commit.""" + for name in tracked: + if not os.path.lexists(source_root / name): + detail = ( + f"tracked file {name} was deleted by the build steps; the " + "archive must hold every file git tracks at the commit" + ) + raise ArtifactBuildError( + _ERROR_TRACKED_FILE_MISSING, detail, {"path": name} + ) + + def _write_built_archive( source_root: Path, archive_path: Path, @@ -502,13 +517,9 @@ def _write_built_archive( """Package the tracked files plus every ``artifact.include`` path. Whatever else the steps left behind (``node_modules/``, caches) stays out. - A tracked file a step deleted is not packaged. + ``_require_tracked`` has already checked every tracked file is there. """ - entries: set[Path] = set() - for name in tracked: - relative = Path(name) - if os.path.lexists(source_root / relative): - entries.add(relative) + entries: set[Path] = {Path(name) for name in tracked} for include in build.include: entries.update(_include_entries(source_root, Path(include.value))) for entry in tuple(entries): diff --git a/engine/src/cloudfall_engine/cli.py b/engine/src/cloudfall_engine/cli.py index cfb1c0e..5e2b88e 100644 --- a/engine/src/cloudfall_engine/cli.py +++ b/engine/src/cloudfall_engine/cli.py @@ -10,6 +10,7 @@ import json import os import shutil +import time from dataclasses import dataclass from functools import partial from importlib.metadata import version @@ -18,6 +19,7 @@ from cloudfall.commands import snake_case_keys from cloudfall.domain import ( + ARTIFACT_CLONE_TIMEOUT_SECONDS, AbsolutePath, PostgresDatabaseName, PostgresRoleName, @@ -274,7 +276,9 @@ class BuiltRelease: ), danger_level="mutating", exit_codes=["PROJECT_INVALID", "CONFIG_INVALID", "ARTIFACT_BUILD_FAILED"], - timeout=3 * GIT_TIMEOUT_SECONDS, + # The limit depends on the component: BuildBudget applies the clone time + # plus every declared step's timeout, unless --timeout is given. + timeout=None, subprocess=Subprocess("git"), required_tools={"git": GIT_MINIMUM_VERSION}, supports_raw_payload=True, @@ -290,7 +294,10 @@ def build(args: BuildArgs, ctx: Ctx, project: Project) -> BuiltRelease: """Clone the component at ``--ref``, build it, and package it. Build step output streams to stderr, never into the envelope on stdout. + The run may take the clone time plus every declared step's timeout; + an explicit ``--timeout`` replaces that limit. """ + budget = BuildBudget.start(ctx, _artifact_timeout(project, args.component)) try: built = build_artifact( project.inventory, @@ -298,10 +305,8 @@ def build(args: BuildArgs, ctx: Ctx, project: Project) -> BuiltRelease: args.ref, project.path(args.output_dir), args.schemas, - run=partial( - ctx.run, timeout=Timeout(GIT_TIMEOUT_SECONDS), check=False - ), - run_step=partial(_run_build_step, ctx), + run=partial(_run_git_or_tool, ctx, budget), + run_step=partial(_run_build_step, ctx, budget), environment=dict(os.environ), ) except ArtifactBuildError as error: @@ -324,15 +329,74 @@ def build(args: BuildArgs, ctx: Ctx, project: Project) -> BuiltRelease: ) +def _artifact_timeout(project: Project, component_id: ResourceId) -> int: + component = next( + ( + candidate + for candidate in project.inventory.components + if candidate.resource_id == component_id + ), + None, + ) + # An unknown component fails in build_artifact, before any child runs. + if component is None: + return ARTIFACT_CLONE_TIMEOUT_SECONDS + return component.artifact_timeout_seconds + + +@dataclass(frozen=True, slots=True) +class BuildBudget: + """The time limit of one ``artifact build`` run, shared by its children. + + ``deadline`` is ``None`` when ``--timeout`` was given: treaty's own + deadline then clamps every child, and this budget adds nothing. + """ + + seconds: int + deadline: float | None + + @classmethod + def start(cls, ctx: Ctx, seconds: int) -> Self: + """Start the clock, unless an explicit ``--timeout`` governs the run.""" + if ctx.remaining is not None: + return cls(seconds, None) + return cls(seconds, time.monotonic() + seconds) + + def timeout(self, seconds: int) -> Timeout: + """Cut a child's own limit to what is left of the run's.""" + if self.deadline is None: + return Timeout(seconds) + left = self.deadline - time.monotonic() + if left <= 0: + raise CliExit( + ExitCodeName("TIMEOUT"), + f"artifact build used its {self.seconds}s limit (the clone plus " + "every declared step timeout); pass --timeout for more", + context={"timeout_ms": self.seconds * 1000}, + ) + return Timeout(min(float(seconds), left)) + + +def _run_git_or_tool( + ctx: Ctx, budget: BuildBudget, argv: Sequence[str] +) -> Completed: + return ctx.run(argv, timeout=budget.timeout(GIT_TIMEOUT_SECONDS), check=False) + + def _run_build_step( - ctx: Ctx, argv: Sequence[str], *, cwd: Path, timeout: BuildTimeoutSeconds + ctx: Ctx, + budget: BuildBudget, + argv: Sequence[str], + *, + cwd: Path, + timeout: BuildTimeoutSeconds, ) -> Completed: """Run one build step, its output streamed to stderr as plain text.""" try: return ctx.run( argv, cwd=cwd, - timeout=Timeout(timeout.value), + timeout=budget.timeout(timeout.value), check=False, stream="always", ) diff --git a/engine/tests/test_artifact_build_steps.py b/engine/tests/test_artifact_build_steps.py index 01f8e20..072f00b 100644 --- a/engine/tests/test_artifact_build_steps.py +++ b/engine/tests/test_artifact_build_steps.py @@ -16,6 +16,7 @@ import subprocess import sys import tarfile +import time from dataclasses import dataclass from pathlib import Path from typing import Any, cast @@ -23,8 +24,10 @@ import pytest import yaml from ansible.parsing.dataloader import DataLoader -from cloudfall.validation import SchemaCatalog -from cloudfall_engine.cli import main +from cloudfall.domain import ARTIFACT_CLONE_TIMEOUT_SECONDS +from cloudfall.inventory import PlatformInventory +from cloudfall.validation import SchemaCatalog, validate_config +from cloudfall_engine.cli import BuildBudget, main ROOT = Path(__file__).parents[2] SCHEMAS = ROOT / "config" / "schemas" / "v1" @@ -79,6 +82,8 @@ time.sleep(60) elif arguments[0] == "noop": print("nothing to do") +elif arguments[0] == "clean": + pathlib.Path("package.json").unlink() else: sys.exit(64) """ @@ -146,7 +151,9 @@ def tracked(self, commit: str) -> list[str]: ) return listed.splitlines() - def build(self, ref: str) -> tuple[int, dict[str, Any], str]: + def build( + self, ref: str, *extra: str + ) -> tuple[int, dict[str, Any], str]: """Run ``artifact build``; answer its exit code, envelope, and stderr.""" completed = subprocess.run( # noqa: S603 - the CLI under test. [ @@ -166,6 +173,7 @@ def build(self, ref: str) -> tuple[int, dict[str, Any], str]: str(self.artifacts), "--format", "json", + *extra, ], env={ "PATH": str(self.bin), @@ -211,6 +219,7 @@ def _fixture( # noqa: PLR0913 - one knob per criterion. build: bool = True, node: str | None = "v22.3.0", pnpm: str = "9.1.0", + timeouts: dict[int, int] | None = None, ) -> _Fixture: fixture = _Fixture(tmp_path) repository = fixture.repository @@ -237,9 +246,8 @@ def _fixture( # noqa: PLR0913 - one knob per criterion. {"workingDirectory": "web", "command": command} for command in (steps if steps is not None else [INSTALL, BUILD]) ] - for step in spec["build"]: - if step["command"] == ["pnpm", "sleep"]: - step["timeoutSeconds"] = 1 + for index, seconds in (timeouts or {}).items(): + spec["build"][index]["timeoutSeconds"] = seconds spec["artifact"] = {"include": include or ["web/dist"]} path.write_text(yaml.safe_dump(document), encoding="utf-8") @@ -377,16 +385,72 @@ def test_s002_7_a_failing_step_names_itself_and_writes_nothing(tmp_path: Path) - def test_s002_7_a_step_past_its_timeout_fails_the_build(tmp_path: Path) -> None: - fixture = _fixture(tmp_path, steps=[["pnpm", "sleep"], BUILD]) + """The step's own ``timeoutSeconds`` fires, not the command's limit.""" + fixture = _fixture( + tmp_path, steps=[INSTALL, ["pnpm", "sleep"], BUILD], timeouts={1: 2} + ) exit_code, envelope, _ = fixture.build("main") assert exit_code == ARTIFACT_BUILD_FAILED context = _failure(envelope) assert context["code"] == "artifact_build_step_failed" - assert context["step"] == 0 - assert context["timeoutSeconds"] == 1 - assert "1s timeout" in envelope["error"]["message"] + assert context["step"] == 1 + assert context["argv"] == ["pnpm", "sleep"] + assert context["timeoutSeconds"] == 2 + assert "2s timeout" in envelope["error"]["message"] + assert fixture.written() == [] + + +def test_s002_7_the_run_limit_covers_every_declared_step(tmp_path: Path) -> None: + """The run allows the clone plus every declared step timeout. + + Three one-hour steps outlast both the old 1800 s command limit and the + SDK's 3600 s step limit. + """ + fixture = _fixture( + tmp_path, + steps=[INSTALL, BUILD, ["pnpm", "noop"]], + timeouts={0: 3600, 1: 3600, 2: 3600}, + ) + inventory = PlatformInventory.from_state( + validate_config(fixture.project, SCHEMAS) + ) + component = next( + c for c in inventory.components if c.resource_id.value == "crm-backend" + ) + + seconds = component.artifact_timeout_seconds + budget = BuildBudget(seconds, time.monotonic() + seconds) + + assert seconds == ARTIFACT_CLONE_TIMEOUT_SECONDS + 3 * 3600 + assert budget.timeout(3600).seconds == 3600 + + +def test_s002_7_an_explicit_timeout_still_wins(tmp_path: Path) -> None: + """``--timeout`` caps the run below a step's own limit: treaty's TIMEOUT.""" + fixture = _fixture(tmp_path, steps=[["pnpm", "sleep"], BUILD], timeouts={0: 30}) + + exit_code, envelope, _ = fixture.build("main", "--timeout", "3") + + assert exit_code != ARTIFACT_BUILD_FAILED + assert envelope["error"]["code"] == "TIMEOUT" + assert fixture.written() == [] + + +def test_s002_3_a_step_that_deletes_a_tracked_file_writes_no_artifact( + tmp_path: Path, +) -> None: + """Every file ``git ls-files`` lists must be in the tarball, or none is.""" + fixture = _fixture(tmp_path, steps=[INSTALL, ["pnpm", "clean"], BUILD]) + + exit_code, envelope, _ = fixture.build("main") + + assert exit_code == ARTIFACT_BUILD_FAILED + context = _failure(envelope) + assert context["code"] == "artifact_tracked_file_missing" + assert context["path"] == "web/package.json" + assert "web/package.json" in envelope["error"]["message"] assert fixture.written() == [] diff --git a/sdk/src/cloudfall/app.py b/sdk/src/cloudfall/app.py index 1329a83..a7687e2 100644 --- a/sdk/src/cloudfall/app.py +++ b/sdk/src/cloudfall/app.py @@ -164,7 +164,6 @@ ) from cloudfall.lifecycle import ( ERROR_EXECUTION_FAILED, - STEP_TIMEOUT_SECONDS, DeployOptions, DomainsRun, EngineContext, @@ -1793,7 +1792,7 @@ def run(step: ExecutionStep) -> StepRun: step.argv, env=step.environment, cwd=directory, - timeout=Timeout(STEP_TIMEOUT_SECONDS), + timeout=Timeout(step.timeout_seconds), check=False, ) diff --git a/sdk/src/cloudfall/domain.py b/sdk/src/cloudfall/domain.py index 8a56c9f..dffa315 100644 --- a/sdk/src/cloudfall/domain.py +++ b/sdk/src/cloudfall/domain.py @@ -29,6 +29,8 @@ _CONTROL_CHARACTER_PATTERN = re.compile(r"[\x00-\x1f\x7f]") BUILD_STEP_DEFAULT_TIMEOUT_SECONDS = 900 BUILD_STEP_MAXIMUM_TIMEOUT_SECONDS = 3600 +ARTIFACT_CLONE_TIMEOUT_SECONDS = 1800 +"""Time ``artifact build`` allows for cloning and packaging, besides the steps""" _FILE_MODE_PATTERN = re.compile(r"^0[0-7]{3}$") _SHA256_PATTERN = re.compile(r"^[a-f0-9]{64}$") _MAX_HOSTNAME_LENGTH = 253 diff --git a/sdk/src/cloudfall/inventory.py b/sdk/src/cloudfall/inventory.py index 557a4da..85c7791 100644 --- a/sdk/src/cloudfall/inventory.py +++ b/sdk/src/cloudfall/inventory.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, cast from cloudfall.domain import ( + ARTIFACT_CLONE_TIMEOUT_SECONDS, AbsolutePath, AlertDuration, AlertSeverity, @@ -593,6 +594,14 @@ class ComponentInventory: build: ComponentBuild | None = None release: ComponentRelease | None = None + @property + def artifact_timeout_seconds(self) -> int: + """How long ``artifact build`` may run: the clone plus every step's limit.""" + steps = () if self.build is None else self.build.steps + return ARTIFACT_CLONE_TIMEOUT_SECONDS + sum( + step.timeout_seconds.value for step in steps + ) + @property def release_server_id(self) -> ResourceId: """The server a release command runs on: the first deployment server.""" diff --git a/sdk/src/cloudfall/lifecycle.py b/sdk/src/cloudfall/lifecycle.py index 645575a..5c7f11f 100644 --- a/sdk/src/cloudfall/lifecycle.py +++ b/sdk/src/cloudfall/lifecycle.py @@ -82,6 +82,10 @@ def as_dict(self) -> dict[str, object]: } +STEP_TIMEOUT_SECONDS = 3600 +"""How long one step (an inventory render or a playbook run) may take.""" + + @dataclass(frozen=True, slots=True) class ExecutionStep: """One subprocess invocation with explicit environment additions.""" @@ -89,6 +93,8 @@ class ExecutionStep: description: str argv: tuple[str, ...] environment: Mapping[str, str] + timeout_seconds: int | None = STEP_TIMEOUT_SECONDS + """The runner's limit on the step; ``None`` for a step that bounds itself.""" class StepRun(Protocol): @@ -113,10 +119,6 @@ def stderr(self) -> str: type StepRunner = Callable[[ExecutionStep], StepRun] """Runs one step to completion without raising on a non-zero exit.""" -STEP_TIMEOUT_SECONDS = 3600 -"""How long one step (an inventory render or a playbook run) may take.""" - - def subprocess_step(step: ExecutionStep) -> StepRun: """Run one step as a child process, the runner outside treaty commands.""" try: @@ -125,11 +127,11 @@ def subprocess_step(step: ExecutionStep) -> StepRun: check=False, capture_output=True, text=True, - timeout=STEP_TIMEOUT_SECONDS, + timeout=step.timeout_seconds, env={**os.environ, **step.environment}, ) except subprocess.TimeoutExpired as error: - detail = f"{step.description} exceeded {STEP_TIMEOUT_SECONDS} seconds" + detail = f"{step.description} exceeded {step.timeout_seconds} seconds" raise LifecycleError(ERROR_EXECUTION_FAILED, detail) from error @@ -586,6 +588,9 @@ def build_release_artifact( description="build release artifact", argv=argv, environment={}, + # The engine limits the build to the clone time plus the component's + # declared step timeouts, which can add up past STEP_TIMEOUT_SECONDS. + timeout_seconds=None, ) output = _execute_step_with_output(step, context.run) # The engine answers one treaty envelope: the build is under ``data``. diff --git a/sdk/tests/test_lifecycle.py b/sdk/tests/test_lifecycle.py index 1a46b34..bc35a2b 100644 --- a/sdk/tests/test_lifecycle.py +++ b/sdk/tests/test_lifecycle.py @@ -16,11 +16,14 @@ from cloudfall.lifecycle import ( DeployOptions, EngineContext, + ExecutionStep, LifecycleError, + StepRun, build_release_artifact, deploy, plan_deploy, plan_health, + subprocess_step, verify_artifact, ) from cloudfall.validation import SchemaCatalog @@ -384,6 +387,50 @@ def test_release_artifact_build_reads_the_engine_envelope(tmp_path: Path) -> Non assert Path(str(built["archive"])).is_file() +def test_s002_7_the_sdk_leaves_the_build_limit_to_the_engine( + tmp_path: Path, +) -> None: + """The SDK and MCP path must not cut a build at ``STEP_TIMEOUT_SECONDS``. + + The engine allows the clone plus every declared step timeout, which can + add up past one hour, so the artifact step carries no limit of its own. + """ + recorded: list[ExecutionStep] = [] + + def record(step: ExecutionStep) -> StepRun: + recorded.append(step) + return subprocess_step(step) + + repository = tmp_path / "repository" + repository.mkdir() + _git("-C", str(repository), "-c", "init.defaultBranch=main", "init", "--quiet") + (repository / "app.py").write_text("print('crm')\n", encoding="utf-8") + _git("-C", str(repository), "add", ".") + _git("-C", str(repository), "commit", "--quiet", "--message", "initial") + project = tmp_path / "project" + shutil.copytree(EXAMPLES, project) + component = project / "components" / "crm-backend.yaml" + component.write_text( + component.read_text(encoding="utf-8").replace( + "https://github.com/example/crm-backend.git", f"file://{repository}" + ), + encoding="utf-8", + ) + context = EngineContext( + project_directory=project, + schema_directory=SCHEMAS, + engine_directory=ENGINE, + inventory_file=tmp_path / "inventory.json", + run=record, + ) + + build_release_artifact( + context, ResourceId("crm-backend"), "main", tmp_path / "artifacts" + ) + + assert [step.timeout_seconds for step in recorded] == [None] + + def test_backup_receipt_schema_accepts_the_playbook_shape() -> None: receipt = { "apiVersion": "cloudfall/v1", From cddeeb87e7ff8e07442c300a66a4a3ae5bebcf9b Mon Sep 17 00:00:00 2001 From: "shipmill-romamo[bot]" <338557544+shipmill-romamo[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:04:34 +0200 Subject: [PATCH 3/5] Keep artifact build --timeout 0 meaning no limit Treaty's --timeout reaches a handler only as the effective timeout, the same whether the flag was omitted or 0, so the computed limit silently replaced an explicit 0. The command now takes --timeout as a field of its own, which treaty allows in place of its framework flag, parsed with Timeout.parse: omitted applies the clone time plus every declared step timeout, 0 disables the limit, and any other value replaces it. Co-Authored-By: Claude Opus 5.5 --- changelog.d/115-component-build-steps.md | 2 +- docs/specs/002-component-build-steps.md | 2 +- engine/README.md | 2 +- engine/src/cloudfall_engine/cli.py | 50 +++++++++++++++++------ engine/tests/test_artifact_build_steps.py | 24 +++++++++++ 5 files changed, 64 insertions(+), 16 deletions(-) diff --git a/changelog.d/115-component-build-steps.md b/changelog.d/115-component-build-steps.md index 9a39d08..528f193 100644 --- a/changelog.d/115-component-build-steps.md +++ b/changelog.d/115-component-build-steps.md @@ -1,3 +1,3 @@ ### Added -- `cloudfall-engine artifact build` runs a Component's declared `spec.build` steps on the controller before packaging: it checks every `spec.buildTools` entry is on `PATH` at its minimum version before cloning (`artifact_build_tool_missing`, `artifact_build_tool_unsupported`), runs each step in the clean checkout with only `PATH`, `HOME`, `LANG`, `TMPDIR` and `CI=true` in its environment and its output on stderr, fails without writing an archive on a non-zero exit or a step outrunning its own `timeoutSeconds` (`artifact_build_step_failed`), a missing include path (`artifact_include_missing`) or a tracked file a step deleted (`artifact_tracked_file_missing`), packages the tracked files plus every `spec.artifact.include` path (a gitignored `web/dist`, say), and records the steps and tool versions as the new optional `spec.build` of the Artifact metadata. The command's time limit is now the clone time (1800 seconds) plus every declared step timeout, on the CLI and the SDK/MCP path alike; an explicit `--timeout` replaces it. A Component without `build` is packaged as before (#115) +- `cloudfall-engine artifact build` runs a Component's declared `spec.build` steps on the controller before packaging: it checks every `spec.buildTools` entry is on `PATH` at its minimum version before cloning (`artifact_build_tool_missing`, `artifact_build_tool_unsupported`), runs each step in the clean checkout with only `PATH`, `HOME`, `LANG`, `TMPDIR` and `CI=true` in its environment and its output on stderr, fails without writing an archive on a non-zero exit or a step outrunning its own `timeoutSeconds` (`artifact_build_step_failed`), a missing include path (`artifact_include_missing`) or a tracked file a step deleted (`artifact_tracked_file_missing`), packages the tracked files plus every `spec.artifact.include` path (a gitignored `web/dist`, say), and records the steps and tool versions as the new optional `spec.build` of the Artifact metadata. The command's time limit is now the clone time (1800 seconds) plus every declared step timeout, on the CLI and the SDK/MCP path alike; an explicit `--timeout` replaces it, and `--timeout 0` still disables it. A Component without `build` is packaged as before (#115) diff --git a/docs/specs/002-component-build-steps.md b/docs/specs/002-component-build-steps.md index 75b2f91..1557d2d 100644 --- a/docs/specs/002-component-build-steps.md +++ b/docs/specs/002-component-build-steps.md @@ -146,7 +146,7 @@ validate`, and the `build_artifact` MCP tool. - S-002-4: `test_s002_4_the_metadata_records_the_steps_and_tool_versions` read `spec.build.steps` (both steps in order) and `spec.build.tools` `{"node": "22.3.0", "pnpm": "9.1.0"}` from the versions the stand-ins printed (`v22.3.0`, `9.1.0`), and validated the file against `artifact.schema.json` - S-002-5: `test_s002_5_a_missing_tool_fails_before_any_clone` exited 81 with `artifact_build_tool_missing` and `tool: node`; the logging `git` wrapper on `PATH` recorded no `clone`, and no file was written - S-002-6: `test_s002_6_an_old_tool_names_the_tool_and_both_versions` (`node` printing `v20.11.1` against `22.0.0`) and `test_s002_6_a_tool_that_prints_no_version_is_unsupported` exited 81 with `artifact_build_tool_unsupported`, naming the tool and both versions -- S-002-7: `test_s002_7_a_failing_step_names_itself_and_writes_nothing` exited 81 with `artifact_build_step_failed`, `step: 1`, `argv: ["pnpm", "fail"]`, `exitCode: 3` and the step's stderr tail; `test_s002_7_a_step_past_its_timeout_fails_the_build` showed a step's own 2-second `timeoutSeconds` firing as `artifact_build_step_failed`; neither left a `.tar.gz` or `.json` in the output directory. The command's limit is the clone time (1800 s) plus every declared step timeout, on the CLI and on the SDK/MCP path, whose engine call carries no 3600 s cap of its own: `test_s002_7_the_run_limit_covers_every_declared_step` (three 3600 s steps), `test_s002_7_the_sdk_leaves_the_build_limit_to_the_engine`, and `test_s002_7_an_explicit_timeout_still_wins` (`--timeout 3` ends in treaty's `TIMEOUT`) +- S-002-7: `test_s002_7_a_failing_step_names_itself_and_writes_nothing` exited 81 with `artifact_build_step_failed`, `step: 1`, `argv: ["pnpm", "fail"]`, `exitCode: 3` and the step's stderr tail; `test_s002_7_a_step_past_its_timeout_fails_the_build` showed a step's own 2-second `timeoutSeconds` firing as `artifact_build_step_failed`; neither left a `.tar.gz` or `.json` in the output directory. The command's limit is the clone time (1800 s) plus every declared step timeout, on the CLI and on the SDK/MCP path, whose engine call carries no 3600 s cap of its own: `test_s002_7_the_run_limit_covers_every_declared_step` (three 3600 s steps), `test_s002_7_the_sdk_leaves_the_build_limit_to_the_engine`, `test_s002_7_an_explicit_timeout_still_wins` (`--timeout 3` ends in `TIMEOUT`), `test_s002_7_without_timeout_the_computed_limit_applies`, and `test_s002_7_timeout_zero_still_means_no_limit` (`--timeout 0` applies no limit, as before). The command takes `--timeout` as a field of its own, which treaty allows in place of its framework flag, so the build can tell an omitted flag apart from 0 - S-002-8: `test_s002_8_a_missing_include_path_writes_no_artifact` exited 81 with `artifact_include_missing` naming `web/dist` and wrote nothing - S-002-9: `test_s002_9_step_stdout_stays_out_of_the_json_envelope` and `test_s002_9_stdout_is_exactly_one_envelope` parsed the command's whole stdout as one JSON envelope, on success and on a failed step; the steps' stdout lines were on stderr - S-002-10: `test_s002_10_a_step_sees_ci_and_none_of_the_controller_secrets` set `HCLOUD_TOKEN` and `SOPS_AGE_KEY` for the command; the environment the build step dumped into the archive had `CI=true`, the controller's `PATH`, `HOME` and `LANG`, and neither secret diff --git a/engine/README.md b/engine/README.md index 56ea3eb..edaec93 100644 --- a/engine/README.md +++ b/engine/README.md @@ -181,7 +181,7 @@ a gitignored `web/dist`, and the metadata records the steps and the tool versions under `spec.build`. A failed or timed-out step, a missing include path, or a tracked file a step deleted writes no archive. The command may take 1800 seconds for the clone plus every step's `timeoutSeconds`; an explicit -`--timeout` replaces that limit. +`--timeout` replaces that limit, and `--timeout 0` disables it. `ansible/playbooks/deploy.yml` deploys one built release to the component's declared servers, one host at a time: diff --git a/engine/src/cloudfall_engine/cli.py b/engine/src/cloudfall_engine/cli.py index 5e2b88e..346211e 100644 --- a/engine/src/cloudfall_engine/cli.py +++ b/engine/src/cloudfall_engine/cli.py @@ -250,6 +250,27 @@ class BuildArgs(ProjectArgs): default=Path("tmp/artifacts"), description="Artifact output directory, relative to the project", ) + # Its own field, in place of treaty's --timeout (REQ-C-012), so the + # handler can tell "not passed" (the computed limit) apart from 0. + timeout: float | None = Flag( + default=None, + description=( + "Seconds before the build aborts with TIMEOUT; 0 disables the limit " + "(default: 1800 for the clone plus every declared step's " + "timeoutSeconds)" + ), + ) + + def __post_init__(self) -> None: + """Refuse a ``--timeout`` treaty's own flag would refuse.""" + self.limit() + + def limit(self) -> Timeout | None: + """Return the explicit ``--timeout``, ``None`` when not passed. + + ``Timeout(None)`` for 0, which disables the limit. + """ + return None if self.timeout is None else Timeout.parse(self.timeout) @dataclass(frozen=True, slots=True) @@ -277,7 +298,7 @@ class BuiltRelease: danger_level="mutating", exit_codes=["PROJECT_INVALID", "CONFIG_INVALID", "ARTIFACT_BUILD_FAILED"], # The limit depends on the component: BuildBudget applies the clone time - # plus every declared step's timeout, unless --timeout is given. + # plus every declared step's timeout, or BuildArgs.timeout when passed. timeout=None, subprocess=Subprocess("git"), required_tools={"git": GIT_MINIMUM_VERSION}, @@ -297,7 +318,9 @@ def build(args: BuildArgs, ctx: Ctx, project: Project) -> BuiltRelease: The run may take the clone time plus every declared step's timeout; an explicit ``--timeout`` replaces that limit. """ - budget = BuildBudget.start(ctx, _artifact_timeout(project, args.component)) + budget = BuildBudget.start( + args.limit(), _artifact_timeout(project, args.component) + ) try: built = build_artifact( project.inventory, @@ -348,31 +371,32 @@ def _artifact_timeout(project: Project, component_id: ResourceId) -> int: class BuildBudget: """The time limit of one ``artifact build`` run, shared by its children. - ``deadline`` is ``None`` when ``--timeout`` was given: treaty's own - deadline then clamps every child, and this budget adds nothing. + ``seconds`` is ``None`` and ``deadline`` too when ``--timeout 0`` + disabled the limit; each child then keeps only its own. """ - seconds: int + seconds: float | None deadline: float | None @classmethod - def start(cls, ctx: Ctx, seconds: int) -> Self: - """Start the clock, unless an explicit ``--timeout`` governs the run.""" - if ctx.remaining is not None: - return cls(seconds, None) + def start(cls, explicit: Timeout | None, computed: int) -> Self: + """Start the clock: ``--timeout`` when passed, else ``computed``.""" + seconds = float(computed) if explicit is None else explicit.seconds + if seconds is None: + return cls(None, None) return cls(seconds, time.monotonic() + seconds) def timeout(self, seconds: int) -> Timeout: """Cut a child's own limit to what is left of the run's.""" - if self.deadline is None: + if self.deadline is None or self.seconds is None: return Timeout(seconds) left = self.deadline - time.monotonic() if left <= 0: raise CliExit( ExitCodeName("TIMEOUT"), - f"artifact build used its {self.seconds}s limit (the clone plus " - "every declared step timeout); pass --timeout for more", - context={"timeout_ms": self.seconds * 1000}, + f"artifact build used its {self.seconds:g}s limit; pass a larger " + "--timeout, or 0 for none", + context={"timeout_ms": int(self.seconds * 1000)}, ) return Timeout(min(float(seconds), left)) diff --git a/engine/tests/test_artifact_build_steps.py b/engine/tests/test_artifact_build_steps.py index 072f00b..e30ef97 100644 --- a/engine/tests/test_artifact_build_steps.py +++ b/engine/tests/test_artifact_build_steps.py @@ -28,6 +28,7 @@ from cloudfall.inventory import PlatformInventory from cloudfall.validation import SchemaCatalog, validate_config from cloudfall_engine.cli import BuildBudget, main +from treaty import Timeout ROOT = Path(__file__).parents[2] SCHEMAS = ROOT / "config" / "schemas" / "v1" @@ -427,6 +428,29 @@ def test_s002_7_the_run_limit_covers_every_declared_step(tmp_path: Path) -> None assert budget.timeout(3600).seconds == 3600 +def test_s002_7_without_timeout_the_computed_limit_applies() -> None: + budget = BuildBudget.start(None, ARTIFACT_CLONE_TIMEOUT_SECONDS + 900) + + assert budget.seconds == ARTIFACT_CLONE_TIMEOUT_SECONDS + 900 + assert budget.deadline is not None + assert budget.timeout(900).seconds == 900 + + +def test_s002_7_timeout_zero_still_means_no_limit(tmp_path: Path) -> None: + """``--timeout 0`` disables the limit, as it does on every treaty command.""" + budget = BuildBudget.start(Timeout.parse(0), ARTIFACT_CLONE_TIMEOUT_SECONDS) + + assert budget.seconds is None + assert budget.deadline is None + assert budget.timeout(3600).seconds == 3600 + + fixture = _fixture(tmp_path) + exit_code, envelope, _ = fixture.build("main", "--timeout", "0") + + assert exit_code == 0, envelope["error"] + assert Path(envelope["data"]["archive"]).is_file() + + def test_s002_7_an_explicit_timeout_still_wins(tmp_path: Path) -> None: """``--timeout`` caps the run below a step's own limit: treaty's TIMEOUT.""" fixture = _fixture(tmp_path, steps=[["pnpm", "sleep"], BUILD], timeouts={0: 30}) From 3f9f4379be21eb720932fb91b16edc481738feed Mon Sep 17 00:00:00 2001 From: "shipmill-romamo[bot]" <338557544+shipmill-romamo[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:10:57 +0200 Subject: [PATCH 4/5] Refuse an artifact.include path that leaves the checkout through a symlink A tracked symlink above an include path (web/link -> /outside, include web/link/secrets) let the build package controller files outside the checkout. The include path itself may still be a symlink, archived as one. Co-Authored-By: Claude Opus 5.5 --- changelog.d/115-component-build-steps.md | 2 +- docs/specs/002-component-build-steps.md | 2 +- engine/README.md | 3 ++- engine/src/cloudfall_engine/artifact.py | 12 ++++++++++++ engine/tests/test_artifact_build_steps.py | 20 ++++++++++++++++++++ 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/changelog.d/115-component-build-steps.md b/changelog.d/115-component-build-steps.md index 528f193..c343739 100644 --- a/changelog.d/115-component-build-steps.md +++ b/changelog.d/115-component-build-steps.md @@ -1,3 +1,3 @@ ### Added -- `cloudfall-engine artifact build` runs a Component's declared `spec.build` steps on the controller before packaging: it checks every `spec.buildTools` entry is on `PATH` at its minimum version before cloning (`artifact_build_tool_missing`, `artifact_build_tool_unsupported`), runs each step in the clean checkout with only `PATH`, `HOME`, `LANG`, `TMPDIR` and `CI=true` in its environment and its output on stderr, fails without writing an archive on a non-zero exit or a step outrunning its own `timeoutSeconds` (`artifact_build_step_failed`), a missing include path (`artifact_include_missing`) or a tracked file a step deleted (`artifact_tracked_file_missing`), packages the tracked files plus every `spec.artifact.include` path (a gitignored `web/dist`, say), and records the steps and tool versions as the new optional `spec.build` of the Artifact metadata. The command's time limit is now the clone time (1800 seconds) plus every declared step timeout, on the CLI and the SDK/MCP path alike; an explicit `--timeout` replaces it, and `--timeout 0` still disables it. A Component without `build` is packaged as before (#115) +- `cloudfall-engine artifact build` runs a Component's declared `spec.build` steps on the controller before packaging: it checks every `spec.buildTools` entry is on `PATH` at its minimum version before cloning (`artifact_build_tool_missing`, `artifact_build_tool_unsupported`), runs each step in the clean checkout with only `PATH`, `HOME`, `LANG`, `TMPDIR` and `CI=true` in its environment and its output on stderr, fails without writing an archive on a non-zero exit or a step outrunning its own `timeoutSeconds` (`artifact_build_step_failed`), a missing include path (`artifact_include_missing`), an include path that leads out of the checkout through a symlink (`artifact_include_outside_checkout`) or a tracked file a step deleted (`artifact_tracked_file_missing`), packages the tracked files plus every `spec.artifact.include` path (a gitignored `web/dist`, say), and records the steps and tool versions as the new optional `spec.build` of the Artifact metadata. The command's time limit is now the clone time (1800 seconds) plus every declared step timeout, on the CLI and the SDK/MCP path alike; an explicit `--timeout` replaces it, and `--timeout 0` still disables it. A Component without `build` is packaged as before (#115) diff --git a/docs/specs/002-component-build-steps.md b/docs/specs/002-component-build-steps.md index 1557d2d..0d92208 100644 --- a/docs/specs/002-component-build-steps.md +++ b/docs/specs/002-component-build-steps.md @@ -147,7 +147,7 @@ validate`, and the `build_artifact` MCP tool. - S-002-5: `test_s002_5_a_missing_tool_fails_before_any_clone` exited 81 with `artifact_build_tool_missing` and `tool: node`; the logging `git` wrapper on `PATH` recorded no `clone`, and no file was written - S-002-6: `test_s002_6_an_old_tool_names_the_tool_and_both_versions` (`node` printing `v20.11.1` against `22.0.0`) and `test_s002_6_a_tool_that_prints_no_version_is_unsupported` exited 81 with `artifact_build_tool_unsupported`, naming the tool and both versions - S-002-7: `test_s002_7_a_failing_step_names_itself_and_writes_nothing` exited 81 with `artifact_build_step_failed`, `step: 1`, `argv: ["pnpm", "fail"]`, `exitCode: 3` and the step's stderr tail; `test_s002_7_a_step_past_its_timeout_fails_the_build` showed a step's own 2-second `timeoutSeconds` firing as `artifact_build_step_failed`; neither left a `.tar.gz` or `.json` in the output directory. The command's limit is the clone time (1800 s) plus every declared step timeout, on the CLI and on the SDK/MCP path, whose engine call carries no 3600 s cap of its own: `test_s002_7_the_run_limit_covers_every_declared_step` (three 3600 s steps), `test_s002_7_the_sdk_leaves_the_build_limit_to_the_engine`, `test_s002_7_an_explicit_timeout_still_wins` (`--timeout 3` ends in `TIMEOUT`), `test_s002_7_without_timeout_the_computed_limit_applies`, and `test_s002_7_timeout_zero_still_means_no_limit` (`--timeout 0` applies no limit, as before). The command takes `--timeout` as a field of its own, which treaty allows in place of its framework flag, so the build can tell an omitted flag apart from 0 -- S-002-8: `test_s002_8_a_missing_include_path_writes_no_artifact` exited 81 with `artifact_include_missing` naming `web/dist` and wrote nothing +- S-002-8: `test_s002_8_a_missing_include_path_writes_no_artifact` exited 81 with `artifact_include_missing` naming `web/dist` and wrote nothing; `test_s002_8_an_include_through_a_symlink_out_of_the_checkout_is_refused` exited 81 with `artifact_include_outside_checkout` for an include path under a tracked symlink to a directory outside the checkout, and wrote nothing - S-002-9: `test_s002_9_step_stdout_stays_out_of_the_json_envelope` and `test_s002_9_stdout_is_exactly_one_envelope` parsed the command's whole stdout as one JSON envelope, on success and on a failed step; the steps' stdout lines were on stderr - S-002-10: `test_s002_10_a_step_sees_ci_and_none_of_the_controller_secrets` set `HCLOUD_TOKEN` and `SOPS_AGE_KEY` for the command; the environment the build step dumped into the archive had `CI=true`, the controller's `PATH`, `HOME` and `LANG`, and neither secret - S-002-11: the `test_s002_11_*` tests in `sdk/tests/test_component_build_validation.py` passed (undeclared tool, absolute and `..` paths, `buildTools` or `artifact` without `build`, malformed steps) diff --git a/engine/README.md b/engine/README.md index edaec93..8b4620a 100644 --- a/engine/README.md +++ b/engine/README.md @@ -179,7 +179,8 @@ SOPS keys stay out of `package.json` scripts. Step output goes to stderr. The archive holds the files git tracks plus the `artifact.include` paths, such as a gitignored `web/dist`, and the metadata records the steps and the tool versions under `spec.build`. A failed or timed-out step, a missing include -path, or a tracked file a step deleted writes no archive. The command may take +path, an include path that leads out of the checkout through a symlink, or a +tracked file a step deleted writes no archive. The command may take 1800 seconds for the clone plus every step's `timeoutSeconds`; an explicit `--timeout` replaces that limit, and `--timeout 0` disables it. diff --git a/engine/src/cloudfall_engine/artifact.py b/engine/src/cloudfall_engine/artifact.py index 58963f0..f37b06e 100644 --- a/engine/src/cloudfall_engine/artifact.py +++ b/engine/src/cloudfall_engine/artifact.py @@ -72,6 +72,7 @@ def __call__( _ERROR_TOOL_UNSUPPORTED = "artifact_build_tool_unsupported" _ERROR_STEP_FAILED = "artifact_build_step_failed" _ERROR_INCLUDE_MISSING = "artifact_include_missing" +_ERROR_INCLUDE_OUTSIDE = "artifact_include_outside_checkout" _ERROR_TRACKED_FILE_MISSING = "artifact_tracked_file_missing" @@ -493,6 +494,17 @@ def _require_includes(build: ComponentBuild, source_root: Path) -> None: raise ArtifactBuildError( _ERROR_INCLUDE_MISSING, detail, {"path": include.value} ) + # The path itself may be a symlink (archived as one, never followed), + # but no directory above it may lead out of the checkout. + parent = (source_root / include.value).parent.resolve() + if not parent.is_relative_to(source_root.resolve()): + detail = ( + f"artifact.include path {include.value} leads out of the " + "checkout through a symlink" + ) + raise ArtifactBuildError( + _ERROR_INCLUDE_OUTSIDE, detail, {"path": include.value} + ) def _require_tracked(tracked: Sequence[str], source_root: Path) -> None: diff --git a/engine/tests/test_artifact_build_steps.py b/engine/tests/test_artifact_build_steps.py index e30ef97..c91bc32 100644 --- a/engine/tests/test_artifact_build_steps.py +++ b/engine/tests/test_artifact_build_steps.py @@ -490,6 +490,26 @@ def test_s002_8_a_missing_include_path_writes_no_artifact(tmp_path: Path) -> Non assert fixture.written() == [] +def test_s002_8_an_include_through_a_symlink_out_of_the_checkout_is_refused( + tmp_path: Path, +) -> None: + """A tracked symlink must not let an include path package outside files.""" + outside = tmp_path / "outside" / "secrets" + outside.mkdir(parents=True) + (outside / "id_rsa").write_text("PRIVATE KEY\n", encoding="utf-8") + fixture = _fixture(tmp_path, include=["web/dist", "web/link/secrets"]) + (fixture.repository / "web" / "link").symlink_to(tmp_path / "outside") + fixture.commit("

linked

\n") + + exit_code, envelope, _ = fixture.build("main") + + assert exit_code == ARTIFACT_BUILD_FAILED + context = _failure(envelope) + assert context["code"] == "artifact_include_outside_checkout" + assert context["path"] == "web/link/secrets" + assert fixture.written() == [] + + def test_s002_9_step_stdout_stays_out_of_the_json_envelope(tmp_path: Path) -> None: fixture = _fixture(tmp_path) From 65b1670762c29d85739c476fd7221dd70beb6166 Mon Sep 17 00:00:00 2001 From: "shipmill-romamo[bot]" <338557544+shipmill-romamo[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:21:27 +0200 Subject: [PATCH 5/5] Sort the engine CLI's type-only imports after the rebase onto #139 Co-Authored-By: Claude Opus 5.5 --- engine/src/cloudfall_engine/cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/engine/src/cloudfall_engine/cli.py b/engine/src/cloudfall_engine/cli.py index 346211e..0d11d86 100644 --- a/engine/src/cloudfall_engine/cli.py +++ b/engine/src/cloudfall_engine/cli.py @@ -82,9 +82,8 @@ if TYPE_CHECKING: from collections.abc import Callable, Mapping, Sequence - from cloudfall.secrets import OwnedRolePassword - from cloudfall.domain import BuildTimeoutSeconds + from cloudfall.secrets import OwnedRolePassword from treaty import Completed app = App(