From 270d09a06328d6d6f4a21f0140124a2b1254a3ef Mon Sep 17 00:00:00 2001 From: Jason Aricheta Date: Fri, 3 Jul 2026 14:06:07 +1200 Subject: [PATCH 1/2] feat: add direct AWS bootstrap and release tooling --- .github/workflows/aws-runner-target.yml | 22 ++ .github/workflows/ci.yml | 62 +++++ .github/workflows/release.yml | 66 +++++ .gitignore | 1 + CHANGELOG.md | 12 + README.md | 33 ++- docs/installation.md | 103 +++++++ docs/operations.md | 67 +++++ docs/security.md | 48 ++++ docs/testing.md | 53 ++++ examples/basic.yml | 3 +- package.json | 1 + runner-image/Dockerfile | 4 +- runner-image/supervisor.py | 84 +++++- runner-image/test/test_supervisor.py | 4 + scripts/bootstrap-aws.sh | 350 ++++++++++++++++++++++++ scripts/build-microvm-image.sh | 257 +++++++++++++++++ scripts/configure-github.sh | 74 +++++ scripts/package-runner-image.sh | 39 +++ scripts/test-runner-image.sh | 123 +++++++++ 20 files changed, 1392 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/aws-runner-target.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 CHANGELOG.md create mode 100644 docs/installation.md create mode 100644 docs/operations.md create mode 100644 docs/security.md create mode 100644 docs/testing.md create mode 100755 scripts/bootstrap-aws.sh create mode 100755 scripts/build-microvm-image.sh create mode 100755 scripts/configure-github.sh create mode 100755 scripts/package-runner-image.sh create mode 100755 scripts/test-runner-image.sh diff --git a/.github/workflows/aws-runner-target.yml b/.github/workflows/aws-runner-target.yml new file mode 100644 index 0000000..4a81877 --- /dev/null +++ b/.github/workflows/aws-runner-target.yml @@ -0,0 +1,22 @@ +name: AWS runner target + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + target: + runs-on: [self-hosted, lambda-microvm, e2e] + timeout-minutes: 15 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Verify host and Docker + run: | + set -euo pipefail + test "$(uname -m)" = "aarch64" + docker info + docker buildx version + docker compose version + docker run --rm public.ecr.aws/docker/library/busybox:1.37.0 true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2a11f44 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,62 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + action: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "24" + cache: npm + - run: npm ci + - run: npm run check + - name: Verify committed Action bundle + run: git diff --exit-code -- dist + - name: Check shell scripts + run: shellcheck scripts/*.sh + - name: Verify reproducible image artifact + run: | + scripts/package-runner-image.sh + first="$(sha256sum build/runner-image.zip | awk '{print $1}')" + scripts/package-runner-image.sh + second="$(sha256sum build/runner-image.zip | awk '{print $1}')" + test "${first}" = "${second}" + + runner-image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + with: + platforms: arm64 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - name: Build ARM64 runner image + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: runner-image + platforms: linux/arm64 + tags: lambda-microvm-github-runner:test + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Verify immutable image contents + run: | + docker run --rm --platform linux/arm64 \ + lambda-microvm-github-runner:test sh -lc ' + set -e + ! ldd /opt/actions-runner/bin/Runner.Listener | grep "not found" + test ! -S /var/run/docker.sock + /opt/actions-runner/bin/Runner.Listener --version + docker buildx version + docker compose version + aws --version + ' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f21f78d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,66 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "24" + cache: npm + - run: npm ci + - run: npm run check + - name: Verify committed Action bundle + run: git diff --exit-code -- dist + - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + with: + platforms: arm64 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - name: Build runner image + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: runner-image + platforms: linux/arm64 + tags: lambda-microvm-github-runner:release + load: true + - run: mkdir -p build + - name: Generate runner image SBOM + uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0 + with: + image: lambda-microvm-github-runner:release + format: cyclonedx-json + output-file: build/runner-image-sbom.cdx.json + upload-artifact: false + upload-release-assets: false + dependency-snapshot: false + - name: Package release artifacts + run: | + scripts/package-runner-image.sh + npm sbom --sbom-format=cyclonedx > build/action-sbom.cdx.json + sha256sum \ + build/runner-image.zip \ + build/runner-image-sbom.cdx.json \ + build/action-sbom.cdx.json \ + dist/index.js \ + > build/SHA256SUMS + - name: Create GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "${GITHUB_REF_NAME}" \ + --verify-tag \ + --generate-notes \ + --title "${GITHUB_REF_NAME}" \ + build/runner-image.zip \ + build/runner-image-sbom.cdx.json \ + build/action-sbom.cdx.json \ + build/SHA256SUMS diff --git a/.gitignore b/.gitignore index 09ee201..5cffe29 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules/ coverage/ +build/ __pycache__/ *.py[cod] *.log diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..003ecbc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +## Unreleased + +- Node.js 24 start/stop Action with strict validation. +- Repository-scoped, single-use JIT runners. +- Deterministic launch idempotency and quota-aware retries/polling. +- Partial-failure and explicit cleanup. +- Snapshot-safe AL2023 ARM64 runner image with Docker, Buildx, and Compose. +- Lifecycle supervisor with fresh Docker startup and self-termination. +- Direct AWS CLI bootstrap, image build tooling, CI, release SBOMs, and + examples. diff --git a/README.md b/README.md index 6a8e024..c47e649 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,28 @@ The Action implements and tests: - idempotent Lambda MicroVM launch, readiness, cleanup, and termination; - typed GitHub and AWS adapters with mocked-boundary integration tests. -The production AL2023 runner image is implemented and locally validated, -including nested Docker with the documented local `vfs` fallback. AWS image -build and private-repository end-to-end validation remain release gates. +The production AL2023 runner image is implemented and validated locally and +through the AWS image build hooks with production `overlay2`. Private-repository +end-to-end validation remains a release gate. + +## Minimal setup + +The setup is two direct scripts. It does not require an infrastructure +framework: + +```bash +export AWS_REGION=us-east-1 +export GITHUB_REPOSITORY=OWNER/PRIVATE_REPOSITORY + +scripts/bootstrap-aws.sh +scripts/build-microvm-image.sh +``` + +The first command idempotently creates the private S3 artifact bucket, +CloudWatch log groups, GitHub OIDC provider, and three least-privilege IAM +roles. It saves the discovered resource values to `build/aws-setup.json`. The +second command consumes that file automatically and saves the active image +details to `build/microvm-image.json`. ## Usage @@ -52,6 +71,14 @@ repositories with trusted workflow changes. It has no webhook, queue, dispatcher, warm pool, shell ingress, persistent runner, or boot-time package installation. +Detailed guides: + +- [installation](docs/installation.md) +- [security model](docs/security.md) +- [operations and quotas](docs/operations.md) +- [testing and release gates](docs/testing.md) +- [runner image](runner-image/README.md) + ## License MIT diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..37d6650 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,103 @@ +# Installation + +Version 1 is for private repositories with trusted workflow changes. It requires +an ARM64-capable Lambda MicroVM Region and an AWS account with enough MicroVM +memory quota for at least one 4 GiB runner. + +## 1. Create the AWS resources + +Use local AWS credentials that can create IAM roles, an IAM OIDC provider, an S3 +bucket, and CloudWatch log groups: + +```bash +export AWS_REGION=us-east-1 +export GITHUB_REPOSITORY=OWNER/PRIVATE_REPOSITORY + +scripts/bootstrap-aws.sh +``` + +This direct, idempotent script creates: + +- one private, encrypted, versioned S3 artifact bucket; +- build and runtime CloudWatch log groups with 30-day retention; +- the account-level GitHub Actions OIDC provider if it is absent; +- an image build role; +- a restricted MicroVM runtime role; +- a GitHub OIDC launch role trusted only for the repository's `main` branch. + +It writes the resulting values to `build/aws-setup.json`. Run it again to +reconcile the same resources. + +For a different default branch, set `GITHUB_DEFAULT_BRANCH`. For a GitHub +Environment or another exact OIDC subject, set `GITHUB_OIDC_SUBJECT` explicitly. +Do not use a wildcard subject for untrusted pull-request refs. + +No IAM user or stored AWS access key is needed by GitHub. + +## 2. Build the MicroVM image + +The build command reads `build/aws-setup.json` automatically: + +```bash +scripts/build-microvm-image.sh +``` + +It packages and uploads a content-addressed artifact, creates or updates the +image, waits for validation, activates the successful version, and keeps a +bounded rollback set. The active ARN and version are written to +`build/microvm-image.json`. + +## 3. Create a GitHub App + +Create and install a GitHub App only on the runner repository. Grant repository +Administration read/write permission so it can create, inspect, and delete JIT +runners. + +Record its App ID and download its private key. A compatible fine-grained PAT +can be passed directly, but short-lived installation tokens are preferred. + +## 4. Configure the GitHub repository + +With `gh auth status` working, set the generated AWS and image values: + +```bash +scripts/configure-github.sh +``` + +Then set the GitHub App credentials: + +```bash +gh variable set RUNNER_APP_ID --body APP_ID +gh secret set RUNNER_APP_PRIVATE_KEY < path/to/app.private-key.pem +``` + +Alternatively, configure everything in the helper invocation: + +```bash +RUNNER_APP_ID=APP_ID \ +RUNNER_APP_PRIVATE_KEY_FILE=path/to/app.private-key.pem \ +scripts/configure-github.sh +``` + +The helper creates these repository variables: + +- `MICROVM_AWS_REGION`; +- `MICROVM_LAUNCH_ROLE_ARN`; +- `MICROVM_EXECUTION_ROLE_ARN`; +- `MICROVM_RUNTIME_LOG_GROUP`; +- `MICROVM_RUNNER_IMAGE_ARN`; +- `MICROVM_RUNNER_IMAGE_VERSION`. + +Copy [the basic workflow](../examples/basic.yml) into +`.github/workflows/microvm-runner.yml`. Pin every third-party Action and this +Action to reviewed immutable commits before production use. + +## 5. Verify + +Run the workflow manually and confirm: + +1. start emits a unique label and MicroVM ID; +2. the target runs on ARM64 and `docker info`, Buildx, and Compose succeed; +3. the JIT runner processes only that job; +4. the MicroVM reaches `TERMINATED`; +5. no GitHub token or JIT payload appears in Actions or CloudWatch logs. diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..5353547 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,67 @@ +# Operations + +## Quotas + +MicroVM API and memory quotas are shared per AWS account and Region. The +documented baseline includes 5 `RunMicrovm` requests per second with burst 5, 10 +`TerminateMicrovm` requests per second with burst 10, and 100 `GetMicrovm` +requests per second with burst 100. + +The Action uses bounded full-jitter launch and termination retries, a stable +launch client token, randomized sequential polling, and immediate failure for +capacity exhaustion. For sustained rates above the launch quota, request a quota +increase or shape GitHub workflow concurrency. Do not add an internal queue to +this product. + +## Logs + +The AWS bootstrap creates build and runtime log groups with retention. Action +logs contain non-secret stage and resource metadata. Supervisor logs contain +lifecycle state, Docker failure tails, process exit codes, and cleanup outcomes, +but never hook bodies or JIT values. + +## Orphan audit + +Periodically inspect running and suspended MicroVMs: + +```bash +aws lambda-microvms list-microvms \ + --region us-east-1 \ + --query 'items[?state==`RUNNING` || state==`SUSPENDED`].[microvmId,state,startedAt,imageArn]' \ + --output table +``` + +Investigate runners near their maximum duration and terminate confirmed orphans. +Alert on repeated self-termination failures or VMs consistently reaching the +duration backstop. + +## Image updates and rollback + +The build script activates a version only after its `/ready` and `/validate` +hooks succeed. It then makes older versions inactive and retains a bounded +rollback set. Workflows should pin `image-version`. + +To roll back: + +```bash +aws lambda-microvms update-microvm-image-version \ + --image-identifier IMAGE_ARN \ + --image-version PREVIOUS_VERSION \ + --status ACTIVE +``` + +Update the repository's `MICROVM_RUNNER_IMAGE_VERSION` variable after the +version is active. + +## Common failures + +- `ServiceQuotaExceededException`: inspect regional MicroVM memory quota and + currently running/suspended VMs. +- image inactive or missing: verify the pinned ARN/version and activation + status. +- runner remains offline: inspect `/run`, Docker, DNS, and GitHub egress logs; + start cleanup should terminate the VM. +- Docker validation failure: production must use `overlay2`; never enable the + local `vfs` fallback in AWS. +- self-termination denied: correct the runtime role; the explicit stop job and + maximum duration remain active. diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..b2dc4fa --- /dev/null +++ b/docs/security.md @@ -0,0 +1,48 @@ +# Security + +## Supported trust boundary + +A Docker-capable self-hosted runner gives workflow code root-equivalent control +inside its isolated MicroVM. Version 1 therefore supports private repositories +and trusted workflow changes only. Public fork pull requests are unsupported. + +## Credentials + +- Start uses a short-lived GitHub App installation token. The token is masked + before validation or external work. +- The encoded JIT configuration and compressed payload are masked and never + included in errors, outputs, or supervisor logs. +- GitHub-hosted start and stop jobs obtain AWS credentials through OIDC. +- The default MicroVM execution role can write its logs and terminate runner + MicroVMs. It has no application deployment permissions. +- Deployment jobs should assume a separate role through GitHub OIDC. + +The GitHub App private key never enters the MicroVM. No long-lived AWS key is +required or documented. + +## Network + +Normal launches use managed `NO_INGRESS` and `INTERNET_EGRESS` connectors. There +is no public endpoint, shell token, or inbound debug path. Private VPC +connectors require a separate network and IAM review. + +## Cleanup + +Each execution has independent backstops: + +1. the supervisor terminates the MicroVM when the one JIT runner exits; +2. the final workflow job calls terminate idempotently; +3. start cleans up partial launches and unused JIT runners; +4. Lambda enforces `maximumDurationInSeconds`. + +## Supply chain + +- the Lambda AL2023 base is pinned by digest; +- runner, Docker, AWS CLI, Buildx, and Compose versions are pinned; +- downloaded runner and plugin binaries are SHA-256 verified; +- JavaScript dependencies and the committed bundle are lockfile-controlled; +- CI builds ARM64 and verifies the runner's native dependencies; +- releases attach Action and image SBOMs plus checksums. + +Update pins through a reviewed PR. Re-run local, CI, and AWS image validation +after every base, runner, Docker, or supervisor change. diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..d8ce54c --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,53 @@ +# Testing + +## Local gates + +```bash +npm ci +npm run check +shellcheck scripts/*.sh +scripts/package-runner-image.sh +npm run test:image +``` + +`npm run check` covers strict TypeScript, 53 Action tests, 16 supervisor tests, +and the bundled Action. Supervisor tests also run successfully under the image's +Python 3.9 runtime. + +`npm run test:image` requires an ARM64 Docker host. It verifies: + +- the image snapshot has no Docker socket; +- immutable runner, Buildx, Compose, and AWS CLI tools; +- asynchronous `/validate` with nested Docker and external registry DNS; +- local-only `vfs` fallback; +- `/run`, `/resume`, `/suspend`, and `/terminate`; +- runner process-group and Docker teardown; +- absence of the JIT fixture from logs. + +Local `vfs` success does not replace AWS `overlay2` validation. + +## AWS image gate + +Every candidate version must prove in AWS: + +- `/ready` snapshots without Docker or registered runner state; +- `/validate` starts Docker with `overlay2`; +- Lambda link-local DNS works in containers and BuildKit; +- Buildx builds ARM64; +- Compose Node/Redis bridge DNS and TCP work; +- published-port DNAT and container egress work; +- restart and suspend/resume preserve networking. + +## Private repository gate + +The manual `.github/workflows/aws-runner-target.yml` job waits for a runner with +the additional `e2e` label. Launch a candidate with that label to exercise +checkout, the ARM64 host, Docker, Buildx, Compose, and a nested container. + +Run successful and failing target jobs, Docker builds, service containers, +Compose, cancellation, startup timeout cleanup, duplicate launch retry, two +concurrent workflows, five concurrent starts, simulated throttling, capacity +failure, and denied self-termination fallback. + +Do not tag `v1` until logs have been checked for plaintext secrets and the full +private-repository matrix passes. diff --git a/examples/basic.yml b/examples/basic.yml index 17f3e95..01740b9 100644 --- a/examples/basic.yml +++ b/examples/basic.yml @@ -19,7 +19,7 @@ jobs: - uses: aws-actions/configure-aws-credentials@v6 with: role-to-assume: ${{ vars.MICROVM_LAUNCH_ROLE_ARN }} - aws-region: us-east-1 + aws-region: ${{ vars.MICROVM_AWS_REGION }} - uses: actions/create-github-app-token@v3 id: app-token @@ -35,6 +35,7 @@ jobs: image-id: ${{ vars.MICROVM_RUNNER_IMAGE_ARN }} image-version: ${{ vars.MICROVM_RUNNER_IMAGE_VERSION }} execution-role-arn: ${{ vars.MICROVM_EXECUTION_ROLE_ARN }} + cloudwatch-log-group: ${{ vars.MICROVM_RUNTIME_LOG_GROUP }} maximum-duration-seconds: "3600" job: diff --git a/package.json b/package.json index 20c87d9..c7f3d93 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "format:check": "prettier --check .", "lint": "eslint src test", "test": "vitest run", + "test:image": "scripts/test-runner-image.sh", "test:supervisor": "python3 -m unittest discover -s runner-image/test -p 'test_*.py'", "typecheck": "tsc --noEmit", "check": "npm run format:check && npm run lint && npm run typecheck && npm test && npm run test:supervisor && npm run build" diff --git a/runner-image/Dockerfile b/runner-image/Dockerfile index 8f724d2..795890d 100644 --- a/runner-image/Dockerfile +++ b/runner-image/Dockerfile @@ -77,8 +77,8 @@ RUN chmod 0755 /opt/runner-supervisor/supervisor.py ENV HOOK_PORT=9000 \ DOCKER_DNS=169.254.169.253 \ DOCKER_STORAGE_DRIVER=overlay2 \ - DOCKERD_START_ATTEMPTS=2 \ - DOCKERD_START_TIMEOUT=20 \ + DOCKERD_START_ATTEMPTS=1 \ + DOCKERD_START_TIMEOUT=50 \ ALLOW_VFS_FALLBACK=false \ RUNNER_ROOT=/opt/actions-runner \ RUNNER_USER=runner diff --git a/runner-image/supervisor.py b/runner-image/supervisor.py index 234debb..7691d66 100644 --- a/runner-image/supervisor.py +++ b/runner-image/supervisor.py @@ -27,6 +27,7 @@ MAX_HOOK_BODY_BYTES = 8_192 MAX_RUN_HOOK_PAYLOAD_BYTES = 4_096 MAX_JIT_CONFIG_BYTES = 1024 * 1024 +CONTAINERD_SOCKET = "/run/containerd/containerd.sock" def log(message: str) -> None: @@ -38,8 +39,8 @@ class Settings: hook_port: int = 9000 docker_dns: str = "169.254.169.253" docker_storage_driver: str = "overlay2" - docker_start_attempts: int = 2 - docker_start_timeout: int = 20 + docker_start_attempts: int = 1 + docker_start_timeout: int = 50 allow_vfs_fallback: bool = False docker_log: Path = Path("/tmp/dockerd.log") runner_root: Path = Path("/opt/actions-runner") @@ -58,10 +59,10 @@ def from_environment(cls) -> Settings: "DOCKER_STORAGE_DRIVER", "overlay2" ), docker_start_attempts=_integer_environment( - "DOCKERD_START_ATTEMPTS", 2, 1, 10 + "DOCKERD_START_ATTEMPTS", 1, 1, 10 ), docker_start_timeout=_integer_environment( - "DOCKERD_START_TIMEOUT", 20, 1, 300 + "DOCKERD_START_TIMEOUT", 50, 1, 300 ), allow_vfs_fallback=_boolean_environment( "ALLOW_VFS_FALLBACK", False @@ -102,6 +103,7 @@ def __init__( self.clock = clock self.sleeper = sleeper self.process: subprocess.Popen[bytes] | None = None + self.containerd_process: subprocess.Popen[bytes] | None = None self.lock = threading.RLock() def is_ready(self) -> bool: @@ -150,14 +152,26 @@ def start(self) -> bool: for driver in drivers: self._stop_process() Path("/var/run").mkdir(parents=True, exist_ok=True) + Path("/run/containerd").mkdir( + parents=True, exist_ok=True + ) Path("/var/lib/docker").mkdir( parents=True, exist_ok=True ) Path("/var/run/docker.sock").unlink(missing_ok=True) + Path(CONTAINERD_SOCKET).unlink(missing_ok=True) + + deadline = ( + self.clock() + self.settings.docker_start_timeout + ) + if not self._start_containerd(deadline): + self._log_tail() + continue command = [ "dockerd", "--host=unix:///var/run/docker.sock", + f"--containerd={CONTAINERD_SOCKET}", f"--storage-driver={driver}", "--exec-opt", "native.cgroupdriver=cgroupfs", @@ -178,9 +192,6 @@ def start(self) -> bool: stderr=subprocess.STDOUT, ) - deadline = ( - self.clock() + self.settings.docker_start_timeout - ) while self.clock() < deadline: if self.is_ready(): if self._driver_is_accepted(): @@ -192,7 +203,10 @@ def start(self) -> bool: self.process is not None and self.process.poll() is not None ): - log("dockerd exited before becoming ready") + log( + "dockerd exited before becoming ready " + f"(status={self.process.poll()})" + ) self._log_tail() break self.sleeper(1) @@ -210,6 +224,49 @@ def stop(self) -> None: with self.lock: self._stop_process() + def _start_containerd(self, deadline: float) -> bool: + command = [ + "containerd", + "--address", + CONTAINERD_SOCKET, + "--log-level", + "warn", + ] + log("starting containerd") + with self.settings.docker_log.open("ab") as output: + self.containerd_process = self.popen( + command, + stdout=output, + stderr=subprocess.STDOUT, + ) + + while self.clock() < deadline: + try: + result = self.run_command( + ["ctr", "--address", CONTAINERD_SOCKET, "version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=5, + check=False, + ) + if result.returncode == 0: + log("containerd ready") + return True + except (OSError, subprocess.TimeoutExpired): + pass + if ( + self.containerd_process is not None + and self.containerd_process.poll() is not None + ): + log( + "containerd exited before becoming ready " + f"(status={self.containerd_process.poll()})" + ) + return False + self.sleeper(1) + log("containerd readiness timed out") + return False + def _driver_is_accepted(self) -> bool: driver = self.storage_driver() return driver == self.settings.docker_storage_driver or ( @@ -225,6 +282,17 @@ def _stop_process(self) -> None: self.process.kill() self.process.wait(timeout=5) self.process = None + if ( + self.containerd_process is not None + and self.containerd_process.poll() is None + ): + self.containerd_process.terminate() + try: + self.containerd_process.wait(timeout=10) + except subprocess.TimeoutExpired: + self.containerd_process.kill() + self.containerd_process.wait(timeout=5) + self.containerd_process = None def _log_tail(self, lines: int = 40) -> None: try: diff --git a/runner-image/test/test_supervisor.py b/runner-image/test/test_supervisor.py index bf9db0c..7eb9ee0 100644 --- a/runner-image/test/test_supervisor.py +++ b/runner-image/test/test_supervisor.py @@ -169,6 +169,8 @@ def popen( command: list[str], **_kwargs: object ) -> FakeDockerProcess: nonlocal active_driver + if command[0] == "containerd": + return FakeDockerProcess(exited=False) active_driver = next( value.split("=", 1)[1] for value in command @@ -182,6 +184,8 @@ def popen( def run_command( command: list[str], **_kwargs: object ) -> subprocess.CompletedProcess[bytes]: + if command[0] == "ctr": + return subprocess.CompletedProcess(command, 0) if "--format" in command: return subprocess.CompletedProcess( command, diff --git a/scripts/bootstrap-aws.sh b/scripts/bootstrap-aws.sh new file mode 100755 index 0000000..2f25348 --- /dev/null +++ b/scripts/bootstrap-aws.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +REPOSITORY_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" +readonly REPOSITORY_ROOT + +readonly REGION="${AWS_REGION:-${AWS_DEFAULT_REGION:-}}" +readonly GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-}" +readonly PROJECT_NAME="${PROJECT_NAME:-lambda-microvm-github-runner}" +readonly GITHUB_DEFAULT_BRANCH="${GITHUB_DEFAULT_BRANCH:-main}" +readonly GITHUB_OIDC_SUBJECT="${GITHUB_OIDC_SUBJECT:-repo:${GITHUB_REPOSITORY}:ref:refs/heads/${GITHUB_DEFAULT_BRANCH}}" +readonly LOG_RETENTION_DAYS="${LOG_RETENTION_DAYS:-30}" +readonly OUTPUT_FILE="${OUTPUT_FILE:-${REPOSITORY_ROOT}/build/aws-setup.json}" + +log() { + echo "$*" >&2 +} + +fail() { + log "ERROR: $*" + exit 1 +} + +: "${REGION:?Set AWS_REGION or AWS_DEFAULT_REGION}" +: "${GITHUB_REPOSITORY:?Set GITHUB_REPOSITORY to owner/repository}" + +[[ "${GITHUB_REPOSITORY}" =~ ^[^/[:space:]]+/[^/[:space:]]+$ ]] || + fail "GITHUB_REPOSITORY must use owner/repository format" +[[ "${PROJECT_NAME}" =~ ^[A-Za-z0-9+=,.@_-]{1,48}$ ]] || + fail "PROJECT_NAME contains unsupported characters or is longer than 48 characters" +[[ "${LOG_RETENTION_DAYS}" =~ ^[0-9]+$ ]] || + fail "LOG_RETENTION_DAYS must be a positive integer" +((LOG_RETENTION_DAYS > 0)) || + fail "LOG_RETENTION_DAYS must be a positive integer" + +for command in aws jq mktemp sed tr; do + command -v "${command}" >/dev/null 2>&1 || + fail "Required command is unavailable: ${command}" +done + +export AWS_MAX_ATTEMPTS=6 +export AWS_RETRY_MODE=standard +export AWS_PAGER="" + +identity_json="$(aws sts get-caller-identity --region "${REGION}" --output json)" +readonly identity_json +account_id="$(jq -er '.Account' <<<"${identity_json}")" +readonly account_id +caller_arn="$(jq -er '.Arn' <<<"${identity_json}")" +readonly caller_arn +partition="${caller_arn#arn:}" +partition="${partition%%:*}" +readonly partition + +bucket_fragment="$( + tr '[:upper:]_' '[:lower:]-' <<<"${PROJECT_NAME}" | + sed -E 's/[^a-z0-9.-]+/-/g; s/^[^a-z0-9]+//; s/[^a-z0-9]+$//' +)" +readonly bucket_fragment +readonly ARTIFACT_BUCKET="${ARTIFACT_BUCKET:-${bucket_fragment}-${account_id}-${REGION}}" +[[ "${ARTIFACT_BUCKET}" =~ ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ ]] || + fail "ARTIFACT_BUCKET is not a valid S3 bucket name" + +readonly BUILD_ROLE_NAME="${BUILD_ROLE_NAME:-${PROJECT_NAME}-image-build}" +readonly EXECUTION_ROLE_NAME="${EXECUTION_ROLE_NAME:-${PROJECT_NAME}-runtime}" +readonly GITHUB_ROLE_NAME="${GITHUB_ROLE_NAME:-${PROJECT_NAME}-github-launch}" +readonly BUILD_LOG_GROUP="${BUILD_LOG_GROUP:-/lambda-microvms/${PROJECT_NAME}/build}" +readonly RUNTIME_LOG_GROUP="${RUNTIME_LOG_GROUP:-/lambda-microvms/${PROJECT_NAME}/runtime}" +readonly OIDC_PROVIDER_ARN="arn:${partition}:iam::${account_id}:oidc-provider/token.actions.githubusercontent.com" +readonly BUILD_ROLE_ARN="arn:${partition}:iam::${account_id}:role/${BUILD_ROLE_NAME}" +readonly EXECUTION_ROLE_ARN="arn:${partition}:iam::${account_id}:role/${EXECUTION_ROLE_NAME}" +readonly GITHUB_ROLE_ARN="arn:${partition}:iam::${account_id}:role/${GITHUB_ROLE_NAME}" +readonly IMAGE_RESOURCE_ARN="arn:${partition}:lambda:${REGION}:${account_id}:microvm-image:*" +readonly MICROVM_RESOURCE_ARN="arn:${partition}:lambda:${REGION}:${account_id}:microvm:*" +readonly INTERNET_EGRESS_ARN="arn:${partition}:lambda:${REGION}:aws:network-connector:aws-network-connector:INTERNET_EGRESS" +readonly NO_INGRESS_ARN="arn:${partition}:lambda:${REGION}:aws:network-connector:aws-network-connector:NO_INGRESS" + +temporary_directory="$(mktemp -d)" +cleanup() { + rm -rf "${temporary_directory}" +} +trap cleanup EXIT + +create_log_group() { + local log_group="$1" + local count + count="$( + aws logs describe-log-groups \ + --region "${REGION}" \ + --log-group-name-prefix "${log_group}" \ + --query "length(logGroups[?logGroupName=='${log_group}'])" \ + --output text + )" + if [[ "${count}" == "0" ]]; then + log "Creating CloudWatch log group ${log_group}" + aws logs create-log-group \ + --region "${REGION}" \ + --log-group-name "${log_group}" \ + --tags "Project=${PROJECT_NAME},ManagedBy=lambda-microvm-github-runner" + else + log "Using CloudWatch log group ${log_group}" + fi + aws logs put-retention-policy \ + --region "${REGION}" \ + --log-group-name "${log_group}" \ + --retention-in-days "${LOG_RETENTION_DAYS}" +} + +upsert_role() { + local role_name="$1" + local trust_policy="$2" + local permissions_policy="$3" + local description="$4" + + if aws iam get-role --role-name "${role_name}" >/dev/null 2>&1; then + log "Updating IAM role ${role_name}" + aws iam update-assume-role-policy \ + --role-name "${role_name}" \ + --policy-document "file://${trust_policy}" + aws iam tag-role \ + --role-name "${role_name}" \ + --tags \ + "Key=Project,Value=${PROJECT_NAME}" \ + "Key=ManagedBy,Value=lambda-microvm-github-runner" + else + log "Creating IAM role ${role_name}" + aws iam create-role \ + --role-name "${role_name}" \ + --description "${description}" \ + --assume-role-policy-document "file://${trust_policy}" \ + --tags \ + "Key=Project,Value=${PROJECT_NAME}" \ + "Key=ManagedBy,Value=lambda-microvm-github-runner" \ + >/dev/null + fi + + aws iam put-role-policy \ + --role-name "${role_name}" \ + --policy-name "${PROJECT_NAME}-permissions" \ + --policy-document "file://${permissions_policy}" + aws iam wait role-exists --role-name "${role_name}" +} + +if aws s3api head-bucket --bucket "${ARTIFACT_BUCKET}" 2>/dev/null; then + log "Using S3 bucket ${ARTIFACT_BUCKET}" +else + log "Creating S3 bucket ${ARTIFACT_BUCKET}" + if [[ "${REGION}" == "us-east-1" ]]; then + aws s3api create-bucket \ + --region "${REGION}" \ + --bucket "${ARTIFACT_BUCKET}" \ + >/dev/null + else + aws s3api create-bucket \ + --region "${REGION}" \ + --bucket "${ARTIFACT_BUCKET}" \ + --create-bucket-configuration "LocationConstraint=${REGION}" \ + >/dev/null + fi +fi + +aws s3api put-public-access-block \ + --region "${REGION}" \ + --bucket "${ARTIFACT_BUCKET}" \ + --public-access-block-configuration \ + 'BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true' +aws s3api put-bucket-encryption \ + --region "${REGION}" \ + --bucket "${ARTIFACT_BUCKET}" \ + --server-side-encryption-configuration \ + '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":false}]}' +aws s3api put-bucket-versioning \ + --region "${REGION}" \ + --bucket "${ARTIFACT_BUCKET}" \ + --versioning-configuration Status=Enabled +aws s3api put-bucket-tagging \ + --region "${REGION}" \ + --bucket "${ARTIFACT_BUCKET}" \ + --tagging "TagSet=[{Key=Project,Value=${PROJECT_NAME}},{Key=ManagedBy,Value=lambda-microvm-github-runner}]" + +create_log_group "${BUILD_LOG_GROUP}" +create_log_group "${RUNTIME_LOG_GROUP}" + +if oidc_json="$( + aws iam get-open-id-connect-provider \ + --open-id-connect-provider-arn "${OIDC_PROVIDER_ARN}" \ + --output json 2>/dev/null +)"; then + log "Using GitHub Actions OIDC provider ${OIDC_PROVIDER_ARN}" + if ! jq -e '.ClientIDList | index("sts.amazonaws.com")' \ + <<<"${oidc_json}" >/dev/null; then + aws iam add-client-id-to-open-id-connect-provider \ + --open-id-connect-provider-arn "${OIDC_PROVIDER_ARN}" \ + --client-id sts.amazonaws.com + fi +else + log "Creating GitHub Actions OIDC provider" + aws iam create-open-id-connect-provider \ + --url "https://token.actions.githubusercontent.com" \ + --client-id-list sts.amazonaws.com \ + --tags \ + "Key=Project,Value=${PROJECT_NAME}" \ + "Key=ManagedBy,Value=lambda-microvm-github-runner" \ + >/dev/null +fi + +jq -n '{ + Version: "2012-10-17", + Statement: [{ + Effect: "Allow", + Principal: {Service: "lambda.amazonaws.com"}, + Action: ["sts:AssumeRole", "sts:TagSession"] + }] +}' >"${temporary_directory}/lambda-trust.json" + +jq -n \ + --arg provider "${OIDC_PROVIDER_ARN}" \ + --arg subject "${GITHUB_OIDC_SUBJECT}" \ + '{ + Version: "2012-10-17", + Statement: [{ + Effect: "Allow", + Principal: {Federated: $provider}, + Action: "sts:AssumeRoleWithWebIdentity", + Condition: { + StringEquals: { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com", + "token.actions.githubusercontent.com:sub": $subject + } + } + }] + }' >"${temporary_directory}/github-trust.json" + +jq -n \ + --arg objectArn "arn:${partition}:s3:::${ARTIFACT_BUCKET}/*" \ + --arg logsArn "arn:${partition}:logs:${REGION}:${account_id}:log-group:${BUILD_LOG_GROUP}:*" \ + '{ + Version: "2012-10-17", + Statement: [ + { + Sid: "ReadImageArtifact", + Effect: "Allow", + Action: ["s3:GetObject", "s3:GetObjectVersion"], + Resource: $objectArn + }, + { + Sid: "WriteBuildLogs", + Effect: "Allow", + Action: ["logs:CreateLogStream", "logs:PutLogEvents"], + Resource: $logsArn + } + ] + }' >"${temporary_directory}/build-permissions.json" + +jq -n \ + --arg logsArn "arn:${partition}:logs:${REGION}:${account_id}:log-group:${RUNTIME_LOG_GROUP}:*" \ + --arg microvmArn "${MICROVM_RESOURCE_ARN}" \ + '{ + Version: "2012-10-17", + Statement: [ + { + Sid: "WriteRuntimeLogs", + Effect: "Allow", + Action: ["logs:CreateLogStream", "logs:PutLogEvents"], + Resource: $logsArn + }, + { + Sid: "TerminateSelf", + Effect: "Allow", + Action: "lambda:TerminateMicrovm", + Resource: $microvmArn + } + ] + }' >"${temporary_directory}/runtime-permissions.json" + +jq -n \ + --arg imageArn "${IMAGE_RESOURCE_ARN}" \ + --arg microvmArn "${MICROVM_RESOURCE_ARN}" \ + --arg executionRoleArn "${EXECUTION_ROLE_ARN}" \ + --arg internetEgressArn "${INTERNET_EGRESS_ARN}" \ + --arg noIngressArn "${NO_INGRESS_ARN}" \ + '{ + Version: "2012-10-17", + Statement: [ + { + Sid: "ManageRunnerMicrovms", + Effect: "Allow", + Action: [ + "lambda:RunMicrovm", + "lambda:GetMicrovm", + "lambda:TerminateMicrovm", + "lambda:GetMicrovmImage" + ], + Resource: [$imageArn, $microvmArn] + }, + { + Sid: "PassRuntimeRole", + Effect: "Allow", + Action: "iam:PassRole", + Resource: $executionRoleArn + }, + { + Sid: "PassManagedNetworkConnectors", + Effect: "Allow", + Action: "lambda:PassNetworkConnector", + Resource: [$internetEgressArn, $noIngressArn] + } + ] + }' >"${temporary_directory}/github-permissions.json" + +upsert_role \ + "${BUILD_ROLE_NAME}" \ + "${temporary_directory}/lambda-trust.json" \ + "${temporary_directory}/build-permissions.json" \ + "Builds the Lambda MicroVM GitHub runner image" +upsert_role \ + "${EXECUTION_ROLE_NAME}" \ + "${temporary_directory}/lambda-trust.json" \ + "${temporary_directory}/runtime-permissions.json" \ + "Runtime permissions for single-use Lambda MicroVM GitHub runners" +upsert_role \ + "${GITHUB_ROLE_NAME}" \ + "${temporary_directory}/github-trust.json" \ + "${temporary_directory}/github-permissions.json" \ + "GitHub OIDC role for launching and terminating runner MicroVMs" + +mkdir -p "$(dirname -- "${OUTPUT_FILE}")" +jq -n \ + --arg region "${REGION}" \ + --arg artifactBucket "${ARTIFACT_BUCKET}" \ + --arg buildRoleArn "${BUILD_ROLE_ARN}" \ + --arg executionRoleArn "${EXECUTION_ROLE_ARN}" \ + --arg githubLaunchRoleArn "${GITHUB_ROLE_ARN}" \ + --arg buildLogGroup "${BUILD_LOG_GROUP}" \ + --arg runtimeLogGroup "${RUNTIME_LOG_GROUP}" \ + --arg githubOidcSubject "${GITHUB_OIDC_SUBJECT}" \ + '{ + region: $region, + artifactBucket: $artifactBucket, + buildRoleArn: $buildRoleArn, + executionRoleArn: $executionRoleArn, + githubLaunchRoleArn: $githubLaunchRoleArn, + buildLogGroup: $buildLogGroup, + runtimeLogGroup: $runtimeLogGroup, + githubOidcSubject: $githubOidcSubject + }' | tee "${OUTPUT_FILE}" + +log "AWS setup saved to ${OUTPUT_FILE}" diff --git a/scripts/build-microvm-image.sh b/scripts/build-microvm-image.sh new file mode 100755 index 0000000..26a6939 --- /dev/null +++ b/scripts/build-microvm-image.sh @@ -0,0 +1,257 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +REPOSITORY_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" +readonly REPOSITORY_ROOT +readonly SETUP_FILE="${AWS_SETUP_FILE:-${REPOSITORY_ROOT}/build/aws-setup.json}" +readonly OUTPUT_FILE="${OUTPUT_FILE:-${REPOSITORY_ROOT}/build/microvm-image.json}" + +setup_region="" +setup_artifact_bucket="" +setup_build_role_arn="" +setup_build_log_group="" +if [[ -f "${SETUP_FILE}" ]]; then + command -v jq >/dev/null 2>&1 || { + echo "Required command is unavailable: jq" >&2 + exit 1 + } + setup_region="$(jq -r '.region // empty' "${SETUP_FILE}")" + setup_artifact_bucket="$(jq -r '.artifactBucket // empty' "${SETUP_FILE}")" + setup_build_role_arn="$(jq -r '.buildRoleArn // empty' "${SETUP_FILE}")" + setup_build_log_group="$(jq -r '.buildLogGroup // empty' "${SETUP_FILE}")" +fi + +readonly REGION="${AWS_REGION:-${AWS_DEFAULT_REGION:-${setup_region}}}" +readonly ARTIFACT_BUCKET="${ARTIFACT_BUCKET:-${setup_artifact_bucket}}" +readonly BUILD_ROLE_ARN="${BUILD_ROLE_ARN:-${setup_build_role_arn}}" +readonly BUILD_LOG_GROUP="${BUILD_LOG_GROUP:-${setup_build_log_group}}" +readonly NAME="${IMAGE_NAME:-lambda-microvm-github-runner}" +readonly MEMORY_MIB="${MEMORY_MIB:-4096}" +readonly RETAIN_VERSIONS="${RETAIN_VERSIONS:-5}" +readonly BUILD_TIMEOUT_SECONDS="${BUILD_TIMEOUT_SECONDS:-1800}" +readonly BASE_IMAGE_ARN="${BASE_IMAGE_ARN:-arn:aws:lambda:${REGION}:aws:microvm-image:al2023-1}" +readonly INTERNET_EGRESS="${INTERNET_EGRESS_CONNECTOR_ARN:-arn:aws:lambda:${REGION}:aws:network-connector:aws-network-connector:INTERNET_EGRESS}" +artifact_override="${ARTIFACT:-}" +readonly artifact_override +readonly ARTIFACT="${artifact_override:-${REPOSITORY_ROOT}/build/runner-image.zip}" +readonly HOOKS='{ + "port": 9000, + "microvmImageHooks": { + "ready": "ENABLED", + "readyTimeoutInSeconds": 120, + "validate": "ENABLED", + "validateTimeoutInSeconds": 300 + }, + "microvmHooks": { + "run": "ENABLED", + "runTimeoutInSeconds": 60, + "resume": "ENABLED", + "resumeTimeoutInSeconds": 60, + "suspend": "ENABLED", + "suspendTimeoutInSeconds": 30, + "terminate": "ENABLED", + "terminateTimeoutInSeconds": 30 + } +}' + +: "${REGION:?Set AWS_REGION or AWS_DEFAULT_REGION}" +: "${ARTIFACT_BUCKET:?Set ARTIFACT_BUCKET}" +: "${BUILD_ROLE_ARN:?Set BUILD_ROLE_ARN}" + +[[ "${NAME}" =~ ^[A-Za-z0-9_-]{1,64}$ ]] || { + echo "IMAGE_NAME must match [A-Za-z0-9_-] and be at most 64 characters" >&2 + exit 1 +} +if ! [[ "${MEMORY_MIB}" =~ ^[0-9]+$ ]] || ((MEMORY_MIB < 1024)); then + echo "MEMORY_MIB must be an integer of at least 1024" >&2 + exit 1 +fi +if ! [[ "${RETAIN_VERSIONS}" =~ ^[0-9]+$ ]] || + ((RETAIN_VERSIONS < 1)); then + echo "RETAIN_VERSIONS must be a positive integer" >&2 + exit 1 +fi +if ! [[ "${BUILD_TIMEOUT_SECONDS}" =~ ^[0-9]+$ ]] || + ((BUILD_TIMEOUT_SECONDS < 300)); then + echo "BUILD_TIMEOUT_SECONDS must be an integer of at least 300" >&2 + exit 1 +fi + +for command in aws jq shasum; do + command -v "${command}" >/dev/null 2>&1 || { + echo "Required command is unavailable: ${command}" >&2 + exit 1 + } +done + +if [[ -z "${artifact_override}" ]]; then + "${SCRIPT_DIR}/package-runner-image.sh" "${ARTIFACT}" >/dev/null +elif [[ ! -f "${ARTIFACT}" ]]; then + echo "ARTIFACT does not exist: ${ARTIFACT}" >&2 + exit 1 +fi + +ARTIFACT_SHA="$(shasum -a 256 "${ARTIFACT}" | awk '{print $1}')" +readonly ARTIFACT_SHA +readonly ARTIFACT_KEY="${NAME}/${ARTIFACT_SHA}.zip" +readonly CLIENT_TOKEN="image-${ARTIFACT_SHA:0:64}" + +export AWS_MAX_ATTEMPTS=6 +export AWS_RETRY_MODE=standard +export AWS_PAGER="" + +aws s3 cp \ + --region "${REGION}" \ + --only-show-errors \ + "${ARTIFACT}" \ + "s3://${ARTIFACT_BUCKET}/${ARTIFACT_KEY}" + +common_arguments=( + --region "${REGION}" + --base-image-arn "${BASE_IMAGE_ARN}" + --build-role-arn "${BUILD_ROLE_ARN}" + --code-artifact "uri=s3://${ARTIFACT_BUCKET}/${ARTIFACT_KEY}" + --description "Single-use GitHub Actions runner" + --logging "cloudWatch={logGroup=${BUILD_LOG_GROUP:-/lambda-microvms/${NAME}/build}}" + --egress-network-connectors "[\"${INTERNET_EGRESS}\"]" + --cpu-configurations '[{"architecture":"ARM_64"}]' + --resources "[{\"minimumMemoryInMiB\":${MEMORY_MIB}}]" + --additional-os-capabilities '["ALL"]' + --hooks "${HOOKS}" + --client-token "${CLIENT_TOKEN}" + --cli-connect-timeout 10 + --cli-read-timeout 60 +) + +image_arn="$( + aws lambda-microvms list-microvm-images \ + --region "${REGION}" \ + --query "items[?name=='${NAME}'].imageArn | [0]" \ + --output text +)" + +if [[ -n "${image_arn}" && "${image_arn}" != "None" ]]; then + echo "Updating ${image_arn}" >&2 + read -r image_arn image_version < <( + aws lambda-microvms update-microvm-image \ + --image-identifier "${image_arn}" \ + "${common_arguments[@]}" \ + --query '[imageArn,imageVersion]' \ + --output text + ) +else + echo "Creating ${NAME}" >&2 + read -r image_arn image_version < <( + aws lambda-microvms create-microvm-image \ + --name "${NAME}" \ + --tags "Project=${NAME},ManagedBy=lambda-microvm-github-runner" \ + "${common_arguments[@]}" \ + --query '[imageArn,imageVersion]' \ + --output text + ) +fi + +if [[ -z "${image_arn}" || "${image_arn}" == "None" || + -z "${image_version}" || "${image_version}" == "None" ]]; then + echo "AWS did not return an image ARN and version" >&2 + exit 1 +fi + +deadline=$((SECONDS + BUILD_TIMEOUT_SECONDS)) +while ((SECONDS < deadline)); do + read -r state status < <( + aws lambda-microvms get-microvm-image-version \ + --region "${REGION}" \ + --image-identifier "${image_arn}" \ + --image-version "${image_version}" \ + --query '[state,status]' \ + --output text + ) + echo "Image version ${image_version}: state=${state} status=${status}" >&2 + case "${state}" in + SUCCESSFUL) + break + ;; + FAILED | DELETED | DELETE_FAILED) + echo "Image build failed; inspect ${BUILD_LOG_GROUP:-/lambda-microvms/${NAME}/build}" >&2 + exit 1 + ;; + esac + sleep $((3 + RANDOM % 5)) +done + +if [[ "${state:-}" != "SUCCESSFUL" ]]; then + echo "Timed out waiting for image version ${image_version}" >&2 + exit 1 +fi + +aws lambda-microvms update-microvm-image-version \ + --region "${REGION}" \ + --image-identifier "${image_arn}" \ + --image-version "${image_version}" \ + --status ACTIVE \ + >/dev/null + +previous_active_versions=() +previous_active_count=0 +while IFS= read -r version; do + previous_active_versions[previous_active_count]="${version}" + previous_active_count=$((previous_active_count + 1)) +done < <( + aws lambda-microvms list-microvm-image-versions \ + --region "${REGION}" \ + --image-identifier "${image_arn}" \ + --query "items[?status=='ACTIVE' && imageVersion!='${image_version}'].imageVersion" \ + --output text | + tr '\t' '\n' | + sed '/^$/d' +) +for ((index = 0; index < previous_active_count; index++)); do + version="${previous_active_versions[index]}" + aws lambda-microvms update-microvm-image-version \ + --region "${REGION}" \ + --image-identifier "${image_arn}" \ + --image-version "${version}" \ + --status INACTIVE \ + >/dev/null +done + +inactive_versions=() +inactive_count=0 +while IFS= read -r version; do + inactive_versions[inactive_count]="${version}" + inactive_count=$((inactive_count + 1)) +done < <( + aws lambda-microvms list-microvm-image-versions \ + --region "${REGION}" \ + --image-identifier "${image_arn}" \ + --query "items[?status=='INACTIVE' && state=='SUCCESSFUL'].imageVersion" \ + --output text | + tr '\t' '\n' | + sed '/^$/d' | + sort -Vr +) +for ((index = RETAIN_VERSIONS - 1; index < inactive_count; index++)); do + aws lambda-microvms delete-microvm-image-version \ + --region "${REGION}" \ + --image-identifier "${image_arn}" \ + --image-version "${inactive_versions[index]}" \ + >/dev/null +done + +mkdir -p "$(dirname -- "${OUTPUT_FILE}")" +jq -n \ + --arg imageArn "${image_arn}" \ + --arg imageVersion "${image_version}" \ + --arg region "${REGION}" \ + --arg artifactSha256 "${ARTIFACT_SHA}" \ + '{ + imageArn: $imageArn, + imageVersion: $imageVersion, + region: $region, + artifactSha256: $artifactSha256 + }' | tee "${OUTPUT_FILE}" + +echo "MicroVM image details saved to ${OUTPUT_FILE}" >&2 diff --git a/scripts/configure-github.sh b/scripts/configure-github.sh new file mode 100755 index 0000000..2d5aa6e --- /dev/null +++ b/scripts/configure-github.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +REPOSITORY_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" +readonly REPOSITORY_ROOT +readonly SETUP_FILE="${AWS_SETUP_FILE:-${REPOSITORY_ROOT}/build/aws-setup.json}" +readonly IMAGE_FILE="${MICROVM_IMAGE_FILE:-${REPOSITORY_ROOT}/build/microvm-image.json}" + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +for command in gh jq; do + command -v "${command}" >/dev/null 2>&1 || + fail "Required command is unavailable: ${command}" +done + +[[ -f "${SETUP_FILE}" ]] || + fail "Run scripts/bootstrap-aws.sh first; ${SETUP_FILE} does not exist" +[[ -f "${IMAGE_FILE}" ]] || + fail "Run scripts/build-microvm-image.sh first; ${IMAGE_FILE} does not exist" + +repository="${GITHUB_REPOSITORY:-}" +if [[ -z "${repository}" ]]; then + repository="$(gh repo view --json nameWithOwner --jq '.nameWithOwner')" +fi +[[ "${repository}" =~ ^[^/[:space:]]+/[^/[:space:]]+$ ]] || + fail "GITHUB_REPOSITORY must use owner/repository format" + +set_variable() { + local name="$1" + local value="$2" + [[ -n "${value}" && "${value}" != "null" ]] || + fail "Cannot set ${name}: source value is empty" + gh variable set "${name}" --repo "${repository}" --body "${value}" + echo "Set ${name}" >&2 +} + +set_variable \ + MICROVM_AWS_REGION \ + "$(jq -r '.region // empty' "${SETUP_FILE}")" +set_variable \ + MICROVM_LAUNCH_ROLE_ARN \ + "$(jq -r '.githubLaunchRoleArn // empty' "${SETUP_FILE}")" +set_variable \ + MICROVM_EXECUTION_ROLE_ARN \ + "$(jq -r '.executionRoleArn // empty' "${SETUP_FILE}")" +set_variable \ + MICROVM_RUNTIME_LOG_GROUP \ + "$(jq -r '.runtimeLogGroup // empty' "${SETUP_FILE}")" +set_variable \ + MICROVM_RUNNER_IMAGE_ARN \ + "$(jq -r '.imageArn // empty' "${IMAGE_FILE}")" +set_variable \ + MICROVM_RUNNER_IMAGE_VERSION \ + "$(jq -r '.imageVersion // empty' "${IMAGE_FILE}")" + +if [[ -n "${RUNNER_APP_ID:-}" ]]; then + set_variable RUNNER_APP_ID "${RUNNER_APP_ID}" +fi + +if [[ -n "${RUNNER_APP_PRIVATE_KEY_FILE:-}" ]]; then + [[ -f "${RUNNER_APP_PRIVATE_KEY_FILE}" ]] || + fail "RUNNER_APP_PRIVATE_KEY_FILE does not exist" + gh secret set RUNNER_APP_PRIVATE_KEY \ + --repo "${repository}" \ + <"${RUNNER_APP_PRIVATE_KEY_FILE}" + echo "Set RUNNER_APP_PRIVATE_KEY" >&2 +fi + +echo "Configured GitHub repository ${repository}" diff --git a/scripts/package-runner-image.sh b/scripts/package-runner-image.sh new file mode 100755 index 0000000..fd32de0 --- /dev/null +++ b/scripts/package-runner-image.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +REPOSITORY_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" +readonly REPOSITORY_ROOT +readonly SOURCE_DIR="${REPOSITORY_ROOT}/runner-image" +readonly OUTPUT="${1:-${REPOSITORY_ROOT}/build/runner-image.zip}" + +for command in cp mktemp touch zip; do + command -v "${command}" >/dev/null 2>&1 || { + echo "Required command is unavailable: ${command}" >&2 + exit 1 + } +done + +temporary_directory="$(mktemp -d)" +cleanup() { + rm -rf "${temporary_directory}" +} +trap cleanup EXIT + +cp "${SOURCE_DIR}/Dockerfile" "${temporary_directory}/Dockerfile" +cp "${SOURCE_DIR}/supervisor.py" "${temporary_directory}/supervisor.py" + +# Fixed metadata makes the artifact reproducible across clean checkouts. +touch -t 202601010000 \ + "${temporary_directory}/Dockerfile" \ + "${temporary_directory}/supervisor.py" +mkdir -p "$(dirname -- "${OUTPUT}")" +rm -f "${OUTPUT}" +( + cd "${temporary_directory}" + zip -X -q "${OUTPUT}" Dockerfile supervisor.py +) + +echo "${OUTPUT}" +shasum -a 256 "${OUTPUT}" diff --git a/scripts/test-runner-image.sh b/scripts/test-runner-image.sh new file mode 100755 index 0000000..359d071 --- /dev/null +++ b/scripts/test-runner-image.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +REPOSITORY_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" +readonly REPOSITORY_ROOT +readonly IMAGE="${RUNNER_IMAGE_TEST_TAG:-lambda-microvm-github-runner:test}" +readonly FIXTURE_DIR="${REPOSITORY_ROOT}/runner-image/test/fixtures" + +for command in curl docker python3 rg; do + command -v "${command}" >/dev/null 2>&1 || { + echo "Required command is unavailable: ${command}" >&2 + exit 1 + } +done + +docker build --tag "${IMAGE}" "${REPOSITORY_ROOT}/runner-image" + +container_id="" +temporary_directory="$(mktemp -d)" +cleanup() { + if [[ -n "${container_id}" ]]; then + docker rm --force "${container_id}" >/dev/null 2>&1 || true + fi + rm -rf "${temporary_directory}" +} +trap cleanup EXIT + +start_container() { + container_id="$( + docker run \ + --rm \ + --detach \ + "$@" \ + --publish 127.0.0.1::9000 \ + "${IMAGE}" + )" + port="$(docker port "${container_id}" 9000/tcp | sed 's/.*://')" + for _attempt in $(seq 1 50); do + if curl --fail --silent "http://127.0.0.1:${port}/healthz" >/dev/null; then + return + fi + sleep 0.2 + done + echo "Supervisor health check timed out" >&2 + exit 1 +} + +stop_container() { + docker rm --force "${container_id}" >/dev/null + container_id="" +} + +hook() { + local name="$1" + local body="$2" + curl \ + --silent \ + --output "${temporary_directory}/${name}.out" \ + --write-out '%{http_code}' \ + --request POST \ + --header 'Content-Type: application/json' \ + --data "${body}" \ + "http://127.0.0.1:${port}${HOOK_PREFIX}/${name}" +} + +readonly HOOK_PREFIX="/aws/lambda-microvms/runtime/v1" + +start_container +[[ "$(hook ready '{}')" == "200" ]] +docker exec "${container_id}" test ! -S /var/run/docker.sock +stop_container + +start_container --privileged --env ALLOW_VFS_FALLBACK=true +[[ "$(hook validate '{}')" == "503" ]] +for _attempt in $(seq 1 120); do + validation_status="$(hook validate '{}')" + [[ "${validation_status}" == "200" ]] && break + if rg -q 'validation failed' "${temporary_directory}/validate.out"; then + exit 1 + fi + sleep 1 +done +[[ "${validation_status}" == "200" ]] +[[ "$(docker exec "${container_id}" docker info --format '{{.Driver}}')" == "vfs" ]] +docker exec "${container_id}" \ + docker run --rm public.ecr.aws/docker/library/busybox:1.37.0 true +stop_container + +start_container \ + --privileged \ + --env ALLOW_VFS_FALLBACK=true \ + --env RUNNER_ROOT=/opt/fake-runner \ + --volume "${FIXTURE_DIR}:/opt/fake-runner:ro" +payload="$( + python3 -c 'import base64,gzip,json; print(json.dumps({"microvmId":"mvm-local-fixture","runHookPayload":base64.b64encode(gzip.compress(b"fake-jit-config")).decode()}))' +)" +[[ "$(hook run "${payload}")" == "200" ]] +[[ "$(hook resume '{}')" == "200" ]] +[[ "$(hook suspend '{}')" == "200" ]] +[[ "$(hook terminate '{}')" == "200" ]] +for _attempt in $(seq 1 50); do + if ! docker exec "${container_id}" pgrep -x sleep >/dev/null 2>&1; then + break + fi + sleep 0.1 +done +if docker exec "${container_id}" pgrep -x sleep >/dev/null 2>&1; then + echo "Fake runner process survived termination" >&2 + exit 1 +fi +if docker exec "${container_id}" docker info >/dev/null 2>&1; then + echo "Docker daemon survived termination" >&2 + exit 1 +fi +docker logs "${container_id}" >"${temporary_directory}/container.log" 2>&1 +if rg -q 'fake-jit-config' "${temporary_directory}/container.log"; then + echo "JIT fixture content appeared in logs" >&2 + exit 1 +fi + +echo "Runner image smoke tests passed" From d1b53eb7ed75d05362425eb75c8dd5e8bda1b833 Mon Sep 17 00:00:00 2001 From: Jason Aricheta Date: Fri, 3 Jul 2026 14:08:48 +1200 Subject: [PATCH 2/2] fix: make npm lock portable to Linux --- package-lock.json | 26 +++++++++++++++++++++++++- package.json | 2 ++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 2a8eaa1..a164b0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,8 @@ "@aws-sdk/client-lambda-microvms": "^3.1079.0" }, "devDependencies": { + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", "@eslint/js": "10.0.1", "@types/node": "24.13.2", "@vercel/ncc": "0.44.1", @@ -359,13 +361,35 @@ "node": ">=18.0.0" } }, + "node_modules/@emnapi/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@emnapi/wasi-threads": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", "dev": true, "license": "MIT", - "optional": true, "dependencies": { "tslib": "^2.4.0" } diff --git a/package.json b/package.json index c7f3d93..71241c4 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,8 @@ "@aws-sdk/client-lambda-microvms": "^3.1079.0" }, "devDependencies": { + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", "@eslint/js": "10.0.1", "@types/node": "24.13.2", "@vercel/ncc": "0.44.1",