Fix live runner lifecycle cleanup

.github/workflows/aws-runner-target.yml

@@ -19,4 +19,5 @@ jobs:
           docker info
           docker buildx version
           docker compose version
+          aws sts get-caller-identity >/dev/null
           docker run --rm public.ecr.aws/docker/library/busybox:1.37.0 true

.github/workflows/ci.yml

@@ -59,4 +59,5 @@ jobs:
               docker buildx version
               docker compose version
               aws --version
+              aws lambda-microvms help >/dev/null
             '

docs/security.md

@@ -15,6 +15,9 @@ and trusted workflow changes only. Public fork pull requests are unsupported.
 - GitHub-hosted start and stop jobs obtain AWS credentials through OIDC.
 - The default MicroVM execution role can write its logs and terminate runner
   MicroVMs. It has no application deployment permissions.
+- The runtime role's only unscoped resource permission is
+  `lambda:TerminateMicrovm`, because that API does not expose a per-instance IAM
+  resource ARN. No other Lambda or application action is granted by it.
 - Deployment jobs should assume a separate role through GitHub OIDC.
 
 The GitHub App private key never enters the MicroVM. No long-lived AWS key is

runner-image/Dockerfile

@@ -7,11 +7,11 @@ ARG BUILDX_SHA256=c4248d6cbc4a619a7e0b4609c11e509ad4ac0b475e1c64817c0ac20c5d90c7
 ARG COMPOSE_VERSION=5.3.0
 ARG COMPOSE_SHA256=ba0d9f5ce70086b3830448ce2f8a6405513c996065fe45d2f7c144a1f0d99398
 ARG DOCKER_PACKAGE=docker-25.0.16-1.amzn2023.0.2.aarch64
-ARG AWSCLI_PACKAGE=awscli-2-2.33.15-1.amzn2023.0.1.noarch
+ARG AWSCLI_VERSION=2.35.13
+ARG AWSCLI_SHA256=c562b596fa9711f340c82307ca9eff576a4b96e75f1a003d1e6b2fec77820883
 
 RUN dnf install -y --setopt=install_weak_deps=0 \
       "${DOCKER_PACKAGE}" \
-      "${AWSCLI_PACKAGE}" \
       bzip2 \
       ca-certificates \
       curl-minimal \
@@ -36,6 +36,18 @@ RUN dnf install -y --setopt=install_weak_deps=0 \
   && dnf clean all \
   && rm -rf /var/cache/dnf /var/cache/libdnf5
 
+RUN set -eux; \
+    awscli_archive="/tmp/awscliv2.zip"; \
+    curl --fail --location --silent --show-error \
+      "https://awscli.amazonaws.com/awscli-exe-linux-aarch64-${AWSCLI_VERSION}.zip" \
+      --output "${awscli_archive}"; \
+    echo "${AWSCLI_SHA256}  ${awscli_archive}" | sha256sum --check --strict; \
+    unzip -q "${awscli_archive}" -d /tmp; \
+    /tmp/aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli; \
+    rm -rf "${awscli_archive}" /tmp/aws; \
+    aws --version; \
+    aws lambda-microvms help >/dev/null
+
 RUN set -eux; \
     runner_archive="/tmp/actions-runner.tar.gz"; \
     curl --fail --location --silent --show-error \

runner-image/README.md

@@ -8,7 +8,7 @@ ARM64 throughout. The Dockerfile pins and verifies:
 - Docker Engine RPM 25.0.16;
 - Docker Buildx 0.35.0;
 - Docker Compose 5.3.0;
-- AWS CLI 2.33.15.
+- AWS CLI 2.35.13, including the `lambda-microvms` service model.
 
 The image snapshot contains no registered runner, JIT configuration, Docker
 daemon, or live network connection. The lifecycle supervisor starts Docker and
@@ -46,5 +46,5 @@ Production keeps `ALLOW_VFS_FALLBACK=false`; AWS image validation fails unless
 - `/run`, `/resume`, `/suspend`, and `/terminate` runtime hooks enabled.
 
 Runner launches should use managed `NO_INGRESS` and `INTERNET_EGRESS`. The
-MicroVM execution role needs CloudWatch log delivery and scoped
-`lambda:TerminateMicrovm` permission for the supervisor's cleanup call.
+MicroVM execution role needs CloudWatch log delivery and only the
+`lambda:TerminateMicrovm` action for the supervisor's cleanup call.

runner-image/supervisor.py

@@ -9,6 +9,7 @@
 import os
 import pwd
 import random
+import re
 import shutil
 import signal
 import subprocess
@@ -366,16 +367,17 @@ def __init__(
         self.sleeper = sleeper
         self.random_source = random_source
 
-    def terminate(self, microvm_id: str) -> bool:
+    def terminate(self, microvm_id: str, region: str | None = None) -> bool:
         command = [
             "aws",
             "lambda-microvms",
             "terminate-microvm",
             "--microvm-identifier",
             microvm_id,
         ]
-        if self.settings.aws_region:
-            command.extend(["--region", self.settings.aws_region])
+        selected_region = region or self.settings.aws_region
+        if selected_region:
+            command.extend(["--region", selected_region])
 
         for attempt in range(5):
             try:
@@ -426,7 +428,9 @@ def __init__(
 
     def run(self, payload: dict[str, Any]) -> bool:
         try:
-            microvm_id, encoded_jit_config = parse_run_payload(payload)
+            microvm_id, encoded_jit_config, aws_region = parse_run_payload(
+                payload
+            )
         except ValueError:
             log("run hook payload is invalid")
             return False
@@ -445,7 +449,9 @@ def run(self, payload: dict[str, Any]) -> bool:
             self.external_termination = False
 
         if not self.docker.start():
-            self._fail_start(microvm_id, "docker startup failed")
+            self._fail_start(
+                microvm_id, aws_region, "docker startup failed"
+            )
             return False
 
         with self.lock:
@@ -454,21 +460,25 @@ def run(self, payload: dict[str, Any]) -> bool:
         try:
             process = self.launcher.launch(encoded_jit_config)
         except OSError:
-            self._fail_start(microvm_id, "runner spawn failed")
+            self._fail_start(
+                microvm_id, aws_region, "runner spawn failed"
+            )
             return False
         finally:
             encoded_jit_config = ""
 
         if process.poll() is not None:
-            self._fail_start(microvm_id, "runner exited during startup")
+            self._fail_start(
+                microvm_id, aws_region, "runner exited during startup"
+            )
             return False
 
         with self.lock:
             self.runner_process = process
             self.state = RunnerState.RUNNING
             watcher = threading.Thread(
                 target=self._watch_runner,
-                args=(process, microvm_id),
+                args=(process, microvm_id, aws_region),
                 name="runner-watcher",
                 daemon=True,
             )
@@ -505,7 +515,10 @@ def shutdown(self) -> None:
         self.terminate()
 
     def _watch_runner(
-        self, process: subprocess.Popen[bytes], microvm_id: str
+        self,
+        process: subprocess.Popen[bytes],
+        microvm_id: str,
+        aws_region: str | None,
     ) -> None:
         exit_code = process.wait()
         process.args = ["runner", "***"]
@@ -515,16 +528,18 @@ def _watch_runner(
             self.state = RunnerState.TERMINATING
             should_self_terminate = not self.external_termination
         if should_self_terminate:
-            self.terminator.terminate(microvm_id)
+            self.terminator.terminate(microvm_id, aws_region)
 
-    def _fail_start(self, microvm_id: str, reason: str) -> None:
+    def _fail_start(
+        self, microvm_id: str, aws_region: str | None, reason: str
+    ) -> None:
         log(reason)
         self.docker.stop()
         with self.lock:
             self.state = RunnerState.FAILED
         thread = threading.Thread(
             target=self.terminator.terminate,
-            args=(microvm_id,),
+            args=(microvm_id, aws_region),
             name="failed-start-terminator",
             daemon=True,
         )
@@ -756,7 +771,9 @@ def _send_result(self, status: int, message: str) -> None:
         self.wfile.write(body)
 
 
-def parse_run_payload(payload: dict[str, Any]) -> tuple[str, str]:
+def parse_run_payload(
+    payload: dict[str, Any],
+) -> tuple[str, str, str | None]:
     microvm_id = payload.get("microvmId")
     run_hook_payload = payload.get("runHookPayload")
     if (
@@ -767,7 +784,28 @@ def parse_run_payload(payload: dict[str, Any]) -> tuple[str, str]:
         or not isinstance(run_hook_payload, str)
     ):
         raise ValueError("invalid run payload")
-    return microvm_id, decode_jit_payload(run_hook_payload)
+    encoded_jit_config, aws_region = decode_run_hook_payload(run_hook_payload)
+    return microvm_id, encoded_jit_config, aws_region
+
+
+def decode_run_hook_payload(payload: str) -> tuple[str, str | None]:
+    value = decode_jit_payload(payload)
+    try:
+        envelope = json.loads(value)
+    except json.JSONDecodeError:
+        return value, None
+    if not isinstance(envelope, dict) or envelope.get("version") != 1:
+        return value, None
+    encoded_jit_config = envelope.get("jit")
+    aws_region = envelope.get("region")
+    if (
+        not isinstance(encoded_jit_config, str)
+        or not encoded_jit_config
+        or not isinstance(aws_region, str)
+        or re.fullmatch(r"[a-z]{2}(?:-[a-z0-9]+)+-\d", aws_region) is None
+    ):
+        raise ValueError("invalid run hook envelope")
+    return encoded_jit_config, aws_region
 
 
 def decode_jit_payload(payload: str) -> str:

runner-image/test/test_supervisor.py

@@ -110,7 +110,9 @@ def __init__(self) -> None:
         self.calls: list[str] = []
         self.called = threading.Event()
 
-    def terminate(self, microvm_id: str) -> bool:
+    def terminate(
+        self, microvm_id: str, _region: str | None = None
+    ) -> bool:
         self.calls.append(microvm_id)
         self.called.set()
         return True
@@ -129,7 +131,16 @@ def test_payload_round_trip(self) -> None:
                     "runHookPayload": encoded_payload(secret),
                 }
             ),
-            ("mvm-123", secret),
+            ("mvm-123", secret, None),
+        )
+
+    def test_versioned_payload_includes_termination_region(self) -> None:
+        envelope = json.dumps(
+            {"version": 1, "jit": "jit-secret", "region": "us-east-1"}
+        )
+        self.assertEqual(
+            supervisor.decode_run_hook_payload(encoded_payload(envelope)),
+            ("jit-secret", "us-east-1"),
         )
 
     def test_payload_rejects_invalid_and_oversized_values(self) -> None:

scripts/bootstrap-aws.sh

@@ -143,7 +143,7 @@ upsert_role() {
   aws iam wait role-exists --role-name "${role_name}"
 }
 
-if aws s3api head-bucket --bucket "${ARTIFACT_BUCKET}" 2>/dev/null; then
+if aws s3api head-bucket --bucket "${ARTIFACT_BUCKET}" >/dev/null 2>&1; then
   log "Using S3 bucket ${ARTIFACT_BUCKET}"
 else
   log "Creating S3 bucket ${ARTIFACT_BUCKET}"
@@ -256,7 +256,6 @@ jq -n \
 
 jq -n \
   --arg logsArn "arn:${partition}:logs:${REGION}:${account_id}:log-group:${RUNTIME_LOG_GROUP}:*" \
-  --arg microvmArn "${MICROVM_RESOURCE_ARN}" \
   '{
     Version: "2012-10-17",
     Statement: [
@@ -270,7 +269,7 @@ jq -n \
         Sid: "TerminateSelf",
         Effect: "Allow",
         Action: "lambda:TerminateMicrovm",
-        Resource: $microvmArn
+        Resource: "*"
       }
     ]
   }' >"${temporary_directory}/runtime-permissions.json"

src/orchestration.ts

@@ -11,7 +11,7 @@ import {
   createRunnerLabels,
   type WorkflowIdentity,
 } from "./identity.js";
-import { encodeJitPayload } from "./payload.js";
+import { encodeRunHookPayload } from "./payload.js";
 import { pollSequentially } from "./polling.js";
 import {
   getSafeErrorName,
@@ -92,8 +92,9 @@ export async function startRunner(
     }
 
     stage = "JIT payload encoding";
-    const runHookPayload = encodeJitPayload(
+    const runHookPayload = encodeRunHookPayload(
       jitRunner.encodedJitConfig,
+      config.region,
       (secret) => {
         reporter.setSecret(secret);
       },

src/payload.ts

@@ -50,6 +50,28 @@ export function encodeJitPayload(
   return payload;
 }
 
+export function encodeRunHookPayload(
+  encodedJitConfig: string,
+  region: string,
+  maskSecret: SecretMasker,
+  maximumBytes = MAX_RUN_HOOK_PAYLOAD_BYTES,
+): string {
+  maskSecret(encodedJitConfig);
+  return encodeJitPayload(
+    JSON.stringify({
+      version: 1,
+      jit: encodedJitConfig,
+      region,
+    }),
+    (value) => {
+      if (value !== encodedJitConfig) {
+        maskSecret(value);
+      }
+    },
+    maximumBytes,
+  );
+}
+
 export function decodeJitPayload(payload: string): string {
   if (
     payload.length === 0 ||
@@ -60,10 +82,30 @@ export function decodeJitPayload(payload: string): string {
   }
 
   try {
-    return gunzipSync(Buffer.from(payload, "base64"), {
+    const decoded = gunzipSync(Buffer.from(payload, "base64"), {
       maxOutputLength: MAX_JIT_CONFIG_BYTES,
     }).toString("utf8");
+    const envelope = parseRunHookEnvelope(decoded);
+    return envelope ?? decoded;
   } catch {
     throw new PayloadEncodingError("payload decompression failed");
   }
 }
+
+function parseRunHookEnvelope(value: string): string | undefined {
+  try {
+    const parsed: unknown = JSON.parse(value);
+    if (
+      typeof parsed === "object" &&
+      parsed !== null &&
+      Reflect.get(parsed, "version") === 1 &&
+      typeof Reflect.get(parsed, "jit") === "string" &&
+      typeof Reflect.get(parsed, "region") === "string"
+    ) {
+      return Reflect.get(parsed, "jit") as string;
+    }
+  } catch {
+    return undefined;
+  }
+  return undefined;
+}

test/payload.test.ts

@@ -6,6 +6,7 @@ import { describe, expect, it, vi } from "vitest";
 import {
   decodeJitPayload,
   encodeJitPayload,
+  encodeRunHookPayload,
   MAX_RUN_HOOK_PAYLOAD_BYTES,
 } from "../src/payload.js";
 
@@ -37,6 +38,16 @@ describe("JIT payload codec", () => {
     ).toThrow(`${String(byteLength - 1)}-byte limit`);
   });
 
+  it("round-trips the versioned run hook envelope without exposing it", () => {
+    const jit = "encoded-jit-secret";
+    const mask = vi.fn();
+
+    const payload = encodeRunHookPayload(jit, "us-east-1", mask);
+
+    expect(decodeJitPayload(payload)).toBe(jit);
+    expect(mask).toHaveBeenCalledWith(payload);
+  });
+
   it("never includes the JIT value in errors", () => {
     const secret = randomBytes(4_096).toString("hex");
     let message = "";